"""
Damit die Daten eingelesen werden kÃ¶nnen:
Bitte speichern Sie den Ordner "opti", welcher alle Daten inkludiert, am selben Ort, an dem die Pythondatei abgespeichert ist.
Der opti Ordner muss die Unterordner "Mo_Ferien", "Mo_Schule", "Samstag" und "Sonntag" besitzen.
In den jeweiligen Ordnern mÃ¼ssen eine timetable.txt, timetable-blocks.txt und eine timetable-duties.txt Datei hinterlegt sein.

"""

#Sprint5.1

# Ã¼berall kleine Fehler im Output (Daten Schuld? DatenqualitÃ¤t Ã¼berprÃ¼fen)

# Bei DurchfÃ¼hrung Error in den Daten ausgefallen:
#   Bei VehID 9, 77, 63 sind die TeilumlÃ¤ufe nicht in korrekter Reihenfolge
#   Idee: Zeiten aufsteigend sortieren bei Transformieren

#######To-Dos########
## Prio 2: Mehrere Tage simulieren
## Prio 2: GUI optisch und inhaltlich verbessern (mehrere Dateien als AuswahlmÃ¶glichkeit)
    ## Prio 1: Stau einfÃ¼gen --> Erster Aufschlag implementiert
## Prio 1: Mehr und validere VerspÃ¤tungsarten
## Prio 1: ModularitÃ¤t ausbauen
    ## Prio 1: Dem Output die VerspÃ¤tungsart hinzufÃ¼gen --> Erster Aufschlag implementiert
    ## Prio 2: Einleseroutine hinzufÃ¼gen --> Done
## Prio 3: Dateninkonsistenzen beseitigen
## Prio 2: KPIs
## Prio 1: VerspÃ¤tungen in relative Werte wandeln (und runden)
## Prio 1: Pausenzeiten nutzen oder nicht?

## Zu klÃ¤ren:
## Statt mit absoluten VerspÃ¤tungen lieber mit relativen VerspÃ¤tungen arbeiten?
## Warum gibt es eine Haltestelle 244?

####################### Import Packages ###################################
import simpy
import pandas as pd
import numpy
from random import randint
from tkinter import *
import sys, datetime as dt
from io import StringIO

### GUI ###
root = Tk()
root.geometry("400x500")
l1 = Label(root,text='Berufsverkehr',background='grey', font = "Times")
var1 = IntVar()
c1 = Checkbutton(root, text='An diesem Tag gibt es Berufsverkehr.', variable=var1)
l2 = Label(root,text='Wetter',background='grey', font = "Times")
var2 = IntVar()
c2 = Checkbutton(root, text='An diesem Tag gibt es Gewitter.', variable=var2)
var3 = IntVar()
c3 = Checkbutton(root, text='An diesem Tag regnet es.', variable=var3)
var4 = IntVar()
c4 = Checkbutton(root, text='An diesem Tag scheint die Sonne.', variable=var4)
l3 = Label(root,text='Tag, der simuliert wird',background='grey', font = "Times")
var5 = IntVar()
c5 = Checkbutton(root, text='Wochentag - Keine Ferien', variable=var5)
var6 = IntVar()
c6 = Checkbutton(root, text='Wochentag - Ferien', variable=var6)
var7 = IntVar()
c7 = Checkbutton(root, text='Samstag', variable=var7)
var8 = IntVar()
c8 = Checkbutton(root, text='Sonntag', variable=var8)
button = Button(text = "BestÃ¤tigen", bg = "green", font = "Times", command = root.destroy)

l3.pack(fill=X, padx='20', pady='5')
c6.pack(side='top', fill='x', padx='5', pady='5', anchor = "w")
c7.pack(side='top', fill='x', padx='5', pady='5', anchor = "w")
c8.pack(side='top', fill='x', padx='5', pady='5', anchor = "w")
c5.pack(side='top', fill='x', padx='5', pady='5', anchor = "w")
l1.pack(fill=X, padx='20', pady='5')
c1.pack(side='top', fill='x', padx='5', pady='5', anchor = "w")
l2.pack(fill=X, padx='20', pady='5')
c2.pack(side='top', fill='x', padx='5', pady='5', anchor = "w")
c3.pack(side='top', fill='x', padx='5', pady='5', anchor = "w")
c4.pack(side='top', fill='x', padx='5', pady='5', anchor = "w")
button.pack(side='bottom', fill='y', padx='5',pady='10')

mainloop()

###Routine zum Auslesen der Textdateien
def load_table_data(fullfilepath):

    ### Read file
    tab_dict = {}
    with open(fullfilepath) as in_file:

    	### Iterate over rows

        while True:
            try:
                curr_row = in_file.readline()
            except IOError as e:
                sys.exit(1)

            if curr_row == '':      break    ### Empty (last) row found
            if curr_row[0] == '*':  continue ### Comment found
            if curr_row[0] == '$':           ### New table found
                split_list = curr_row.split(':')
                curr_row = split_list[1]
                tab_dict[split_list[0]] = ''

            ### Append current row to table string
            tab_dict[split_list[0]] = tab_dict[split_list[0]] + curr_row
        in_file.close()

    ### Create dict of pd.DataFrames
    tab_df_dict = {}
    for name, t in tab_dict.items():
        tab_df_dict[name] = (pd.read_csv(StringIO(t), sep=';'))
    return tab_df_dict

Weekday_No_Vacation = var5.get()
Weekday_Vacation = var6.get()
Saturday = var7.get()
Sunday = var8.get()

def dayChoice(Weekday_No_Vacation, Weekday_Vacation, Saturday, Sunday):
    if Weekday_No_Vacation == 1:
        data = load_table_data('opti/Mo_Schule/timetable.txt')
        blockdata = load_table_data('opti/Mo_Schule/timetable-blocks.txt')
        dutydata = load_table_data('opti/Mo_Schule/timetable-duties.txt')
        return(data, dutydata, blockdata)
    elif Weekday_Vacation == 1:
        data = load_table_data('opti/Mo_Ferien/timetable.txt')
        blockdata = load_table_data('opti/Mo_Ferien/timetable-blocks.txt')
        dutydata = load_table_data('opti/Mo_Ferien/timetable-duties.txt')
        return(data, dutydata, blockdata)
    elif Saturday == 1:
        data = load_table_data('opti/Samstag/timetable.txt')
        blockdata = load_table_data('opti/Samstag/timetable-blocks.txt')
        dutydata = load_table_data('opti/Samstag/timetable-duties.txt')
        return(data, dutydata, blockdata)
    elif Sunday == 1:
        data = load_table_data('opti/Sonntag/timetable.txt')
        blockdata = load_table_data('opti/Sonntag/timetable-blocks.txt')
        dutydata = load_table_data('opti/Sonntag/timetable-duties.txt')
        return(data, dutydata, blockdata)

data, dutydata, blockdata = dayChoice(Weekday_No_Vacation, Weekday_Vacation, Saturday, Sunday)

#relevante Dataframes extrahieren
servicefahrten = data["$SERVICEJOURNEY"]
dutyelements = dutydata["$DUTYELEMENT"]
blocks = blockdata["$BLOCK"]
blockelements = blockdata["$BLOCKELEMENT"]
dutytype = dutydata["$DUTY"]

allBlocks = blockelements
tableFinal = dutyelements
allLines = servicefahrten

tableFinal = tableFinal.drop(['ServiceJourneyCode', 'TransferType', 'GroupTransferIndex', 'DeadRunTransferBlockID', 'DeadRunTransferDutyID'], axis=1)
LineID = []

count_rows_duty = dutyelements.shape[0]
count_rows_dutytype = dutytype.shape[0]
count_rows_service = servicefahrten.shape[0]
count_blocks = blocks.shape[0]

for x in range(count_rows_duty):
    for y in range(count_rows_service):
        if tableFinal.iat[x,2] == allLines.iat[y,0]:
            LineID.append(allLines.iat[y,1])
    if tableFinal.iat[x,2] == -1:
            LineID.append(0)

VehTypeID = []
DepotID = []
for x in range(count_rows_duty):
    if tableFinal.iat[x,1] == -1:
        VehTypeID.append(tableFinal.iat[(x+1),1])
        DepotID.append(tableFinal.iat[x,4])
    else:
        for y in range(count_blocks):
            if tableFinal.iat[x,1] == blocks.iat[y,0]:
                VehTypeID.append(blocks.iat[y,1])
                DepotID.append(blocks.iat[y,2])

###Routine zum HinzufÃ¼gen des Duty Typs
df_dutytype = []
for x in range(count_rows_duty):
    for y in range(count_rows_dutytype):
        if tableFinal.iat[x,0] == dutytype.iat[y,0]:
            df_dutytype.append(dutytype.iat[y,1])

tableFinal["DutyType"] = df_dutytype
tableFinal["DepotID"] = DepotID
tableFinal["VehTypeID"] = VehTypeID
tableFinal["LineID"] = LineID

df = tableFinal
###export_csv = tableFinal.to_csv(r'TESTtableFinal.csv', index=None, header=True)

####################### Daten einlesen ####################################

#df = pd.read_csv("tableFinal.csv", sep=";")
####################### Daten transformieren und neue Zeitspalten in Dataframe einfÃ¼gen (Zeit) ########################
# Umrechnung der Start- & Endzeit in Minuten (fÃ¼r Simulationsuhr)
StartTime = []
count_row = df.shape[0]
for x in range(0, count_row):
    min_time = df.iloc[x, 5].split(":")
    minutes = int(min_time[1]) * 60
    minutes = minutes + int(min_time[2])
    StartTime.append(minutes)
EndTime = []
count_row = df.shape[0]
for x in range(0, count_row):
    min_time = df.iloc[x, 6].split(":")
    minutes = int(min_time[1]) * 60
    minutes = minutes + int(min_time[2])
    EndTime.append(minutes)
df["EndTime"] = EndTime
df["StartTime"] = StartTime

################################ Daten fÃ¼r Simulation erzeugen ##################################
# Gesamtanzahl an Fahrzeugen ermitteln
numberVeh = df["BlockID"].unique()
numberVeh = numberVeh.tolist()  # von Array in Liste formatieren
numberVeh = [elem for elem in numberVeh if elem >= 0]  # -1 entfernen
print("Die Gesamtanzahl von Fahrzeugen betrÃ¤gt %d." % len(numberVeh))

# DepotID (jedem Fahrzeug die unique DepotID zuordnen)
DepotID = []
counter = 1
for i in range(1, len(numberVeh) + 1):
    for j in range(0, df.shape[0]):
        if df.BlockID[j] == i and counter == i:
            DepotID.append(df.DepotID[j])
            counter += 1
print("Es gibt %d verschiedene Depots." %(len(set(DepotID))))
print("Jedes Fahrzeug wurde einem Depot zugeordnet: %s." % (
        len(DepotID) == len(numberVeh)))

# StartTimes der einzelnen Fahrzeuge in jedem Teilumlauf (in Dictionary gespeichert)
DepotID_plusOne = [0] + DepotID  # DepotIDListe verlÃ¤ngern, damit Schleife mit i funktioniert
startTime_dic = {}
for i in range(1, len(numberVeh) + 1):
    startTimes_Block = []
    for j in range(len(df)):
        if df.BlockID[j] == i:
            if df.FromStopID[j] == DepotID_plusOne[i]:
                startTimes_Block.append(df.StartTime[j])
    startTimes_Block = startTimes_Block + [
        1440]  # Add to every list a 1440 as last element for simulation loop (timeout)
    startTime_dic.update(
        {(i - 1): startTimes_Block})  # i-1 damit Index bei 0 anfÃ¤ngt (wichtig fÃ¼r Schleife (nachtrÃ¤gliche Ãnderung!)

StartTime_dic = startTime_dic

numTU_proF = []
for i in range(0,len(numberVeh)):
    numTU_proF.append(len(StartTime_dic[i])-1)
print("Die Fahrzeuge fahren im Schnitt %d TeilumlÃ¤ufe am Tag."
      " Das Maximum ist %d und das Minimum %d." % (sum(numTU_proF)/len(numTU_proF), max(numTU_proF), min(numTU_proF)))

# Listen von Haltestellen (from/to) (Liste mit vielen Listen)
FromStopID = []
ToStopID = []
for i in range(1, len(numberVeh) + 1):
    fromStopID_cache = []
    toStopID_cache = []
    for j in range(0, df.shape[0]):
        if df.BlockID[j] == i:
            fromStopID_one = df.FromStopID
            fromStopID_cache.append(fromStopID_one)
            toStopID_one = df.ToStopID
            toStopID_cache.append(toStopID_one)
    FromStopID.append(fromStopID_cache)
    ToStopID.append(toStopID_cache)

############ab hier Dictionaries erstellen #####################

DriveDuration_dic = {}
FromHS_dic = {}
ToHS_dic = {}
PartStartTime_dic = {}
PartEndTime_dic = {}
ElementID_dic = {}

for i in range(1, len(numberVeh) + 1):
    DifTime = []
    PartDif = []

    FromHS = []
    PartFromHS = []

    ToHS = []
    PartToHS = []

    StartTime = []
    PartStartTime = []

    EndTime = []
    PartEndTime = []

    Umlauf = []
    Teilumlauf = []

    for j in range(0, count_row):
        if df.BlockID[j] == i:

            difTime = df.EndTime[j] - df.StartTime[j]
            fromhs = df.FromStopID[j]
            tohs = df.ToStopID[j]
            stTime = df.StartTime[j]
            eTime = df.EndTime[j]
            elementID = df.ElementType[j]

            if df.ToStopID[j] == DepotID[i - 1]:
                PartDif.append(difTime)
                DifTime.append(PartDif)
                PartDif = []

                PartFromHS.append(fromhs)
                FromHS.append(PartFromHS)
                PartFromHS = []

                PartToHS.append(tohs)
                ToHS.append(PartToHS)
                PartToHS = []

                PartStartTime.append(stTime)
                StartTime.append(PartStartTime)
                PartStartTime = []

                PartEndTime.append(eTime)
                EndTime.append(PartEndTime)
                PartEndTime = []

                Teilumlauf.append(elementID)
                Umlauf.append(Teilumlauf)
                Teilumlauf = []

            else:
                PartDif.append(difTime)
                PartFromHS.append(fromhs)
                PartToHS.append(tohs)
                PartStartTime.append(stTime)
                PartEndTime.append(eTime)
                Teilumlauf.append(elementID)

    DriveDuration_dic.update({i - 1: DifTime})
    FromHS_dic.update({i - 1: FromHS})
    ToHS_dic.update({i - 1: ToHS})
    PartStartTime_dic.update({i - 1: StartTime})
    PartEndTime_dic.update({i - 1: EndTime})
    ElementID_dic.update({i - 1: Umlauf})

    ###################Art des Journeys#############################
Journey_dic = {}

for i in range(1,len(numberVeh) + 1):
    Umlauf = []
    Teilumlauf = []

    for j in range(0, count_row):
        if df.BlockID[j] == i:
            journey = df.iloc[j, 7]

            if df.ToStopID[j] == DepotID[i-1]:
                Teilumlauf.append(journey)
                Umlauf.append(Teilumlauf)
                Teilumlauf = []

            else:
                Teilumlauf.append(journey)

    Journey_dic.update({i-1: Umlauf})

###########Funktion um random Haltestellen auszusuchen, an denen Stau entsteht############
##Berechnet Anzahl Haltestellen
numberHS = 0
for i in range(0, count_row):
    if df.FromStopID[i] > numberHS:
        numberHS = df.FromStopID[i]

##Berechnet, welche Haltestellen am meisten frequentiert sind
HScount = []
for j in range(0, numberHS):
    count = 0
    for i in range(0, count_row):
        if df.FromStopID[i] == j:
            count += 1
    HScount.append(count)

##fÃ¼r die Annahme, dass ein Stau eher im Zentrum ist (Annahme, dass Zentrum dort ist, wo eine Haltestelle stark befahren wird)
cumulativeHS = []
cumulative = 0
for j in range(0, len(HScount)):
    cumulative += HScount[j]
    cumulativeHS.append(cumulative)

##Random mit gewichteter Wahrscheinlichkeit 10 Haltestellen, an denen Stau entsteht
Jam = []
for i in range(0,10):   ##Anzahl an Haltestellen mit Stau
    x = (randint(0, count_row))
    for j in range(0, len(cumulativeHS)):
        if x <= cumulativeHS[j]:
            Jam.append(j)
            break

############################## Funktionen fÃ¼r Objekt Vehicle #############################

# Abfrage: Fahrtzeit Ã¼ber Simulationsdauer
def drive_outOfTime(time, clock):
    doOT = time + clock > 1440
    return doOT

############################ StÃ¶rgenerator##################################
S = 5 # Einstellen des StÃ¶rfaktors

def stoerfaktor(s):  # n = Eingabeparameter um StÃ¶rausmaÃ zu steuern
    if (s == 0):
        factorX = numpy.random.choice(numpy.arange(0, 11), p=[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                                                              0.0])  # Wahrscheinlichkeiten von StÃ¶rungen / fÃ¼r jeden
        # Teilfahrt neuen StÃ¶rfaktor
    elif (s == 1):
        factorX = numpy.random.choice(numpy.arange(0, 11), p=[0.6, 0.2, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                                                              0.0])  # Wahrscheinlichkeiten von StÃ¶rungen / fÃ¼r jeden
        # Teilfahrt neuen StÃ¶rfaktor
    elif (s == 2):
        factorX = numpy.random.choice(numpy.arange(0, 11), p=[0.6, 0.0, 0.0, 0.2, 0.0, 0.2, 0.0, 0.0, 0.0, 0.0,
                                                              0.0])  # Wahrscheinlichkeiten von StÃ¶rungen / fÃ¼r jeden
        # Teilfahrt neuen StÃ¶rfaktor
    elif (s == 3):
        factorX = numpy.random.choice(numpy.arange(0, 11), p=[0.33, 0.0, 0.0, 0.0, 0.0, 0.33, 0.0, 0.0, 0.0, 0.0,
                                                              0.34])  # Wahrscheinlichkeiten von StÃ¶rungen / fÃ¼r
        # jeden Teilfahrt neuen StÃ¶rfaktor
    elif (s == 4):
        factorX = numpy.random.choice(numpy.arange(0, 11), p=[0.0, 0.0, 0.0, 0.0, 0.0, 0.5, 0.0, 0.0, 0.0, 0.0,
                                                              0.5])  # Wahrscheinlichkeiten von StÃ¶rungen / fÃ¼r
        # jeden Teilfahrt neuen StÃ¶rfaktor
    elif (s == 5):
        factorX = 20
    return factorX

##VerspÃ¤tungen auÃerhalb der Simulation bilden
"""
def verspÃ¤tung():

    for fzg in range(1,len(numberVeh) + 1):
        shift = 0
        for partNbr in range(0, len(StartTime_dic[fzg-1])-1):
            if partNbr > 0:                ##wenn es nicht der erste teilumlauf ist, kann es zu abhÃ¤ngigkeiten zw den umlÃ¤ufen kommen
                lastTime = len(PartEndTime_dic[fzg-1][partNbr-1])-1   ##Zeit des letzten Umlaufs
                ##Wenn Umlauf starten soll, obwohl letzter Umlauf noch nicht beendet:
                if PartStartTime_dic[fzg-1][partNbr][0] < PartEndTime_dic[fzg-1][partNbr-1][lastTime]:
                    shift += PartEndTime_dic[fzg-1][partNbr-1][lastTime] - PartStartTime_dic[fzg-1][partNbr][0]
                    PartStartTime_dic[fzg-1][partNbr][0] = PartEndTime_dic[fzg-1][partNbr-1][lastTime]
                ##wenn Umlauf starten soll und letzter Umlauf auch beendet ist: (Zeitverschiebung kann verringert werden)
                if PartStartTime_dic[fzg-1][partNbr][0] > PartEndTime_dic[fzg-1][partNbr-1][lastTime]:
                    shift -= PartStartTime_dic[fzg-1][partNbr][0] - PartEndTime_dic[fzg-1][partNbr-1][lastTime]
                if shift < 0:
                    shift = 0

                ##FÃ¼r jeden Teilumlauf die einzelnen Fahrten durchiterieren
            for journey in range(0, len(PartStartTime_dic[fzg-1][partNbr])):
                if journey > 0:
                    PartStartTime_dic[fzg-1][partNbr][journey] = PartEndTime_dic[fzg-1][partNbr][journey-1]
                ##Wenn es keine Standzeit ist:
                if Journey_dic[fzg-1][partNbr][journey] != 9:
                    delaytime = stoerfaktor(2) ##Hier werden die StÃ¶rfaktoren eingebaut
                    shift += delaytime
                    PartEndTime_dic[fzg-1][partNbr][journey] = PartStartTime_dic[fzg-1][partNbr][journey] + DriveDuration_dic[fzg-1][partNbr][journey] + delaytime
                ##Wenn es eine Standzeit ist:
                if Journey_dic[fzg-1][partNbr][journey] == 9:
                    ##Wenn Verschiebung grÃ¶Ãer/gleich der Pausenzeit
                    if DriveDuration_dic[fzg-1][partNbr][journey] < shift:
                        shift = shift - DriveDuration_dic[fzg-1][partNbr][journey]
                        PartEndTime_dic[fzg-1][partNbr][journey] = PartStartTime_dic[fzg-1][partNbr][journey]
                    ##Wenn Verschiebung kleiner Pausenzeit ist
                    elif DriveDuration_dic[fzg-1][partNbr][journey] >= shift:
                        if shift < 0:
                            shift = 0
                        PartEndTime_dic[fzg-1][partNbr][journey] = PartStartTime_dic[fzg-1][partNbr][journey] + DriveDuration_dic[fzg-1][partNbr][journey] - shift
                        shift = 0
                if shift < 0:
                    shift = 0

verspÃ¤tung()

"""

def carTraffic(time):
    if var1.get() == 1:
        if (time >= 390 and time <= 510) or (time >= 1110 and time <= 1200):
            return (stoerfaktor(4))
    ##oder ggf prozentual die duration erhÃ¶ren? (30% mehr zu den stoÃzeiten?)
        else:
            return (stoerfaktor(1))
    else:
        return(stoerfaktor(0))

def weather():
     if var2.get() == 1:
        return(stoerfaktor(4))
     elif var3.get() == 1:
        return(stoerfaktor(3))
     elif var4.get() == 1:
        return(stoerfaktor(1))
     else:
        return(stoerfaktor(0))

def trafficjam(FromHS, ToHS):
    if FromHS in Jam or ToHS in Jam:
        return(stoerfaktor(4))
    else:
        return(stoerfaktor(0))

def breaktime(vehID, teilumlaufnummer, fahrtnummer, delayTime):
    if ElementID_dic[vehID][teilumlaufnummer][fahrtnummer] == 9:
        DriveDuration_dic[vehID][teilumlaufnummer][fahrtnummer] = \
        DriveDuration_dic[vehID][teilumlaufnummer][fahrtnummer] - delayTime
        if DriveDuration_dic[vehID][teilumlaufnummer][fahrtnummer] < 0:
            delayTime = abs(DriveDuration_dic[vehID][teilumlaufnummer][fahrtnummer])
            DriveDuration_dic[vehID][teilumlaufnummer][fahrtnummer] = 0
        else:
            delayTime = 0
    return(delayTime)

def Type(weather, traffic, jam):
    delayType = ""
    if weather > 0:
        delayType += ("Wetter")
    if traffic > 0 and delayType == "":
        delayType += ("Verkehr")
    elif traffic > 0 and delayType != "":
        delayType += (",Verkehr")
    if jam > 0 and delayType == "":
        delayType += ("Stau")
    elif jam > 0 and delayType != "":
        delayType += (",Stau")
    else:
        delayType += ("-")
    return(delayType)

############################## Daten fÃ¼r CSV-Datei ###############################
# Header fÃ¼r CSV-Datei
print("vehID Teilumlaufnummer Standort Dep/Arr Uhrzeit(Soll) Uhrzeit(Ist) FahrtverspÃ¤tung GesamtverspÃ¤tung VerspÃ¤tungsursache",
      file=open("eventqueue5.2.csv", "a"))
########################## Objekt Vehicle #########################################
def vehicle(env, vehID):  # Eigenschaften von jedem Fahrzeug
    while True:
        delayTime = 0  # DelayTime initialisieren (gilt fÃ¼r den ganze Tag des Fahrzeugs)
        delayType = ""
        for teilumlaufnummer in range(0, len(StartTime_dic)-1):  #Loop der durch die einzelnen TeilumlÃ¤ufe fÃ¼hrt
            try:
                if StartTime_dic[vehID][teilumlaufnummer] - env.now >= 0:
                    delayTime = 0
                    delayType = ""
                    yield env.timeout(
                        StartTime_dic[vehID][teilumlaufnummer] - env.now)
                else:
                    yield env.timeout(0)
                    delayTime = env.now - StartTime_dic[vehID][teilumlaufnummer]
                    delayType = ""

                umlaufstatus = 1  # Wenn Startzeit erreicht, Fahrzeug im Umlauf (umlaufstatus = 1)

                while umlaufstatus == 1:  # while Fahrzeug im Umlauf
                    for fahrtnummer in range(0, len(FromHS_dic[vehID][teilumlaufnummer])):

                        if ElementID_dic[vehID][teilumlaufnummer][fahrtnummer] == 9:
                            fahrtstatus = 2
                        else:
                            fahrtstatus = 0 # 0 = Abfahrt / 1 = Ankunft / 2 = Pause

                        delayTime_perDrive = 0 #fÃ¼r PrintAusgabe hier definiert
                        print(vehID + 1, teilumlaufnummer + 1, FromHS_dic[vehID][teilumlaufnummer][fahrtnummer],
                              fahrtstatus, PartStartTime_dic[vehID][teilumlaufnummer][fahrtnummer], env.now,
                              "-", delayTime, "-",
                              file=open("eventqueue5.2.csv", "a"))

                        # VerspÃ¤tung auf Fahrt ermitteln
                        delayType = ""
                        if ElementID_dic[vehID][teilumlaufnummer][fahrtnummer] == 9:
                            delayTime_perDrive = 0
                            delayType = "-"
                        else:
                            delayWeather = weather()
                            delayCarTraffic = carTraffic(env.now)
                            delayTrafficJam = trafficjam(FromHS_dic[vehID][teilumlaufnummer][fahrtnummer], ToHS_dic[vehID][teilumlaufnummer][fahrtnummer])
                            delayTime_perDrive = delayWeather + delayCarTraffic + delayTrafficJam
                            # FÃ¼ge der Outputvariable "delayType" den Grund der VerspÃ¤tung hinzu
                            delayType = Type(delayWeather, delayCarTraffic, delayTrafficJam)

                        # Aufsummieren der VerspÃ¤tungen im Teilumlauf
                        delayTime += delayTime_perDrive

                        # Abfrage, ob Fahrt auÃerhalb der Simulationszeit liegen wÃ¼rde
                        if drive_outOfTime(
                                DriveDuration_dic[vehID][teilumlaufnummer][fahrtnummer], env.now):
                            yield env.timeout(1440)
                            break

                        # Abfrage, ob Pausenzeit & VerspÃ¤tungszeit anpassen
                        delayTime = breaktime(vehID, teilumlaufnummer, fahrtnummer, delayTime)

                        # Timeout fÃ¼r Fahrtdauer zur nÃ¤chsten Haltestelle
                        yield (env.timeout(DriveDuration_dic[vehID][teilumlaufnummer][fahrtnummer] + delayTime_perDrive))

                        if ElementID_dic[vehID][teilumlaufnummer][fahrtnummer] == 9:
                            fahrtstatus = 2
                        else:
                            fahrtstatus = 1

                        # Abfrage ob Bus im Depot angekommen
                        if ToHS_dic[vehID][teilumlaufnummer][fahrtnummer] == DepotID[vehID]:
                            print(vehID + 1, teilumlaufnummer + 1, ToHS_dic[vehID][teilumlaufnummer][fahrtnummer],
                                  fahrtstatus, PartEndTime_dic[vehID][teilumlaufnummer][fahrtnummer],
                                  env.now, delayTime_perDrive, delayTime, delayType,
                                  file=open("eventqueue5.2.csv", "a"))
                            umlaufstatus = 0
                        else:
                            print(vehID + 1, teilumlaufnummer + 1, ToHS_dic[vehID][teilumlaufnummer][fahrtnummer],
                                  fahrtstatus, PartEndTime_dic[vehID][teilumlaufnummer][fahrtnummer],
                                  env.now, delayTime_perDrive, delayTime, delayType,
                                  file=open("eventqueue5.2.csv", "a"))

            except:
                if env.now >= 1440: # to avoid RunTimeError: GeneratorExit
                    return False
                continue

            ########################## Simulationsumgebung ##############################

env = simpy.Environment()

# Initialisierung von Fahrzeugen
for i in range(0,len(numberVeh)):  # Anzahl von Fahrzeugen = len(numberVeh)
    env.process(vehicle(env, i))  # Inputdaten Eigenschaften Fahrzeugen
# Simulation starten und Laufzeit festlegen
env.run(until=1440)  # Ein Tag simulieren: in Minuten ausdrÃ¼cken. 24h = 1440min


from typing import Optional
from fastapi import APIRouter, Query
#from apis.api_components.wikipedia_fetch import wikipedia_fetch_prof
import urllib.request as req
import urllib.parse as up
#import lxml.html as lh
import json

router = APIRouter()

@router.get("/get_wikipedia_prof/")
def get_wikipedia_prof(q: str, count: Optional[int] = Query(None, ge=1)):
    """
    try:
        prof = wikipedia_fetch_prof(q)
    except Exception as e:
        # Wikipedia にページ/Infobox がない場合？
        prof = {}

    if count:
        # python>=3.7
        prof = dict(list(prof.items())[:count])
    """
    prof = {}
    try:
        s_url = 'https://ja.wikipedia.org/api/rest_v1/page/summary/' + up.quote(q)
        request = req.Request(url=s_url)
        res = req.urlopen(request, timeout=10).read()
        resd = json.loads(res.decode('utf-8'))
        prof = resd
    except:
        return None, 404

    return prof, 200


import scrapy
import requests
import sys
import pandas as pd
from scrapy.http import FormRequest
from scrapy.utils.response import open_in_browser
import json
from scrapy_splash import SplashRequest, SplashFormRequest

if sys.version_info[0] < 3:
    from StringIO import StringIO
else:
    from io import StringIO

script = """
function main(splash)
  splash:init_cookies(splash.args.cookies)
  assert(splash:go{
    splash.args.url,
    headers=splash.args.headers,
    http_method=splash.args.http_method,
    body=splash.args.body,
    })
  assert(splash:wait(0.5))

  local entries = splash:history()
  local last_response = entries[#entries].response
  return {
    url = splash:url(),
    headers = last_response.headers,
    http_status = last_response.status,
    cookies = splash:get_cookies(),
    html = splash:html(),
  }
end
"""

class PdonlineSpider(scrapy.Spider):
    name = 'pdonline'

    def start_requests(self):
        self.dynamic_cookie = {}

        yield SplashRequest(
            url="https://pdonline.brisbane.qld.gov.au/MasterPlan/Modules/Enquirer/PropertySearch.aspx",
            # cookies=self.dynamic_cookie,
            callback=self.search_result,
            endpoint='execute',
            cache_args=['lua_source'],
            args={
                'timeout': '90',
                'wait': '30',
                'lua_source': script,
            },
            session_id='1',
            meta={
                "input_data":{
                    'Lot Number':'',
                    'Plan Number': '',
                    'Unit Number To':'',
                    'Unit Number From':'',
                    'Street Name': 'Mcconaghy St',
                    'Street Number From': 92,
                    'Street Number To': 92,
                    'Suburb': 'Mitchelton'
                    },
                "current_url":"https://pdonline.brisbane.qld.gov.au/MasterPlan/Modules/Enquirer/PropertySearch.aspx"
                }
            )

        # Commented below section is about google sheet reading etc.

        """
        google_sheet_id =  "1ItXlYbNKUh9buALC-3WObEFJIrXILlCiRBIUJHvX3AA" # https://docs.google.com/spreadsheets/d/1ItXlYbNKUh9buALC-3WObEFJIrXILlCiRBIUJHvX3AA/edit?usp=sharing
        try:
            res = requests.get('https://docs.google.com/spreadsheet/ccc?key=' + google_sheet_id + '&output=csv')
        except:
            pass
        if not res.status_code == 200:
            print("Wrong status code")
            sys.exit(1)
        df = pd.read_csv(StringIO(res.text), sep=",")
        for index, row in df.iterrows():
            self.input_data.append(row.to_dict())

        self.input_data.pop(0)
        urls = list()
        urls = 'https://pdonline.brisbane.qld.gov.au/MasterPlan/Modules/Enquirer/PropertySearch.aspx'

        if self.input_data: # input_data
            yield scrapy.Request(url=urls, cookies=self.dynamic_cookie, callback=self.search_result, meta={"input_data": self.input_data.pop(), "current_url":urls})
        """

    def search_result(self, response):
        if response.css("#ctl00_RadWindow1_C_btnOk"):

            form_data = dict()
            input_fields = response.css("form input")
            for ifield in input_fields:
                if ifield.css("input::attr(name)").extract_first() == None:
                    continue
                if ifield.css("input::attr(value)").extract_first() == None:
                    form_data[ifield.css("input::attr(name)").extract_first()] = ''
                elif ifield.css("input::attr(value)").extract_first() == 'I Disagree':
                    continue
                elif ifield.css("input::attr(value)").extract_first() == '':
                    form_data[ifield.css("input::attr(name)").extract_first()] = ''
                else:
                    form_data[ifield.css("input::attr(name)").extract_first()] = ifield.css("input::attr(value)").extract_first()

            yield SplashFormRequest(
                url='https://pdonline.brisbane.qld.gov.au/MasterPlan/Modules/Enquirer/PropertySearch.aspx',
                formdata=form_data,
                callback=self.search_result,
                endpoint='execute',
                args={
                    'timeout': '90',
                    'wait': '30',
                    'lua_source': script,
                },
                session_id='1',
                meta={
                    'input_data': response.meta["input_data"],
                }
                )
        else:
            # Cookie handling
            for cookie in response.data["cookies"]:
                self.dynamic_cookie[ cookie["name"] ] = cookie["value"]

            form_data = dict()
            input_fields = response.css("form input")
            for ifield in input_fields:
                if ifield.css("input::attr(name)").extract_first() == None:
                    continue
                if ifield.css("input::attr(value)").extract_first() == None:
                    form_data[ifield.css("input::attr(name)").extract_first()] = ''
                else:
                    form_data[ifield.css("input::attr(name)").extract_first()] = ifield.css("input::attr(value)").extract_first()
            for key in response.meta["input_data"]:
                #  if response.meta["input_data"][key]:

                if key == "Suburb":
                    form_data["ctl00$MainContent$SuburbCombo"] = response.meta["input_data"][key]
                elif key == "Street Name":
                    street_name_define = {"logEntries":[],"value":"","text":"","enabled":True,"checkedIndices":[],"checkedItemsTextOverflows":False}
                    street_name_define["value"] = response.meta["input_data"][key]
                    street_name_define["text"] =response.meta["input_data"][key]
                    form_data["ctl00_MainContent_StreetCombo_ClientState"] = street_name_define
                    form_data["ctl00$MainContent$StreetCombo"] = response.meta["input_data"][key]
                elif key == "Unit Number From":
                    form_data["ctl00$MainContent$FromUnitNumberTextBox"] = response.meta["input_data"][key]
                    ctl00_MainContent_FromUnitNumberTextBox_ClientState = {"enabled":True,"emptyMessage":"","validationText":"","valueAsString":"","lastSetTextBoxValue":""}
                    ctl00_MainContent_FromUnitNumberTextBox_ClientState["validationText"] = response.meta["input_data"][key]
                    ctl00_MainContent_FromUnitNumberTextBox_ClientState["valueAsString"] = response.meta["input_data"][key]
                    ctl00_MainContent_FromUnitNumberTextBox_ClientState["lastSetTextBoxValue"] = response.meta["input_data"][key]
                    form_data["ctl00_MainContent_FromUnitNumberTextBox_ClientState"] = ctl00_MainContent_FromUnitNumberTextBox_ClientState
                elif key == "Unit Number To":
                    form_data["ctl00$MainContent$ToUnitNumberTextBox"] = response.meta["input_data"][key]
                    ctl00_MainContent_ToUnitNumberTextBox_ClientState = {"enabled":True,"emptyMessage":"","validationText":"","valueAsString":"","lastSetTextBoxValue":""}
                    ctl00_MainContent_ToUnitNumberTextBox_ClientState["validationText"] = response.meta["input_data"][key]
                    ctl00_MainContent_ToUnitNumberTextBox_ClientState["valueAsString"] = response.meta["input_data"][key]
                    ctl00_MainContent_ToUnitNumberTextBox_ClientState["lastSetTextBoxValue"] = response.meta["input_data"][key]
                    form_data["ctl00_MainContent_ToUnitNumberTextBox_ClientState"] = ctl00_MainContent_ToUnitNumberTextBox_ClientState
                elif key == "Street Number From":
                    form_data["ctl00$MainContent$FromStreetNumberTextBox"] = response.meta["input_data"][key]
                    ctl00_MainContent_FromStreetNumberTextBox_ClientState = {"enabled":True,"emptyMessage":"","validationText":"","valueAsString":"","lastSetTextBoxValue":""}
                    ctl00_MainContent_FromStreetNumberTextBox_ClientState["validationText"] = response.meta["input_data"][key]
                    ctl00_MainContent_FromStreetNumberTextBox_ClientState["valueAsString"] = response.meta["input_data"][key]
                    ctl00_MainContent_FromStreetNumberTextBox_ClientState["lastSetTextBoxValue"] = response.meta["input_data"][key]
                    form_data["ctl00_MainContent_FromStreetNumberTextBox_ClientState"] = ctl00_MainContent_FromStreetNumberTextBox_ClientState
                elif key == "Street Number To":
                    form_data["ctl00$MainContent$ToStreetNumberTextBox"] = response.meta["input_data"][key]
                    ctl00_MainContent_ToStreetNumberTextBox_ClientState = {"enabled":True,"emptyMessage":"","validationText":"","valueAsString":"","lastSetTextBoxValue":""}
                    ctl00_MainContent_ToStreetNumberTextBox_ClientState["validationText"] = response.meta["input_data"][key]
                    ctl00_MainContent_ToStreetNumberTextBox_ClientState["valueAsString"] = response.meta["input_data"][key]
                    ctl00_MainContent_ToStreetNumberTextBox_ClientState["lastSetTextBoxValue"] = response.meta["input_data"][key]
                    form_data["ctl00_MainContent_ToStreetNumberTextBox_ClientState"] = ctl00_MainContent_ToStreetNumberTextBox_ClientState

                elif key == "Plan Number":
                    form_data["ctl00$MainContent$PlanTextBox"] = response.meta["input_data"][key]
                    ctl00_MainContent_PlanTextBox_ClientState = {"enabled":True,"emptyMessage":"","validationText":"","valueAsString":"","lastSetTextBoxValue":""}
                    ctl00_MainContent_PlanTextBox_ClientState["validationText"] = response.meta["input_data"][key]
                    ctl00_MainContent_PlanTextBox_ClientState["valueAsString"] = response.meta["input_data"][key]
                    ctl00_MainContent_PlanTextBox_ClientState["lastSetTextBoxValue"] = response.meta["input_data"][key]
                elif key == "Lot Number":
                    form_data["ctl00$MainContent$LotTextBox"] = response.meta["input_data"][key]
                    ctl00_MainContent_LotTextBox_ClientState = {"enabled":True,"emptyMessage":"","validationText":"","valueAsString":"","lastSetTextBoxValue":""}
                    ctl00_MainContent_LotTextBox_ClientState["validationText"] = response.meta["input_data"][key]
                    ctl00_MainContent_LotTextBox_ClientState["valueAsString"] = response.meta["input_data"][key]
                    ctl00_MainContent_LotTextBox_ClientState["lastSetTextBoxValue"] = response.meta["input_data"][key]
                    form_data["ctl00_MainContent_LotTextBox_ClientState"] = ctl00_MainContent_LotTextBox_ClientState
                else:
                    pass

            for key in form_data:
                if isinstance(form_data[key], dict):
                    form_data[key] = json.dumps(form_data[key])
                if isinstance(form_data[key], int):
                    form_data[key] = str(form_data[key])

            # first method that i used to submit form
            yield FormRequest(
                url='https://pdonline.brisbane.qld.gov.au/MasterPlan/Modules/Enquirer/PropertySearch.aspx',
                formdata=form_data,
                callback=self.submit_form,
                cookies=self.dynamic_cookie
                )

            # Second method that i have used to submit form
            yield SplashFormRequest(
            url='https://pdonline.brisbane.qld.gov.au/MasterPlan/Modules/Enquirer/PropertySearch.aspx',
            formdata=form_data,
            callback=self.submit_form,
            endpoint='execute',
            args={
                'timeout': '90',
                'wait': '30',
                'lua_source': script,
            },
            session_id='1'
            )

    def submit_form(self, response):
        var_temp = 1


import argparse
import io
from nltk.translate.bleu_score import corpus_bleu
import sys

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--input", required=False, type=str)
    parser.add_argument("--order", type=int, default=4)
    args = parser.parse_args()

    weights = [1. / args.order] * args.order

    input_lines = (io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')
                   if args.input is None else open(args.input, encoding='utf-8'))

    hypotheses = [line.strip().split(' ') for line in input_lines]

    references = [hypotheses[:k] + hypotheses[k + 1:]
                  for k in range(len(hypotheses))]

    bleu = corpus_bleu(references, hypotheses, weights=weights)
    print(bleu)

if __name__ == '__main__':
    main()


import scrapy

from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

from scrapy.http import FormRequest
import logging

#from sicop_project.items import PgsCategoryItem

class ProvidersPerWgscategorySpider(scrapy.Spider):
    name = 'providers_per_wgscategory'
    allowed_domains = ['www.sicop.go.cr']
    #global custom_count
    #custom_count = 0

#   -------------------------------------------------
#   CONFIGURATION VARIABLES
#   -------------------------------------------------

    log_root_dir =  '/usr/src/app/sicop_project/sicop_project/spider_logs'
    log_filename =  'providers_per_wgscategory_level4_spider_2023_test_v5.log'
    log_level = 'INFO'

#   -------------------------------------------------

    custom_settings = {
        'LOG_FILE': log_root_dir + '/' + log_filename,
        'LOG_LEVEL': log_level
     }

    def start_requests(self):
        import csv

        #   -------------------------------------------------
        #   CONFIGURATION VARIABLES
        #   -------------------------------------------------

        input_root_dir =  '/usr/src/app/sicop_project/sicop_project/crawled_data'
        input_filename = 'wgscategories_level4_onlyid_noheader_test_v5.csv'

        #   -------------------------------------------------
        category_id_list = []
        with open(input_root_dir + '/' + input_filename,'rt', encoding='utf-16') as csvfile:
            reader = csv.reader(csvfile, delimiter=',', quotechar='"')
            for row in reader:
                #print(', '.join(row))
                #logging.info(row[0])
                category_id_list.append(row[0])

        #category_id_list = [
        #    "10101599"
        #    ,"10101501"
        #    ,"10101502"
#
#
#        ]

        for category_id in category_id_list:
            # logging.info(category_id)
            frmdata = {
                "page_size": "1000000"
                ,"cate_id": category_id
                ,"query_fl" : "Y"
                ,"cate_nm" : category_id
            }
            url = "https://www.sicop.go.cr/usemn/co/UM_COJ_CIQ003.jsp"
            yield FormRequest(url, callback=self.parse, formdata=frmdata, headers={
                "Content-Type": "application/x-www-form-urlencoded"
            })

    def parse(self, response):
        #category_id = response.css('input[name="cate_id"]::attr(value)').get()
        category_id = response.request.body.decode(encoding="utf-8").split("cateId=",1)[1].split("&",1)[0]
        for row in response.css('.trow'):
            #columns = row.css('td::text').getall()
            values = []
            for column in row.css('td'):
                value = column.css('::text').get()
                if value is None:
                    value = ''
                values.append(value)
            if values[0].strip() != 'Los datos consultados no existen.':
                yield {
                    'category_id' : category_id
                   ,'provider_id' : values[1].strip()
                   ,'provider_name' : values[3].strip()
                   ,'provider_location' : values[5].strip()
                }
"""        from scrapy.shell import inspect_response
        inspect_response(response, self)
"""
"""        global custom_count
        custom_count = custom_count + 1
        filename = f'providers-' + str(custom_count) + '.html'
        with open(filename, 'wb') as f:
            f.write(response.body)
        self.log(f'Saved file {filename}')
"""
"""
        category_id = response.css('input[name="cate_id"]::attr(value)').get()
        for row in response.css('.trow'):
            #columns = row.css('td::text').getall()
            values = []
            for column in row.css('td'):
                value = column.css('::text').get()
                if value is None:
                	value = ''
                values.append(value)
            yield {
                'category_id' : category_id
               ,'provider_id' : values[1].strip()
               ,'provider_name' : values[3].strip()
               ,'provider_location' : values[5].strip()
            }

            #logging.info(values[0].strip())
"""

"""        from scrapy.shell import inspect_response
        inspect_response(response, self)"""
    	#for raw_category in response.css('.tdl a'):
    	#	logging.info(category_id)
"""filename = f'providers.html'
        with open(filename, 'wb') as f:
            f.write(response.body)
        self.log(f'Saved file {filename}')"""

""""for row in response.css('.trow'):
            #columns = row.css('td::text').getall()
            columns = row.css('td').getall()

            logging.info(columns[0].strip())"""
"""response.css('.trow').css('td')"""
"""
        filename = f'providers.html'
        with open(filename, 'wb') as f:
            f.write(response.body)
        self.log(f'Saved file {filename}')"""


#https://leetcode.com/problems/find-the-winner-of-the-circular-game/

class Solution:
    def findTheWinner(self, n: int, k: int) -> int:
        '''
        #Solution with list

        que=[]
        for friend in range(n):
            que.append(friend+1)
        #print(que)
        while len(que)>1:
            for i in range(k-1):
                que.append(que.pop(0))
                #print(que)
            que.pop(0)
            #print(que)

        return que[0]
        '''

        '''
        #Using deque

        from collections import deque
        que=deque([])
        for friends in range(n):
            que.append(friends+1)
        #print(que)
        while len(que)>1:
            for i in range(k-1):
                que.append(que.popleft())
                #print(que)
            que.popleft()
            #print(que)

        return que[0]

        '''
        '''
        #Using head pointer

        from collections import deque
        que=deque([])
        for friends in range(n):
            que.append(friends+1)
        #print(que)

        head=0
        while len(que)!=1:
            if head!=k-1:
                que.append(que.popleft())
                head+=1
            else:
                que.popleft()
                head=0
            #print(que, head)

        return que[0]

        '''
        '''
        #Improving Head pointer method (Better solution)

        from collections import deque
        que=[i+1 for i in range(n)]

        head=0
        while len(que)>1:
            head=(head+k-1)%len(que)
            que.pop(head)

        return que[0]
        '''
        #Using rotating deque (Best solution for python amoung these)
        #rotate deque with negative value means rotating 0 to k ie from front

        import collections
        que=collections.deque([i for i in range(1, n + 1)])

        while len(que)>1:
            que.rotate(-k)
            que.pop()
            #print(que)

        return que[0]


"""
Infers the requirement of a lambda by detecting which expressions use the
``?`` placeholder.

Only expressions in an assignment or in parameter positions are transformed.

>>> pipe.map( 'foo' + ?.lower() )
  # pipe.map( lambda __1, *args: 'foo' + __1.lower() )

TODO: Support multiple placeholders `?1 + ?2` -- `??` is whole varargs

"""

import ast
from io import StringIO

from pysh.transforms import TokenIO

PLACEHOLDER = '__INFERRED_LAMBDA_PLACEHOLDER__'

def lexer(code: StringIO) -> StringIO:
    """
    Lexer just replaces the ``?`` placeholder with something that doesn't
    make the Python parser chock up.
    """
    out = TokenIO()
    tokens = TokenIO(code).iter_tokens()
    for tkn in tokens:
        if tkn.string == '?':
            out.write_token(tkn, override=PLACEHOLDER)
        else:
            out.write_token(tkn)

    return out

class InferredLambdaTransformer(ast.NodeTransformer):

    def __init__(self):
        self.placeholder = PlaceholderTransformer()
        self.found = False

    def do_placeholder(self, node: ast.expr) -> ast.expr:
        self.placeholder.reset()
        node = self.placeholder.visit(node)
        if self.placeholder.replaced:
            node = ast.Lambda(
                args=ast.arguments(
                    args=[],
                    vararg=ast.arg(arg='__varargs'),
                    kwonlyargs=[],
                    kw_defaults=[],
                    kwarg=None,
                    defaults=[]),
                body=node,
                lineno=node.lineno,
                col_offset=node.col_offset)

            ast.fix_missing_locations(node)

        return node

    def visit_Name(self, node: ast.Name) -> ast.Name:
        if node.id == PLACEHOLDER:
            self.found = True

        return node

    def visit_Assign(self, node: ast.Assign) -> ast.Assign:
        self.found = False
        self.generic_visit(node)
        if self.found:
            node.value = self.do_placeholder(node.value)

        return node

    def visit_Call(self, node: ast.Call) -> ast.Call:
        self.found = False
        self.generic_visit(node)
        if self.found:
            node.args = [self.do_placeholder(x) for x in node.args]
            node.keywords = [(arg, self.do_placeholder(value)) for arg,value in node.keywords]

        return node

class PlaceholderTransformer(ast.NodeTransformer):

    def __init__(self):
        self.replaced = False

    def reset(self):
        self.replaced = False

    def visit_Name(self, node: ast.Name) -> ast.Name:
        if node.id == PLACEHOLDER:
            self.replaced = True
            node = ast.Subscript(
                value=ast.Name(id='__varargs', ctx=ast.Load()),
                slice=ast.Index(value=ast.Num(0)),
                ctx=ast.Load(),
                lineno=node.lineno,
                col_offset=node.col_offset)
            ast.fix_missing_locations(node)

        return node

class PlaceholderVisitor(ast.NodeVisitor):

    def __init__(self, fname):
        self.fname = fname

    def visit_Name(self, node: ast.Name) -> ast.Name:
        if node.id == PLACEHOLDER:
            from warnings import warn_explicit
            message = (
                'Detected an un-transformed lazyexpr `?` symbol. '
                'Only valid in assignments or argument positions. '
                '(from pysh.transform.alpha.lazyexpr)'
            ).format(id)

            warn_explicit(message, SyntaxWarning, self.fname, node.lineno)

def parser(node: ast.AST, fname: str) -> ast.AST:
    node = InferredLambdaTransformer().visit(node)
    PlaceholderVisitor(fname).visit(node)
    return node

code = '''\
print(list( map(?.lower(), ['Foo', 'Bar']) ))
'''

out = lexer(StringIO(code))
print(out.getvalue())

import ast, pprint
node = ast.parse(out.getvalue())
node = parser(node, '<code>')
print(ast.dump(node).replace("],", "\n],\n").replace('=[', '=[\n'))
co_code = compile(node, '<code>', 'exec')
exec(co_code)


from django.shortcuts import render, redirect, HttpResponse, Http404
from django.template.loader import render_to_string
from django.contrib.auth.decorators import login_required
from django.contrib.auth.mixins import LoginRequiredMixin
from django.utils.decorators import method_decorator
from django.template .context_processors import csrf
from django.views.generic import ListView, DetailView, CreateView, UpdateView, View
from ratelimit.decorators import ratelimit
from .models import *
import json

class IndexView(ListView):
    model = SKU
    context_object_name = 'skus'
    template_name = 'products/index.html'
    paginate_by = 8

    def get_context_data(self, **kwargs):
        context = super(IndexView, self).get_context_data(**kwargs)
        context['nodes'] = Category.objects.all()
        context['can_search'] = True
        return context

    def get_queryset(self):
        only_fields = ['product__name', 'product__manufacturer__name', 'batch',]
        queryset = self.model.objects.select_related().only(*only_fields).order_by('-batch__count', '-likes')
        return queryset

class FavouritesView(LoginRequiredMixin, ListView):
    model = SKU
    context_object_name = 'liked_skus'
    template_name = 'products/favourites.html'
    paginate_by = 12

    def get_context_data(self, **kwargs):
        context = super(FavouritesView, self).get_context_data(**kwargs)
        context['can_search'] = True
        context['is_empty'] = not bool(context[self.context_object_name].count())
        return context

    def get_queryset(self):
        user_profile = UserProfile.objects.get(user__id=self.request.user.id)

        only_fields = ['product__name', 'product__manufacturer__name', 'batch',]
        queryset = user_profile.favourites.select_related().only(*only_fields)
        return queryset

class SearchView(ListView):
    model = SKU
    context_object_name = 'skus'
    template_name = 'products/search.html'

    def get(self, request, *args, **kwargs):
        if request.is_ajax():
            self.object_list = self.get_queryset()
            context = self.get_context_data_for_ajax()
            html = render_to_string('products/includes/search-div.html', context)
            return HttpResponse(html)

        return super(SearchView, self).get(request, *args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(SearchView, self).get_context_data(**kwargs)
        context['can_search'] = True
        context['categories'] = Category.objects.only('name').order_by('level')
        context['manufacturers'] = Manufacturer.objects.only('name').order_by('name')
        context['is_empty'] = not bool(context[self.context_object_name].count())
        context['colors'] = sorted([x[1] for x in SKU.COLOR_CHOICES])
        context['expr'] = self.get_expr()
        return context

    def get_context_data_for_ajax(self, **kwargs):
        context = super(SearchView, self).get_context_data(**kwargs)
        context['is_empty'] = not bool(context[self.context_object_name].count())
        context['expr'] = self.get_expr()
        return context

    def get_queryset(self):
        only_fields = ['product__name', 'product__manufacturer__name', 'batch',]
        queryset = self.model.objects.select_related().only(*only_fields)

        try:
            if 'cat' in self.request.GET:
                category = self.request.GET['cat']
                queryset = queryset.filter(product__in=Category.objects.get(name=category).products.only('id'))

            if 'clrs' in self.request.GET:
                colors = json.loads(self.request.GET['clrs'])
                queryset = queryset.filter(color__in=colors)

            if 'manufs' in self.request.GET:
                manufacturers = json.loads(self.request.GET['manufs'])
                queryset = queryset.filter(product__manufacturer__name__in=manufacturers)

            if 'has_wifi' in self.request.GET:
                if self.request.GET['has_wifi']:
                    queryset = queryset.filter(product__has_wifi=True)

            if 'has_bluetooth' in self.request.GET:
                if self.request.GET['has_bluetooth']:
                    queryset = queryset.filter(product__has_bluetooth=True)

            if 'expr' in self.request.GET:
                string = self.get_expr()
                strings = string.split(' ')

                skus = list(queryset)

                for s in strings:
                    skus = list(filter(lambda x: re.search(s, x.product.manufacturer.name +
                                                           ' ' + x.product.name, re.IGNORECASE), skus))

                ids = map(lambda x: x.id, skus)
                queryset = SKU.objects.filter(id__in=ids)
        except:
            queryset = self.model.objects.none()

        return queryset.order_by('-batch__count', '-likes')

    def get_expr(self):
        return self.request.GET.get('expr', None)

    @method_decorator(ratelimit(key='ip', rate='5/s', block=True))
    def dispatch(self, request, *args, **kwargs):
        return super(SearchView, self).dispatch(request, *args, **kwargs)

class ProductView(DetailView):
    model = SKU
    context_object_name = 'sku'
    template_name = 'products/product.html'

    def get_context_data(self, **kwargs):
        context = super(ProductView, self).get_context_data(**kwargs)
        context['can_search'] = False
        context['product'] = self.object.product
        context['comments'] = self.object.comments.select_related().only('owner__user__username').order_by('-time')
        context['is_liked'] = self.is_liked_product()
        context['batch'] = self.object.batch
        return context

    def get_object(self, queryset=None):
        if queryset is None:
            queryset = self.get_queryset()

        try:
            sku_id = self.kwargs['sku_id']
            obj = queryset.get(id=sku_id)
        except:
            raise Http404()

        return obj

    def is_liked_product(self):
        try:
            user_profile = UserProfile.objects.get(user__id=self.request.user.id)
            return bool(user_profile.favourites.filter(id=self.kwargs['sku_id']))
        except:
            return False

class LikeUpdate(LoginRequiredMixin, UpdateView):
    http_method_names = ['get']
    template_name = 'products/includes/btns.html'

    def get(self, request, *args, **kwargs):
        if request.is_ajax():
            sku_id = int(request.GET['id'])
            sku = SKU.objects.get(id=sku_id)

            action = request.GET['action']
            status = self.get_action_status(action, sku_id)
            batch = sku.batch
            product = sku.product

            context = {'is_liked': status, 'batch': batch, 'product': product}
            html = render_to_string(self.template_name, context)
            return HttpResponse(html)

        raise Http404()

    def get_action_status(self, action, sku_id):
        user_profile = UserProfile.objects.get(user__id=self.request.user.id)
        sku = SKU.objects.get(id=sku_id)
        result = True

        if action == 'lk':
            sku.likes = sku.likes + 1
            user_profile.favourites.add(sku_id)
        else:
            sku.likes = sku.likes - 1
            user_profile.favourites.remove(sku_id)
            result = False

        sku.save()
        user_profile.save()
        return result

    def http_method_not_allowed(self, request, *args, **kwargs):
        raise Http404()

class CommentCreate(LoginRequiredMixin, CreateView):
    model = Comment
    context_object_name = 'comments'
    http_method_names = ['post']
    template_name = 'products/includes/comments.html'
    fields = ['content']

    def post(self, request, *args, **kwargs):
        if request.is_ajax():
            text = request.POST['content']
            sku_id = int(self.kwargs['sku_id'])

            self.create_comment(text, sku_id)

            html = render_to_string(self.template_name, self.get_context_data())
            return HttpResponse(html)

        raise Http404()

    def create_comment(self, text, sku_id):
        user_profile = UserProfile.objects.only('id').get(user__id=self.request.user.id)
        self.sku = SKU.objects.get(id=sku_id)
        self.sku.comments.create(owner=user_profile, content=text)

    def get_context_data(self, **kwargs):
        context ={self.context_object_name: self.get_queryset()}
        return context

    def get_queryset(self):
        only_fields = ['owner__user__username']
        queryset = self.sku.comments.select_related().only(*only_fields).order_by('-time')
        return queryset

    def http_method_not_allowed(self, request, *args, **kwargs):
        raise Http404()

    @method_decorator(ratelimit(key='user', rate='5/m', block=True))
    def dispatch(self, request, *args, **kwargs):
        return super(CommentCreate, self).dispatch(request, *args, **kwargs)

class Payment(LoginRequiredMixin, View):
    http_method_names = ['post',]

    # UPDATE
    def post(self, request, *args, **kwargs):
        prev_cost = int(self.kwargs['cost'])
        sku_id = self.kwargs['sku_id']
        batch = SKU.objects.get(id=sku_id).batch

        if batch.count == 0:
            print('no products')
        elif prev_cost != batch.cost:
            print('product cost changed')
        else:
            batch.count = batch.count - 1
            batch.save()

        redirect_url = '/product/id' + sku_id
        return redirect(redirect_url)

    def http_method_not_allowed(self, request, *args, **kwargs):
        raise Http404()


import matplotlib.pyplot
import cv2
import numpy as np
import os

VID_DIR = "inputVideo"
OUT_DIR = "outputVideo"
OUT_IMG = "outImage"

actions = {'boxing': 1, 'handclapping': 2, 'handwaving': 3, 'jogging': 4, 'running': 5, 'walking': 6}

# Helper Functions are from previous Homework
def helper_video_frame_generator(filename):
    video = cv2.VideoCapture(filename)
    while video.isOpened():
        ret, frame = video.read()
        if ret:
            yield frame
        else:
            break
    video.release()
    yield None

def helper_video_create(video_name, predict, framesStart, frameLength, outVideo):

    # Calculate Frame Pairs
    framesEnd = dict()
    for act, values in framesStart.items():
        framesEnd[act] = [i + frameLength[act] for i in values]

    frames = list()
    for keys in framesStart.keys():
        for j in range(len(framesStart[keys])):
            frames.append((framesStart[keys][j], framesEnd[keys][j]))

    # Suppose to contain all 36 frames
    frames = sorted(frames, key=lambda x: x[0])
    y = sorted([1, 2, 3, 4, 5, 6] * 6)

    fps = 20

    video = os.path.join(VID_DIR, video_name)
    image_gen = helper_video_frame_generator(video)

    image = image_gen.__next__()
    h, w, d = image.shape

    out_path = os.path.join(OUT_DIR, outVideo)
    video_out = helper_mp4_video_writer(out_path, (w, h), fps)

    save_image_counter = 1

    frame_num = 1
    while image is not None:

        for i, (start, end) in enumerate(frames):
            if (frame_num >= start) & (frame_num <= end):
                helper_mark_location(image, i, y[i], predict[i])

                if frame_num == start + 10:
                    out_str = "Test_frame_output" + "-{}.png".format(frame_num)
                    helper_save_image(out_str, image)
                    save_image_counter += 1

                video_out.write(image)
                break

        image = image_gen.__next__()
        frame_num += 1

    video_out.release()

def helper_mp4_video_writer(filename, frame_size, fps=40):
    fourcc = cv2.VideoWriter_fourcc(*'MP4V')
    return cv2.VideoWriter(filename, fourcc, fps, frame_size)

def helper_save_image(filename, image):
    cv2.imwrite(os.path.join(OUT_IMG, filename), image)

def helper_mark_location(image, sample, result, predict):

    actions = ['boxing', 'handclapping', 'handwaving', 'jogging', 'running', 'walking']

    color1 = (205, 0, 0)
    color2 = (0, 0, 205)
    h, w, d = image.shape
    p1 = [int(w/10), int(h/5)]

    font = cv2.FONT_HERSHEY_DUPLEX
    cv2.putText(image, "(Sample:{})".format(sample+1), (p1[0]-5, p1[1]+0), font, 0.3, color1, 1)
    cv2.putText(image, "(Actual:{})".format(actions[result-1]), (p1[0]-5, p1[1]+20), font, 0.3, color1, 1)

    if actions[result-1] == actions[predict-1]:
        cv2.putText(image, "(Prediction:{})".format(actions[predict-1]), (p1[0]+5, p1[1]+40), font, 0.3, color1, 1)
    else:
        cv2.putText(image, "(Prediction:{})".format(actions[predict-1]), (p1[0]+5, p1[1]+40), font, 0.3, color2, 1)

class MHIClassifier():

    def __init__(self, imageHuMoment, lableMatrix):
        self.classifier = cv2.ml.KNearest_create()
        self.X = imageHuMoment
        self.y = lableMatrix
        self.neighbours = list()

    def crossValidation(self):
        # train the model and predict the test y
        cnfmt = np.zeros((6, 6))
        for i in range(len(self.X)):
            j = list(range(0, i)) + list(range(i+1, self.X.shape[0]))
            # train dataset
            Xt = np.array(self.X[j])
            yt = np.array(self.y[j])

            # train knn classifier
            self.classifier.train(Xt, cv2.ml.ROW_SAMPLE, yt)

            Xte = np.array([self.X[i]])
            yte = np.array([self.y[i]])
            # predict test data labels, res is an (1,1) array, but only 1 element, so extract it
            retval, res, neighborResponses, dists = self.classifier.findNearest(Xte, 5)

            cnfmt[yte-1, int(res[0])-1] += 1

        self.confusionMatrix(cnfmt, actions=['boxing', 'handclapping', 'handwaving', 'jogging', 'running', 'walking'])

    def train(self):
        # train dataset
        # train knn classifier
        self.classifier.train(self.X, cv2.ml.ROW_SAMPLE, self.y)

    def predict(self, X_test):
        # predict test data labels, res is an (1,1) array, but only 1 element, so extract it
        ret, results, neighbours, dist = self.classifier.findNearest(X_test.T, 5)

        self.y_hat = results[0, 0]
        self.neighbours.append(neighbours[0])

    def confusionMatrix(self, cm, actions):
        cm = (cm * 100 / cm.sum()).astype(np.uint) / 100.0
        th = cm.max() / 2.
        matplotlib.pyplot.figure(figsize=(8, 6.5))
        matplotlib.pyplot.imshow(cm, interpolation='nearest', cmap=matplotlib.pyplot.cm.Oranges)
        matplotlib.pyplot.title('Confusion matrices')
        matplotlib.pyplot.ylabel('Actual')
        matplotlib.pyplot.xlabel('Predicted')
        tm = np.arange(len(actions))
        matplotlib.pyplot.xticks(tm, actions)
        matplotlib.pyplot.yticks(tm, actions)
        matplotlib.pyplot.colorbar()
        matplotlib.pyplot.tight_layout()
        filename = 'confusion_matrices.png'
        for i in range(cm.shape[0]):
            for j in range(cm.shape[1]):
                matplotlib.pyplot.text(j, i, cm[i, j], horizontalalignment="center",
                                       color="white" if cm[i, j] > th else "black")

        matplotlib.pyplot.savefig(os.path.join(OUT_IMG, filename))

class MHIModel():
    def __init__(self):
        '''
        self.frameLength = {'boxing': 36,
                            'handclapping': 27,
                            'handwaving': 48,
                            'jogging': 35,
                            'running': 25,
                            'walking': 45}
        '''

        self.frameLength = {'boxing': 30,
                            'handclapping': 25,
                            'handwaving': 50,
                            'jogging': 35,
                            'running': 25,
                            'walking': 45}

        self.frameLengthSet = [25, 30, 35, 40, 45, 50]

        # theta = [15, 15, 15, 15, 15, 15]
        # theta = [8, 8, 10, 25, 15, 35]
        # theta = [3, 3, 10, 30, 15, 45]
        # theta = [3, 3, 10, 30, 30, 30]

        # tau needs to be chosen and differs with actions
        # tau = [50, 50, 50, 50, 50, 50]
        # tau = [50, 30, 50, 35, 15, 55]
        # tau = [35, 15, 50, 40, 25, 65]
        # tau = [35, 20, 50, 45, 45, 45]

        self.theta = [8, 8, 35, 35, 25, 40]
        self.tau = [15, 18, 65, 55, 45, 65]

        # self.validateTheta = [17] * 6
        # self.validateTau = [40] * 6

        self.validateTheta = [15] * 6
        self.validateTau = [40] * 6

    def createBinaryMotionSignal(self, blurKernel, blurSigma, video_name, start, end, theta):

        video = os.path.join(VID_DIR, video_name)

        image_gen = helper_video_frame_generator(video)
        image = image_gen.__next__()

        frame = 0

        # Define my Return Data
        binaryImageSet = []
        imageBset = []

        while image is not None:

            # Start Processing Images
            if (frame >= start) and (frame <= end):
                # Create Image and ImageB for t and t+1
                image1 = cv2.cvtColor(np.copy(image), cv2.COLOR_BGR2GRAY)
                image1 = cv2.GaussianBlur(image1, blurKernel, blurSigma)

                imageB = image_gen.__next__()

                if imageB is None:
                    break
                else:
                    image2 = cv2.cvtColor(np.copy(imageB), cv2.COLOR_BGR2GRAY)
                    image2 = cv2.GaussianBlur(image2, blurKernel, blurSigma)

                    # Binary Image Bt(x, y)
                    binaryImage = np.abs(cv2.subtract(image2, image1)) >= theta
                    binaryImage = binaryImage.astype(np.uint8)
                    # print(np.any(binaryImage > 0))

                    # "clean up" the difference Image by Applying morphologyEx()
                    # https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_morphological_ops/py_morphological_ops.html
                    kernel = np.ones((3, 3), np.uint8)
                    binaryImage = cv2.morphologyEx(binaryImage, cv2.MORPH_OPEN, kernel)
                    imageBset.append(image2)
                    binaryImageSet.append(binaryImage)

                    # Assign Image B to Image A - use Copy to make sure they won't change together
                    image = np.copy(imageB)
                    frame += 1

            # move to next Image if not executed
            else:
                image = image_gen.__next__()
                frame += 1

        return binaryImageSet, imageBset

    def createMHI(self, start, end, binaryImageSet, tau):
        # Input shall be Binary Motion Signal from Function createBinaryMotionSignal
        # return MHI image

        Itau = np.zeros(binaryImageSet[0].shape, dtype=np.float)

        for i, image in enumerate(binaryImageSet):
            if i == (end - start):
                break
            mask1 = (image == 1)
            mask0 = (image == 0)
            Itau = tau * mask1 + np.clip(np.subtract(Itau, np.ones(Itau.shape)), 0, 255) * mask0

        return Itau.astype(np.uint8)

    def createHuMoment(self, image):
        # refer to paper
        # return 3 dictionary
        # derive all 7 mu's - exclude mu00, mu10, mu01, since mu10 = mu01 = 0
        pqpair = [(2, 0), (1, 1), (0, 2), (3, 0), (2, 1), (1, 2), (0, 3)]

        # calculate central moments upq
        u00 = image.sum()
        h, w = image.shape

        # Reshape to two dimension to secure broadcasting
        x_bar = np.sum(np.arange(w).reshape(1, -1) * image, keepdims=True) / u00
        y_bar = np.sum(np.arange(h).reshape(-1, 1) * image, keepdims=True) / u00

        mus = dict()
        eta = dict()
        for i, (p, q) in enumerate(pqpair):
            pq = str(p) + str(q)
            # Same idea to calculate all mu's
            diffx = np.arange(w).reshape(1, -1) - x_bar
            diffy = np.arange(h).reshape(-1, 1) - y_bar

            # calculate the central moments
            upq = np.sum((diffx ** p).reshape(1, -1) * (diffy ** q).reshape(-1, 1) * image, keepdims=True)
            mus[pq] = upq

            # calculate the scale invariance moments
            eta[pq] = mus[pq] / u00 ** (1 + (p + q) / 2)

        # Generate all invariant Hu Moment once have all eta
        momentHu = np.zeros(len(pqpair))

        momentHu[0] = eta['20'] + eta['02']
        momentHu[1] = (eta['20'] - eta['02']) ** 2 + 4 * eta['11'] ** 2
        momentHu[2] = (eta['30'] - 3 * eta['12']) ** 2 + (3 * eta['21'] - eta['03']) ** 2
        momentHu[3] = (eta['30'] + eta['12']) ** 2 + (eta['21'] + eta['03']) ** 2
        momentHu[4] = ((eta['30'] - 3 * eta['12']) * (eta['30'] + eta['12']) * ((eta['30'] + eta['12']) ** 2 - 3 * (eta['21'] + eta['03']) ** 2)
                       + (3 * eta['21'] - eta['03']) * (eta['21'] + eta['03']) * (3 * (eta['30'] + eta['12']) ** 2 - (eta['21'] + eta['03']) ** 2))
        momentHu[5] = ((eta['20'] - eta['02']) * ((eta['30'] + eta['12']) ** 2 - (eta['21'] + eta['03']) ** 2)
                       + 4 * eta['11'] * (eta['30'] + eta['12']) * (eta['21'] + eta['03']))
        momentHu[6] = ((3 * eta['21'] - eta['03']) * (eta['30'] + eta['12']) * ((eta['30'] + eta['12']) ** 2 - 3 * (eta['21'] + eta['03']) ** 2)
                       - (eta['30'] - 3 * eta['12']) * (eta['21'] + eta['03']) * (3 * (eta['30'] + eta['12']) ** 2 - (eta['21'] + eta['03']) ** 2))

        return mus, eta, momentHu

    def classifier(self):

        actions = [('boxing', 1), ('handclapping', 2), ('handwaving', 3), ('jogging', 4), ('running', 5), ('walking', 6)]

        video_name = {'boxing': "person15_boxing_d1_uncomp.avi",
                      'handclapping': "person15_handclapping_d1_uncomp.avi",
                      'handwaving': "person15_handwaving_d1_uncomp.avi",
                      'jogging': "person15_jogging_d1_uncomp.avi",
                      'running': "person15_running_d1_uncomp.avi",
                      'walking': "person15_walking_d1_uncomp.avi"}

        # person 15's training data
        # person15_boxing_d1		frames	1-150, 151-216, 217-290, 291-408
        # person15_handclapping_d1	frames	1-65, 66-164, 165-232, 233-312
        # person15_handwaving_d1	frames	1-136, 137-279, 280-440, 441-581
        # person15_jogging_d1		frames	1-75, 140-205, 240-300, 360-420
        # person15_running_d1		frames	1-50, 100-150, 180-225, 280-325
        # person15_walking_d1		frames	1-105, 215-338, 422-540, 630-741

        # frames = {'boxing': [(1, 150), (151, 216), (217, 290), (291, 408)],
        #           'handclapping': [(1, 65), (66, 164), (165, 232), (233, 312)],
        #           'handwaving': [(1, 136), (137, 279), (280, 440), (441, 581)],
        #           'jogging': [(1, 75), (140, 205), (240, 300), (360, 420)],
        #           'running': [(1, 50), (100, 150), (180, 225), (280, 325)],
        #           'walking': [(1, 105), (215, 338), (422, 540), (630, 741)]
        #           }

        framesStart = {'boxing': [1, 36, 72, 108, 160, 220],
                       'handclapping': [1, 27, 54, 81, 108, 135],
                       'handwaving': [1, 48, 96, 138, 160, 208],
                       'jogging': [10, 145, 155, 245, 255, 365],
                       'running': [10, 105, 115, 190, 195, 285],
                       'walking': [15, 220, 235, 430, 440, 640]}

        framesEnd = dict()
        for act, values in framesStart.items():
            framesEnd[act] = [i + self.frameLength[act] for i in values]

        frames = dict()
        for keys in framesStart.keys():
            frames[keys] = list()
            for j in range(len(framesStart[keys])):
                frames[keys].append((framesStart[keys][j], framesEnd[keys][j]))

        # choose theta for Binary Image Creation
        # theta = [15, 15, 15, 15, 15, 15]
        # theta = [8, 8, 10, 25, 15, 35]
        # theta = [3, 3, 10, 30, 15, 45]
        # theta = [3, 3, 10, 30, 30, 30]
        theta = self.theta

        # tau needs to be chosen and differs with actions
        # tau = [50, 50, 50, 50, 50, 50]
        # tau = [50, 30, 50, 35, 15, 55]
        # tau = [35, 15, 50, 40, 25, 65]
        # tau = [35, 20, 50, 45, 45, 45]
        tau = self.tau

        # create all the action MHI, MEI, and labels
        MotionHistoryImgs = []
        y = []

        frame_ids = [5, 10, 15]
        MotionEnergyImgs = []

        for i, (key, action) in enumerate(actions):
            print("key = " + key)

            for j, (start, end) in enumerate(frames[key]):

                # call createBinaryMotionSignal to build binary motion signal, prepare for MHI
                binaryImageSet, imageBset = self.createBinaryMotionSignal((5, 5), 0, video_name[key], start, end, theta[i])

                # MHI image and derive MEI based on MHI
                MHI = self.createMHI(start=start, end=end, binaryImageSet=binaryImageSet, tau=tau[i]).astype(np.float)

                # normalize the motion history image & prepare motion energy images
                cv2.normalize(MHI, MHI, 0.0, 255.0, cv2.NORM_MINMAX)
                MEI = (255 * MHI > 0).astype(np.uint8)

                # save the sample MHI
                out_str = "MHI" + "-{}-{}.png".format(key, j)
                helper_save_image(out_str, MHI)

                out_str = "MEI" + "-{}-{}.png".format(key, j)
                helper_save_image(out_str, (255 * MEI))

                MotionHistoryImgs.append(MHI)
                MotionEnergyImgs.append(MEI)
                y.append(action)

            # Output some sample Binary Image & ImageB image
            for j in frame_ids:
                out_str = "binaryImageSet" + "-{}-{}.png".format(key, j)
                outBinaryImageSet = binaryImageSet[j]

                cv2.normalize(outBinaryImageSet, outBinaryImageSet, 0, 255, cv2.NORM_MINMAX)
                helper_save_image(out_str, outBinaryImageSet)

                out_str = "imageBset" + "-{}-{}.png".format(key, j)
                outImageBset = imageBset[j]
                helper_save_image(out_str, outImageBset)

        # call createHuMoment, create central moments and scale invariant moment based on paper
        mus = []
        etas = []
        HuMoment = []

        for MHI, MEI in zip(MotionHistoryImgs, MotionEnergyImgs):
            # print(MHI.shape)
            MHImus, MHIeta, MHIHuMoment = self.createHuMoment(MHI)
            MEImus, MEIeta, MEIHuMoment = self.createHuMoment(MEI)

            mus.append(np.append(MHImus, MEImus))
            etas.append(np.append(MHIeta, MEIeta))
            HuMoment.append(np.append(MHIHuMoment, MEIHuMoment))

        # now change cms and sims to be array, row:number of actions, column: number of moments
        # training Data, classify 6 actions

        y = np.array(y).astype(np.int).reshape(-1, 1)
        HuMoment = np.array(HuMoment).astype(np.float32)

        self.classifier = MHIClassifier(HuMoment, y)
        self.classifier.crossValidation()
        self.classifier.train()

    def recognition(self, video_name, outVideoName):

        actions = [('boxing', 1), ('handclapping', 2), ('handwaving', 3), ('jogging', 4), ('running', 5),
                   ('walking', 6)]

        # video_name = {'boxing': "person04_boxing_d1_uncomp.avi",
        #               'handclapping': "person14_handclapping_d1_uncomp.avi",
        #               'handwaving': "person07_handwaving_d1_uncomp.avi",
        #               'jogging': "person09_jogging_d1_uncomp.avi",
        #               'running': "person18_running_d1_uncomp.avi",
        #               'walking': "person14_walking_d1_uncomp.avi"}

        # Random Person's validation data
        # boxing - frame: 328
        # handclapping - frame: 524
        # handwaving - frame: 782
        # jogging - frame: 465
        # running -  frame: 375
        # walking - frame: 720
        # framesStart = {'boxing': [5, 36, 72, 108, 144, 180],
        #                'handclapping': [5, 27, 54, 81, 108, 135],
        #                'handwaving': [5, 48, 96, 138, 160, 208],
        #                'jogging': [1, 10, 150, 270, 280, 400],
        #                'running': [1, 15, 110, 223, 235, 330],
        #                'walking': [1, 15, 220, 235, 410, 610]}

        framesStart = {'boxing': [5, 36, 72, 108, 144, 180],
                       'handclapping': [i + 328 for i in [5, 27, 54, 81, 108, 135]],
                       'handwaving': [i + 852 for i in [5, 48, 96, 138, 160, 208]],
                       'jogging': [i + 1634 for i in [1, 10, 150, 270, 280, 400]],
                       'running': [i + 2099 for i in [1, 15, 110, 223, 235, 330]],
                       'walking': [i + 2474 for i in [1, 65, 220, 235, 410, 610]]
                       }

        # choose theta for Binary Image Creation
        # theta = [15, 15, 15, 15, 15, 15]
        # theta = [8, 8, 10, 25, 15, 35]
        # theta = [3, 3, 10, 30, 15, 45]
        # theta = [3, 3, 10, 30, 30, 30]
        theta = self.validateTheta

        # tau needs to be chosen and differs with actions
        # tau = [50, 50, 50, 50, 50, 50]
        # tau = [50, 30, 50, 35, 15, 55]
        # tau = [35, 15, 50, 40, 25, 65]
        # tau = [35, 20, 50, 45, 45, 45]
        # tau = [35, 20, 45, 35, 35, 45]
        tau = self.validateTau

        # create all the action MHI, MEI, and labels
        MotionHistoryImgs = []
        y = []
        y_hat = []

        frame_ids = [5, 10, 15]
        MotionEnergyImgs = []

        for i, (key, action) in enumerate(actions):
            print("key = " + key)

            for j, start in enumerate(framesStart[key]):
                frames = []
                y_sim = []
                y.append(action)
                for z in self.frameLengthSet:
                    framesEnd = framesStart[key][j] + z
                    frames.append((framesStart[key][j], framesEnd))

                for i, (start, end) in enumerate(frames):
                    # call createBinaryMotionSignal to build binary motion signal, prepare for MHI
                    binaryImageSet, imageBset = self.createBinaryMotionSignal((5, 5), 0, video_name, start, end,
                                                                              theta[i])

                    # MHI image and derive MEI based on MHI
                    MHI = self.createMHI(start=start, end=end, binaryImageSet=binaryImageSet, tau=tau[i]).astype(np.float)

                    # normalize the motion history image & prepare motion energy images
                    cv2.normalize(MHI, MHI, 0.0, 255.0, cv2.NORM_MINMAX)
                    MEI = (255 * MHI > 0).astype(np.uint8)

                    MotionHistoryImgs.append(MHI)
                    MotionEnergyImgs.append(MEI)

                    MHImus, MHIeta, MHIHuMoment = self.createHuMoment(MHI)
                    MEImus, MEIeta, MEIHuMoment = self.createHuMoment(MEI)

                    HuMoment = np.append(MHIHuMoment, MEIHuMoment)

                    # now change cms and sims to be array, row:number of actions, column: number of moments
                    # training Data, classify 6 actions

                    # y = np.array(y).astype(np.int).reshape(-1, 1)
                    HuMoment = np.array(HuMoment).astype(np.float32)

                    self.classifier.predict(np.array(HuMoment).reshape(-1, 1))
                    y_sim.append(self.classifier.y_hat.astype(np.int8))

                y_hat.append(np.argmax(np.bincount(y_sim)))
                self.classifier.neighbours = list()

        acc = np.sum([1 for i in range(len(y)) if y[i] == y_hat[i]]) / len(y)
        print(y)
        print(y_hat)
        print("prediction accuracy is " + str(acc))

        print("----- Generating Testing Video ------")
        helper_video_create(video_name, y_hat, framesStart, self.frameLength, outVideoName)

        print("------ Creating Confusion Matrix ------------")
        confusionMatrix = np.zeros((6, 6))
        for (y, y_hat) in zip(y, y_hat):
            confusionMatrix[y-1, y_hat - 1] += 1

        confusionMatrix = (confusionMatrix * 100.0 / confusionMatrix.sum()).astype(np.uint) / 100.0
        th = confusionMatrix.max() / 2.
        matplotlib.pyplot.figure(figsize=(8, 6.5))
        matplotlib.pyplot.imshow(confusionMatrix, interpolation='nearest', cmap=matplotlib.pyplot.cm.Oranges)
        matplotlib.pyplot.title('Confusion matrices')
        matplotlib.pyplot.ylabel('Actual')
        matplotlib.pyplot.xlabel('Predicted')

        inAction = ['boxing', 'handclapping', 'handwaving', 'jogging', 'running', 'walking']
        tm = np.arange(len(inAction))
        matplotlib.pyplot.xticks(tm, inAction)
        matplotlib.pyplot.yticks(tm, inAction)
        matplotlib.pyplot.colorbar()
        matplotlib.pyplot.tight_layout()

        for i in range(confusionMatrix.shape[0]):
            for j in range(confusionMatrix.shape[1]):
                matplotlib.pyplot.text(j, i, confusionMatrix[i, j], horizontalalignment="center",
                                       color="white" if confusionMatrix[i, j] > th else "black")

        matplotlib.pyplot.savefig(os.path.join(OUT_IMG, 'ValidationConfusionMatrix'))

if __name__=='__main__':
    print("Running....")
    cls = MHIModel()
    print("Training...")
    cls.classifier()
    print("Validating...")
    cls.recognition("testVideo.avi", "outputTestVideo.avi")


from PyQt5.QtWidgets import QFileDialog
import webbrowser
from numpy import load, array
from os.path import splitext
from PIL import Image, ImageQt, ImageEnhance
from numpy.lib.npyio import save
DarkStyleSheet = '''QSlider::groove:horizontal{height:10px;margin:0 0}QSlider::handle:horizontal{background-color:#0096fa;border:1px;border-radius:3px;height:40px;width:40px;margin:0 0}QSlider::handle:horizontal:hover{background-color:#00c896;border:1px;border-radius:3px;height:40px;width:40px;margin:0 0}*{font:8pt "MS Shell Dlg 2"}#Tool_Box QLabel{font:10pt "MS Shell Dlg 2"}QGroupBox{color:#fff}QPushButton{background-color:#282828;border:none;border-radius:5px}QPushButton:hover{background-color:#323232;border:none}QPushButton:pressed{background-color:#0a0a0a;border:none}#VImage_Main{background-color:#1e1e1e}QLabel{color:#fff}QMenuBar{background:#282828;color:#fff}QMenu{background:#282828;color:#fff}QPushButton{color:white}'''
LightStyleSheet = '''QSlider::groove:horizontal{height:10px;margin:0 0}QSlider::handle:horizontal{background-color:#0096fa;border:1px;border-radius:3px;height:40px;width:40px;margin:0 0}QSlider::handle:horizontal:hover{background-color:#00c896;border:1px;border-radius:3px;height:40px;width:40px;margin:0 0}*{font:8pt "MS Shell Dlg 2"}#Tool_Box QLabel{font:10pt "MS Shell Dlg 2"}QPushButton{background-color:#e6f0f0;border:none;border-radius:5px}QPushButton:hover{background-color:#e6ffff;border:none}QPushButton:pressed{background-color:#c8ffff;border:none}#VImage_Main{background-color:#fff}'''

def Open():
    src_path, _ = QFileDialog.getOpenFileName(filter="Images (*.png *.jpg *.npy *.jpeg)")
    if(src_path != "" or src_path != None):
        name, exten = splitext(src_path)
        image = None
        if(exten == ".npy"):
            data = load(src_path)
            image = Image.fromarray(data)
        else:
            image = Image.open(src_path)
        return(src_path, ImageQt.ImageQt(image), image)
    else:
        return
def Save(image, dst_path):
    data = array(image)
    save(dst_path, data)

def SaveAs(image):
    dst_path, _ = QFileDialog.getSaveFileName(filter="VImage (*.npy)")
    if(dst_path == ""):
        return
    elif(dst_path != None):
        data = array(image)
        save(dst_path, data)
        return dst_path
    else:
        return

def DarkMode(window):
    window.setStyleSheet(DarkStyleSheet)

def LightMode(window):
    window.setStyleSheet(LightStyleSheet)

def AboutVCF():
    webbrowser.open("http://vcfstudio.in/about")

def AboutVImage():
    webbrowser.open("http://vcfstudio.in/vimage")

def ExportPNG(image):
    dst_path, _ = QFileDialog.getSaveFileName(filter="PNG (*.png)")
    if(dst_path == ""):
        return
    image.save(dst_path)

def ExportJPG(image):
    dst_path, _ = QFileDialog.getSaveFileName(filter="JPG (*.jpg *.jpeg)")
    if(dst_path == ""):
        return
    jpg_image = image.convert("RGB")
    jpg_image.save(dst_path)

def ExportPDF(image):
    dst_path, _ = QFileDialog.getSaveFileName(filter="PDF (*.pdf)")
    if(dst_path == ""):
        return
    pdf_image = image.convert("RGB")
    pdf_image.save(dst_path)


import argparse
import os

import torch
import torch.distributed as dist
from guided_diffusion import dist_util, logger
from guided_diffusion.script_util import add_dict_to_argparser

def main():
    args = create_argparser().parse_args()
    dist_util.setup_dist(args.gpus)
    logger.configure(dir=args.log_path)

    from deepfloyd_if.modules import IFStageI, IFStageII, StableStageIII
    from deepfloyd_if.pipelines.uncond_dream import uncond_dream
    if args.hf_token:
        from huggingface_hub import login
        login(token=args.hf_token)
    logger.log("creating diffusion models...")
    logger.log(f"stage: {args.stage}")
    if_I = None
    if_II = None
    if args.stage >= 1:
        os.makedirs(os.path.join(logger.get_dir(), "I"), exist_ok=True)
        if_I = IFStageI('IF-I-L-v1.0', device=dist_util.dev())
    if args.stage >= 2:
        os.makedirs(os.path.join(logger.get_dir(), "II"), exist_ok=True)
        if_II = IFStageII('IF-II-M-v1.0', device=dist_util.dev())

    n_saved = 0
    while n_saved * dist.get_world_size() + dist.get_rank() < args.num_samples:
        result = uncond_dream(
            if_I=if_I, if_II=if_II, if_III=None,
            batch_size=args.batch_size,
            if_I_kwargs={
                "sample_timestep_respacing": "smart100",
                "sample_loop": "ddim"
            },
            if_II_kwargs={
                "sample_timestep_respacing": "smart50",
                "sample_loop": "ddim"
            },
            disable_watermark=True,
            return_tensors=False
        )
        img = result

        for key, value in img.items():
            n_saved_tmp = n_saved
            for i in value:
                imgpath = os.path.join(logger.get_dir(), f"{key}/{n_saved_tmp * dist.get_world_size() + dist.get_rank():06}.png")
                if n_saved_tmp * dist.get_world_size() + dist.get_rank() > args.num_samples:
                    break
                i.save(imgpath)
                n_saved_tmp += 1
        n_saved = n_saved_tmp
        logger.log(f"{n_saved * dist.get_world_size() + dist.get_rank()} images are saved.")
    logger.log("sampling complete")

def create_argparser():
    default = dict(
        num_samples=10000,
        batch_size=10,
        log_path="sample_logs",
        hf_token=None,
        stage=3
    )
    parser = argparse.ArgumentParser()
    add_dict_to_argparser(parser, default)
    parser.add_argument("--gpus", type=str, nargs="+", default="0")
    return parser

if __name__ == "__main__":
    main()


from enum import IntEnum
from abc import ABC, abstractmethod
import random
import numpy as np
from typing import List
from copy import deepcopy

from .game import Game, Agent, Action

# Tic-Tac-Toe implementation

class GamePlayer(IntEnum):
  EMPTY = 0
  NAUGHT = 1
  CROSS = 2

BOARD_DIM = 3
BOARD_SIZE = BOARD_DIM**2

# Easier to just hardcode the lines we have to check for the winner.
# Board dimensions won't change anyways.
CHECK_LINES = [
  [0, 1, 2], # rows
  [3, 4, 5],
  [6, 7, 8],
  [0, 3, 6], # cols
  [1, 4, 7],
  [2, 5, 8],
  [0, 4, 8], # diagonals
  [2, 4, 6]
]

def agent_id_to_char(cell: GamePlayer):
  if (cell == GamePlayer.CROSS):
    return "x"
  if (cell == GamePlayer.NAUGHT):
    return "o"
  return " "

class TicTacToeAction(Action):
  def __init__(self, i_agent: int, position: int):
    super().__init__(i_agent)
    self.position = position

  def is_legal(self, game: 'TicTacToeGame') -> bool:
    return game.board[self.position] == GamePlayer.EMPTY

  def run(self, game: 'TicTacToeGame'):
    assert self.is_legal(game)
    game.board[self.position] = game.players[self.i_agent]

class TicTacToeGame(Game):
  def __init__(self):
    super().__init__(n_agents = 2)
    self.players = [GamePlayer.NAUGHT, GamePlayer.CROSS]
    self.board = np.ndarray(shape=(1, BOARD_SIZE), dtype=int)[0]
    self.board.fill(GamePlayer.EMPTY)

  def is_game_over(self) -> bool:
    for indexes in CHECK_LINES:
      line = [self.board[i] for i in indexes]
      if (line[0] != GamePlayer.EMPTY and line[0] == line[1] == line[2]):
        return True

    for value in self.board:
      if (value == GamePlayer.EMPTY):
        return False

    return True

  def get_score(self, i_agent: int) -> int:
    winner = self.get_winner()
    if (winner == self.players[i_agent]):
      return 1
    return 0

  def get_winner(self) -> GamePlayer:
    for indexes in CHECK_LINES:
      line = [self.board[i] for i in indexes]
      if (line[0] != GamePlayer.EMPTY and line[0] == line[1] == line[2]):
        return line[0]
    return None

  def get_legal_actions(self, i_agent: int) -> List[TicTacToeAction]:
    return [TicTacToeAction(i_agent, i) for i in range(len(self.board)) if self.board[i] == GamePlayer.EMPTY]

  def get_hash(self) -> int:
    res = 0
    for i in range(BOARD_SIZE):
        res *= 3
        res += self.board[i]
    return res

  def __str__(self) -> str:
    result = ""
    for y in range(BOARD_DIM):
      if y > 0:
        result += "\n-----\n"
      for x in range(BOARD_DIM):
        if x > 0:
          result += "|"
        result += agent_id_to_char(self.board[x + y * BOARD_DIM])

    return result


'''
Author       : ZHP
Date         : 2021-05-28 23:05:16
LastEditors  : ZHP
LastEditTime : 2022-04-24 12:32:58
FilePath     : /train_partseg.py
Description  : 项目说明
Copyright    : ZHP
'''
import os
import sys
import argparse
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import traceback
import torch
from trainer import shapePart_train
import models
from utils import utils

SEED = 10

def get_args():
    # Training settings
    parser = argparse.ArgumentParser(description='ShapePart Segmentation!')
    parser.add_argument('--task', type=str, default='PartSeg', help='Task')
    parser.add_argument('--model_name', type=str, default='Trans_Global_Part_SSG', help='model name')
    parser.add_argument('--ckpt_folder', type=str, default='Results/TrainLogs', help='Train Logs save dir')
    parser.add_argument('--model_save_dir', type=str, default='Results/models', help='model save dir')
    parser.add_argument('--enc_num', type=int, default=1, help='the transformer encoder num')
    parser.add_argument('--share', action='store_true', default=False, help='Resume training at a breakpoint')

    parser.add_argument('--batch_size', type=int, default=16, help='batch Size during training')
    parser.add_argument('--optimizer', type=str, default='Adam', help='Adam or SGD')
    parser.add_argument('--learning_rate', default=0.001, type=float, help='initial learning rate')
    parser.add_argument('--weight_decay', type=float, default=1e-4, help="weight decay")
    parser.add_argument('--min_lr', type=float, default=0.001, help="weight decay")
    parser.add_argument('--lr_decay', action='store_false', default=True, help='decay rate for learning rate')
    parser.add_argument('--lr_step', type=int, default=40, help='decay step for learning rate')
    parser.add_argument('--decay_ratio', type=float, default=0.5, help='decay ratio for learning rate')
    parser.add_argument('--nEpochs', type=int, default=200, help='number of epochs to train')
    parser.add_argument('--init', type=str, default='xavier', help='weight initial mode')
    parser.add_argument('--gpus', type=str, default='0,1', help='specify GPU devices')

    # data config
    parser.add_argument('--data_dir', type=str, default='/root/zhp/dataset/ShapeNet', help='data dir')
    parser.add_argument('--npoints', type=int, default=2048, help='point Number')
    parser.add_argument('--normalize', action='store_true', default=True, help='use normals')
    parser.add_argument('--resume', action='store_true', default=False, help='Resume training at a breakpoint')

    args = parser.parse_args()
    print('主要参数配置如下：\n')
    for key, value in args.__dict__.items():
        print(f'{key:^20} : {str(value):<}')
    return args

def main():
    torch.cuda.empty_cache()
    utils.fix_seed(SEED)
    args = get_args()
    model_dict = {
        "target_num" : 50,
        "embed_list" : [[128, 256], [256, 512], [512, 1024]],
        "pos_mode" : "relative",
        "res" : False
    }
    partseg = shapePart_train.PartSegTrainer(args.__dict__, models.get_loss, model_dict)
    try:
        partseg.run()
    except:
        partseg.trace_end()

if __name__ == "__main__":
    main()
    # python train_partseg.py --model_name=PPLT_Model_Seg


from sortedcontainers import SortedList
class MKAverage:

    def __init__(self, m: int, k: int):
        self.low = SortedList()
        self.mid = SortedList()
        self.high = SortedList()
        self.m = m
        self.sides = k
        self.mid1 = m - 2 * k
        self.deq = deque()
        self.sum1 = 0

    def addElement(self, num: int) -> None:
        self.deq.append(num)
        if not self.low or (num <= self.low[-1]):
            self.low.add(num)
        elif self.high and num >= self.high[0]:
            self.high.add(num)
        else:
            self.mid.add(num)
            self.sum1 += num
        if len(self.deq) > self.m:
            x = self.deq.popleft()
            if x <= self.low[-1]:
                self.low.remove(x)
            elif x >= self.high[0]:
                self.high.remove(x)
            else:
                self.mid.remove(x)
                self.sum1 -= x
        while len(self.low) < self.sides and self.mid:
            x = self.mid.pop(0)
            self.low.add(x)
            self.sum1 -= x
        while len(self.low) > self.sides:
            x = self.low.pop()
            self.mid.add(x)
            self.sum1 += x
        while len(self.high) < self.sides and self.mid:
            x = self.mid.pop()
            self.high.add(x)
            self.sum1 -= x
        while len(self.high) > self.sides:
            x = self.high.pop(0)
            self.mid.add(x)
            self.sum1 += x

    def calculateMKAverage(self) -> int:
        return -1 if len(self.mid) < self.mid1 else self.sum1 // self.mid1