import nuke
import os
import re
import glob
import subprocess
import json

class customreads():
    def __init__(self, node=None):
        super(customreads, self).__init__()
        self.group = nuke.thisNode() if node is None else node
        self.project = self.group.knob('project').value()
        self.structure = self.group.knob('structure').value()
        self.customfile = os.path.abspath(self.group.knob('Filepath').value())
        self.aovfolder = []
        self.framenr = "%04d"
        self.colorspace = "linear"
        self.extension = "exr"
        self.depthaov = 'depth'
        self.userbuttons = ["episode", "seq", "shot", "layer", "version", " ", "create", "populate",]
        self.buttontype = ('shader', 'data', 'crypto', 'lightgroups', 'custom')
        self.shaderaov = ['diffuse', 'coat', 'specular', 'sss', 'emission','sheen','check_user_albedo']
        self.dataaov = ['check_user_P','N',self.depthaov,'motionvector','background','cputime']
        self.cryptoaov = ['crypto_asset','crypto_object','crypto_material']
        self.lightgroupsaov = []
        self.customaov = []
        self.dictread()
        self.customtypehidden()
        # self.knobstart() probar con un if para que no se ejecute cada vez que abro un script.
        nuke.addKnobChanged(self.read, nodeClass='Group', node=self.group)

    def customtypehidden(self):
        customknobs = self.group.allKnobs()
        for knobs in customknobs:
            if "customreadclass" in knobs.name():
                self.group.removeKnob(knobs)
        customtype = nuke.Text_Knob('customreadclass', 'customreadclass')
        self.group.addKnob(customtype)
        customtype.setVisible(False)

    def dictread(self):
        self.seqsdictjson = open('P:/AndreJukebox/aj_seq_dict.json')
        self.seqsdict = json.load(self.seqsdictjson)

    def knobstart(self):
        self.createreadtabs()
        epsearchexp = f"{self.project}/*"
        self.epsearchpath = os.path.abspath(epsearchexp)
        epfolder = glob.glob(self.epsearchpath)
        self.episode = epfolder[0]
        seqsearchexp = f"{self.episode}/*"
        self.seqsearchpath = os.path.abspath(seqsearchexp)
        seqfolder = glob.glob(self.seqsearchpath)
        if seqfolder:
            self.seq = seqfolder[0]
        else:
            self.seq = ['']

        shotsearchexp = f"{self.seq}/*"
        self.shotsearchpath = os.path.abspath(shotsearchexp)
        shotfolder = glob.glob(self.shotsearchpath)
        if shotfolder:
            self.shot = shotfolder[0]
        else:
            self.shot = ['']

        layersearchexp = f"{self.shot}/{self.structure}/*"
        self.layersearchpath = os.path.abspath(layersearchexp)
        layerfolder = glob.glob(self.layersearchpath)
        if layerfolder:
            self.layer = layerfolder[0]
        else:
            self.layer = ['']

        versionsearchexp = f"{self.layer}/*"
        self.versionsearchpath = os.path.abspath(versionsearchexp)
        versionfolder = glob.glob(self.versionsearchpath)
        if versionfolder:
            self.version = versionfolder[-1].rsplit('\\')[-1]
            # print(self.version)
        else:
            self.version = ['']

        buttondict = {
            "episode": self.epsearchpath,
            "seq": self.seqsearchpath,
            "shot": self.shotsearchpath,
            "layer": self.layersearchpath,
            "version": self.versionsearchpath
        }

        for buttons in buttondict:
            searchpath = buttondict[buttons]
            self.buttongenerate(button=buttons,searchpath=searchpath)
        self.usergui()
        self.group.knobs()['seq'].setValue(nuke.root().knobs()['seqs'].value())
        self.group.knobs()['shot'].setValue(nuke.root().knobs()['shots'].value())

    def cleanui(self):
        all_knobs = self.group.allKnobs()
        for b in all_knobs:
            if b.name() in self.userbuttons:
                self.group.removeKnob(b)
            if "check_user" in b.name():
                self.group.removeKnob(b)
        self.group.knob('Filepath').setValue("")
        self.group.knob('Start').setValue(int(0))
        self.group.knob('End').setValue(int(0))
        self.group.knob('label').setValue('')
        self.cleantabs()
        self.cleanreadtabs()

    # Clears the user tab, deletes all nodes inside the self.group (except for "Ouput") and resets all labels and
    def cleannodes(self):
        all_nodes = self.group.nodes()
        for delnodes in all_nodes:
            if not "Output" in delnodes.name():
                nuke.delete(delnodes)

    def createreadtabs(self):
        tab_name = 'reads'
        readtab = nuke.Tab_Knob(tab_name)
        if not "reads" in self.group.knobs():
            self.group.addKnob(readtab)

    def cleanreadtabs(self):
        tab_knobs = self.group.allKnobs()
        tab_name = 'reads'
        for tabs in tab_knobs:
            if tab_name in tabs.name():
                self.group.removeKnob(tabs)

    def createtabs(self):
        self.cleantabs()
        aovtab = nuke.Tab_Knob("aov_tab")
        self.group.addKnob(aovtab)

        createallbutton = nuke.PyScript_Knob('all', 'all')
        self.group.addKnob(createallbutton)
        nuke.knobTooltip('Group.create', ('Enable all aov components'))
        enableallcode = """import customread
customread.customreads().enableallaovs()"""
        self.group.knob('all').setValue(enableallcode)

        createnonebutton = nuke.PyScript_Knob('none', 'none')
        self.group.addKnob(createnonebutton)
        nuke.knobTooltip('Group.create', ('Enable all aov components'))
        disableallcode = """import customread
customread.customreads().disableallaovs()"""

        for aovbutton in self.buttontype:
            createshaderbutton = nuke.PyScript_Knob(aovbutton, aovbutton)
            self.group.addKnob(createshaderbutton)
            nuke.knobTooltip('Group.create', ('Enable and disable aov components'))
            enablecode = """import customread
customread.customreads().""" + aovbutton + 'aovbutton()'
            self.group.knob(aovbutton).setValue(enablecode)
        self.group.knob('none').setValue(disableallcode)
        lgtstring = nuke.String_Knob('lgt', 'Lightgroups pattern', 'lgroup')
        self.group.addKnob(lgtstring)
        ctmstring = nuke.String_Knob('ctm', 'Custom pattern', '')
        self.group.addKnob(ctmstring)

    def cleantabs(self):
        tab_knobs = self.group.allKnobs()
        for tabs in tab_knobs:
                if tabs.name() in self.buttontype:
                    self.group.removeKnob(tabs)
                if tabs.name() in ("aov_tab","all","none","lgt","ctm"):
                    self.group.removeKnob(tabs)

    def shaderaovbutton(self):
        self.enableaovs(self.shaderaov)

    def dataaovbutton(self):
        self.enableaovs(self.dataaov)

    def cryptoaovbutton(self):
        self.enableaovs(self.cryptoaov)

    def lightgroupsaovbutton(self):
        knobchecks = self.group.allKnobs()
        lgrouppattern = self.group.knob('lgt').value()
        for i in knobchecks:
            if lgrouppattern in i.name():
                if self.group.knob(i.name()).getValue() == 1:
                    self.group.knob(i.name()).setValue(0)
                else:
                    if self.group.knob(i.name()).getValue() == 0:
                        self.group.knob(i.name()).setValue(1)
            else:
                pass

    def customaovbutton(self):
        knobchecks = self.group.allKnobs()
        ctmpattern = self.group.knob('ctm').value()
        for i in knobchecks:
            if ctmpattern in i.name():
                if self.group.knob(i.name()).getValue() == 1:
                    self.group.knob(i.name()).setValue(0)
                else:
                    if self.group.knob(i.name()).getValue() == 0:
                        self.group.knob(i.name()).setValue(1)
            else:
                pass

    def enableallaovs(self):
        knobchecks = self.group.allKnobs()
        for i in knobchecks:
            if "check_user" in i.name():
                self.group.knob(i.name()).setValue(1)
            else:
                pass

    def disableallaovs(self):
        knobchecks = self.group.allKnobs()
        for i in knobchecks:
            if "check_user" in i.name():
                self.group.knob(i.name()).setValue(0)
            else:
                pass

    def enableaovs(self,checkaovlist):
        knobchecks = self.group.allKnobs()
        for i in knobchecks:
            for t in checkaovlist:
                if t in i.name():
                    if self.group.knob(i.name()).getValue() == 1:
                        self.group.knob(i.name()).setValue(0)
                    else:
                        if self.group.knob(i.name()).getValue() == 0:
                            self.group.knob(i.name()).setValue(1)
                else:
                    pass

    def buttongenerate(self,button,searchpath):
        pathitem = os.path.abspath(searchpath)
        item = glob.glob(pathitem)
        additem = []
        for i in item:
            p = i.rsplit("\\")
            if not ".db" in p[-1]:
                additem.append(str(p[-1]))
        if button not in self.group.knobs():
            newknob = nuke.Enumeration_Knob(button, button, additem)
            self.group.addKnob(newknob)
        else:
            itemknob = self.group.knobs()[button]
            itemknob.setValues(additem)
        if button == "version":
            if len(additem) >= 1:
                self.group.knob(button).setValue(additem[-1])

    def checkclean(self):
        allchecks = self.group.allKnobs()
        self.checklist = []
        for checks in allchecks:
            if "check_user" in checks.name():
                self.checklist.append(checks)

                # self.group.removeKnob(checks)
        # print(self.checklist)

    def checkbox(self,checkbox):
        self.checklist = []
        self.checks = self.group.knobs()
        if not checkbox in ("rgba","beauty"):
            self.checkname = "check_user_" + checkbox
            checkstate = 0
            if self.checkname in self.checks:
                checkstate = self.group.knobs()[self.checkname].getValue()
                self.checklist.append(self.checkname)
                # print(f'This one {self.checkname} already existed')
                removecheck = self.group.knobs()[self.checkname]
                self.group.removeKnob(removecheck)

            else:
                print(f'creating new {self.checkname}')

            knobcheck = nuke.Boolean_Knob(self.checkname, checkbox, 0)
            self.group.addKnob(knobcheck)
            # knobcheck.setValue(1)
            knobcheck.setValue(checkstate)
            self.group[self.checkname].setFlag(nuke.STARTLINE)

    def connectchecks(self):
        selectreads = nuke.allNodes("Read")
        for t in selectreads:
            if not "beauty" in t.name():
                t.knob("disable").setExpression('parent.check_user_' + t.name() + ' == 1 ? 0 : 1')
                # shufflenames = t.name() + '_shf'
                # shuffles = nuke.toNode(shufflenames)
                # if not "crypto" in t.name():
                #     shuffles.knob("disable").setExpression('parent.check_user_' + t.name() + ' == 1 ? 0 : 1')

    def currentknobs(self):
        self.episode = self.group.knob('episode').value()
        self.seq = self.group.knob('seq').value()
        self.shot = self.group.knob('shot').value()
        self.layer = self.group.knob('layer').value()

    def update(self):
        self.currentknobs()
        buttoncurrent = ['episode','seq','shot','layer','version']
        for buttons in buttoncurrent:
            episode = self.group.knob('episode').value()
            seq = self.group.knob('seq').value()
            shot = self.group.knob('shot').value()
            layer = self.group.knob('layer').value()
            # Dictionary with current knob values
            buttonupdate = {
                "episode": f"{self.project}/*",
                "seq": f"{self.project}/{episode}/*",
                "shot": f"{self.project}/{episode}/{seq}/*",
                "layer": f"{self.project}/{episode}/{seq}/{shot}/{self.structure}/*",
                "version": f"{self.project}/{episode}/{seq}/{shot}/{self.structure}/{layer}/*"
            }
            for butup in buttonupdate:
                searchpath = buttonupdate[butup]
                currentvalue = self.group.knob(butup).value()
                self.buttongenerate(button=butup,searchpath=searchpath)

    def nodeconnect(self):
        # reads connect for reformat
        reads = nuke.allNodes("Read")
        self.zconnect()
        readnodes = []

        for r in reads:
            refin = nuke.toNode(r.name() + '_ref')
            # shfin = nuke.toNode(r.name() + '_shf')
            readin = nuke.toNode(r.name())
            readnodes.append(refin)
            if r.name() == "Z":
                zmergeshuffle = nuke.toNode('Z_shf')
                readnodes.append(zmergeshuffle)

            if not r.name() == "Z":
                nuke.Layer(r.name(),[r.name()+'.red',r.name()+'.green',r.name()+'.blue',r.name()+'.alpha'])

            if r.name() not in ('crypto_asset','crypto_object','crypto_material',self.depthaov):
                refin.setInput(0, r)
                # shfin.knob('out1').setValue(r.name())
                # if r.name() == "beauty":
                #     shfin.knob('out1').setValue('rgba')
                # shfin.knob('fromInput1').setValue("{1} B A")
                # shfin.setInput(1, refin)
            if "crypto" in r.name():
                refin.setInput(0, readin)
        createMerge = nuke.nodes.Merge2(name="aovmerge", A="none", also_merge="all", metainput="All")
        reformatnodes = nuke.allNodes(filter='Reformat')
        nro = 0
        for plugs in readnodes:
            nro = nro + 1
            if nro == 2:
                nro = nro + 1
            createMerge.setInput(nro, plugs)
        for refplugs in reformatnodes:
            if "beauty" in refplugs.name():
                createMerge.setInput(0, refplugs)

        # cryptoreads = nuke.allNodes(filter='Reformat')
        # for plugs in cryptoreads:
        #     nro = nro + 1
        #     if nro == 2:
        #         nro = nro + 1
        #     if "crypto" in plugs.name():
        #         createMerge.setInput(nro, plugs)
        nuke.toNode("Output1").setInput(0, nuke.toNode('aovmerge'))

    def aovharvest(self):
        # with self.group:
        # Define where to read from the items of the path
        project = self.group.knob('project').value()
        structure = self.group.knob('structure').value()
        episode = self.group.knob('episode').value()
        seq = self.group.knob('seq').value()
        shot = self.group.knob('shot').value()
        layer = self.group.knob('layer').value()
        version = self.group.knob('version').value()

        # STRING EXTRACTION
        # Sequence path
        framepath = f"{project}/{episode}/{seq}/{shot}/{structure}/{layer}/{version}/"
        # Converting all / to system defaults
        searchaovs = os.path.abspath(framepath + "*")

        # listing aov folders
        aovfolders = glob.glob(searchaovs)
        aovsplit = aovfolders[0].rsplit("\\")
        aovfolder = aovsplit[-1]
        searchpaths = os.path.abspath(framepath + aovfolder + "/*" + "." + self.extension)
        files = glob.glob(searchpaths)
        self.old_checks_list = []
        self.new_checks_list = []
        if files:
            # Storing the clean numbers in a variable
            setframestart = self.seqsdict[self.group.knob('seq').value()][self.group.knob('shot').value()]['start']
            setframeend = self.seqsdict[self.group.knob('seq').value()][self.group.knob('shot').value()]['end']
            for self.old_checks in self.group.knobs():
                if "check_user_" in self.old_checks:
                    self.old_checks_list.append(self.old_checks)
                    # print(self.old_checks)
                    # print(self.group.knob(self.old_checks))
                # print(self.old_checks)
                # checknode = nuke.toNode(self.old_checks)
                # print(checknode)
            #     if "check_user_" in self.current_checks['name']:
            #         self.old_checks_list.append(self.current_checks)

            #Create aov tab
            self.createtabs()
            # forming the full paths for found aovs

            for aovcomp in aovfolders:
                singleaov = aovcomp.rsplit("\\", -1)[-1]
                aovfullpath = f"{framepath}{singleaov}/{layer}_{singleaov}.{self.framenr}.{self.colorspace}.{self.extension}"
                self.checkbox(singleaov)
                new_check = "check_user_" + singleaov
                for new_checks in self.group.knobs():
                    if new_check == new_checks:
                        self.new_checks_list.append(new_check)

            #set paths strings and frame range
                if "beauty" in singleaov:
                    self.group.knob('Filepath').setValue(aovfullpath)
                    self.group.knob('Start').setValue(int(setframestart))
                    self.group.knob('End').setValue(int(setframeend))

                # create the reads
                if not "rgba" in singleaov:
                    nukepaths = re.sub(r"\\", "/", aovfullpath)
                    nuke.nodes.Read(file=nukepaths, first=setframestart, last=setframeend, name=singleaov,
                                    on_error="nearest frame")

                    # Create reformats and shuffles
                    if singleaov not in ('crypto_asset','crypto_object','crypto_material','depth','P','N','AA_inv_density','Pref'):# self.depthaov):
                        nuke.nodes.Reformat(name=singleaov + '_ref', filter='Lanczos4')
                        # nuke.nodes.Shuffle2(name=singleaov + '_shf')
                    elif singleaov not in self.depthaov:
                        nuke.nodes.Reformat(name=singleaov + '_ref', filter='impulse')
                        # nuke.nodes.Shuffle2(name=singleaov + '_shf')

            for items in self.old_checks_list:
                if items not in self.new_checks_list:
                    self.group.removeKnob(self.group.knob(items))

        else:
            print('No proper structure found for f´{layer}')

    def customaovharvest(self):
        self.old_checks_list = []
        self.new_checks_list = []
        # Clean self.group
        self.cleannodes()
        # STRING EXTRACTION
        # Sequence path
        aovseparator = 'beauty'
        beautysearch = os.path.abspath(self.customfile.rsplit(aovseparator)[-3]+aovseparator+'\*')
        aovsearch = self.customfile.split("beauty")[0] + '*'

        seqlist = glob.glob(beautysearch)
        # Storing digits between . and a .
        framestart = re.search("\.\d{4}(?:\.*)*", os.path.abspath(seqlist[0]))
        frameend = re.search("\.\d{4}(?:\.*)*", os.path.abspath(seqlist[-1]))
        for self.old_checks in self.group.knobs():
                if "check_user_" in self.old_checks:
                    self.old_checks_list.append(self.old_checks)
        # Matching the digits only
        framestartnumber = re.search("\d{4}", framestart.group(0))
        frameendnumber = re.search("\d{4}", frameend.group(0))

        # Storing the clean numbers in a variable
        setframestart = self.seqsdict[self.group.knob('seq').value()][self.group.knob('shot').value()]['start']#framestartnumber.group(0)
        setframeend = self.seqsdict[self.group.knob('seq').value()][self.group.knob('shot').value()]['end']#frameendnumber.group(0)

        #create tabs
        self.createtabs()

        # create the reads and the reformats
        aovpaths = glob.glob(aovsearch)

        for aovitem in aovpaths:
            singleaov = aovitem.rsplit('\\')[-1]
            self.checkbox(singleaov)
            new_check = "check_user_" + singleaov
            for new_checks in self.group.knobs():
                    if new_check == new_checks:
                        self.new_checks_list.append(new_check)
            if "beauty" in aovitem:
                self.group.knob('Start').setValue(int(setframestart))
                self.group.knob('End').setValue(int(setframeend))
            if not "rgba" in aovitem:
                setpath = re.sub("beauty", singleaov, self.customfile)
                nuke.nodes.Read(file=re.sub(r'\\','/',setpath), first=setframestart, last=setframeend, name=singleaov,
                                on_error="nearest frame")
                if singleaov not in ('crypto_asset','crypto_object','crypto_material','depth','P','N','AA_inv_density','Pref'):
                    nuke.nodes.Reformat(name=singleaov + '_ref', filter='Lanczos4')
                    nuke.nodes.Shuffle2(name=singleaov + '_shf')
                elif singleaov not in self.depthaov:
                    nuke.nodes.Reformat(name=singleaov + '_ref', filter='impulse')
                    nuke.nodes.Shuffle2(name=singleaov + '_shf')
        for items in self.old_checks_list:
                if items not in self.new_checks_list:
                    self.group.removeKnob(self.group.knob(items))
        self.nodeconnect()
        self.connectchecks()
        self.customlabel()
        self.setrootframes()

    def usergui(self):
        # if divide line exist, it deletes it and re creates it
        if not ' ' in self.group.knobs():
            divideline = nuke.Text_Knob(' ')
            self.group.addKnob(divideline)

        # if create button exist, it deletes it and re creates it
        if not "create" in self.group.knobs():
            # self.group.removeKnob(n)
            createbutton = nuke.PyScript_Knob("create", "Create")
            self.group.addKnob(createbutton)
            nuke.knobTooltip('Group.create', ('Creates all the reads for the found aovs'))
            createcode = """import customread
customread.customreads().createfunc()
"""
            self.group.knob("create").setValue(createcode)

        # if populate button exist, it deletes it and re creates it
        allthepopulates = self.group.allKnobs()

        if not "populate" in self.group.knobs():
            populatebutton = nuke.PyScript_Knob("populate", "Re-populate")
            self.group.addKnob(populatebutton)
            nuke.knobTooltip('Group.populate', 'Updates all pulldown items')
            populatecode = """import customread
customread.customreads().realoadfunc()"""
            self.group.knob("populate").setValue(populatecode)
    # For the callback, condition that this knobs show change execute the update() func.

    def read(self):
        knobflick = nuke.thisKnob()
        if knobflick.name() in ("episode","seq","shot","layer"):#,"version"):
            self.update()

    # This function will execute the reload from every read node inside the self.group

    def reloadreads(self):
        allreads = nuke.allNodes(filter='Read')
        for node in allreads:
            node['reload'].execute()

    # This function will override the frames from the GUI into all read nodes inside the self.group
    def overrideframes(self):
        all_nodes = nuke.allNodes(filter="Read")
        overridestart = self.group.knob('Start').value()
        overrideend = self.group.knob('End').value()
        for readnodes in all_nodes:
            readnodes.knob('first').setValue(int(overridestart))
            readnodes.knob('last').setValue(int(overrideend))

    def labelset(self):
        seq = self.group.knob('seq').value()
        shot = self.group.knob('shot').value()
        layer = self.group.knob('layer').value()
        version = self.group.knob('version').value()
        self.grouplabel = f'{seq}_{shot}\n{layer}\n{version}'
        self.group.knob('label').setValue(self.grouplabel)

    def customlabel(self):
        seq = self.customfile.split('\\')[4]
        shot = self.customfile.split('\\')[5]
        file = self.customfile.split('\\')[10]
        layer = file.split('_')[0] + '_' + file.split('_')[1]
        ststart = self.group.knob('Start').value()
        stend = self.group.knob('End').value()
        version = self.customfile.split('\\')[8]
        self.grouplabel = f'{seq}_{shot}\n{layer}\n{version}'
        self.group.knob('label').setValue(self.grouplabel)

    def openexplorer(self):
        finalpath = self.group.knob('Filepath').value()
        pathsplit = os.path.abspath(finalpath.rsplit('/',2)[0])
        subprocess.call(f'explorer {pathsplit}', shell=True)

    def openseqrv(self):
        finalpath = self.group.knob('Filepath').value()
        pathsplit = os.path.abspath(finalpath.rsplit('/', 2)[0]) + '\\' + 'rgba'
        rv = "rv.exe"
        cmline = f'{rv} {pathsplit}'
        os.system(f'start cmd /c {cmline}')

    def setrootframes(self):
        rootframestart = self.seqsdict[self.group.knob('seq').value()][self.group.knob('shot').value()]['start']
        rootframeend = self.seqsdict[self.group.knob('seq').value()][self.group.knob('shot').value()]['end']
        nuke.root().knobs()['first_frame'].setValue(int(rootframestart))
        nuke.root().knobs()['last_frame'].setValue(int(rootframeend))

    def zconnect(self):
        allreads = nuke.allNodes("Read")
        for zread in allreads:
            if zread.name() in self.depthaov:
                zread = nuke.toNode(self.depthaov)
                createZref = nuke.nodes.Reformat(name=self.depthaov + '_ref', filter='impulse')
                # createZmerge = nuke.nodes.Merge2(name=self.depthaov + "_shf", A="none", output = self.depthaov )
                createZref.setInput(0, zread)
                # createZmerge.setInput(0, createZref)

    # Funtions for the buttons
    def createfunc(self):
        with self.group:
            self.cleannodes()
            self.aovharvest()
            self.checkclean()
            self.nodeconnect()
            self.connectchecks()
            self.labelset()
            self.setrootframes()

    def realoadfunc(self):
        self.update()


"""
This file as well as the other examples will store the augmented images in the vyn-augment/images/output_data folder.
On the other hand, the folder notebooks contains the same examples with the images being plotted in the same file
instead of being saved in memory. In addition, a better explanation about the augmentor options is provided.
"""
import os
import sys

import numpy as np
from skimage.io import imread, imsave

sys.path.append(os.path.dirname(os.path.dirname(__file__)))
from utils.data_processing import set_generator_segmentor
from src.vyn_augment.augmentor import Augmentor

def pre_processing_function(mask_filename: str, filename: str, augmentor: Augmentor = None):
    """
    Pre-processing function. This function is run within the generator, which calls it for each individual image
    regardless the batch size.
    :param mask_filename: The filename where the mask is stored. This function will read the image into a numpy array.
    :param filename: The complete path to the image file. This function will read the image into a numpy array.
    :param augmentor: An object of type Augmentor
    :return:
    """
    image = imread(filename)
    mask = imread(mask_filename)
    if augmentor is not None:
        image, mask = augmentor.run([image, mask], mask_positions=[1])
        image = np.round(image).astype(np.uint8)
        mask = np.round(mask).astype(np.uint8)

    # Returning the name or label is not mandatory, it will just be used to give a name to the file when saving the
    # augmented image
    name = os.path.basename(filename)
    name = name[:name.rfind('.')]
    return image, [mask, name]

def set_augmentor():
    """
    Set the augmentor.
    1. Select the operations and create the config dictionary
    2. Pass it to the Augmentor class with any other information that requires
    3. Return the instance of the class.
    :return:
    """
    config = {'blur': {'values': ('gaussian', 0.7, 1.0), 'prob': 0.3},
              'brightness': {'values': (0.6, 1.0), 'prob': 0.1},
              'brightness1': {'values': (1.0, 1.5), 'prob': 0.1},
              'flip': {'values': ('random',), 'prob': 0.5},
              'occlusion': {'values': ('hide_and_seek', 0.2, 0.4, 0.2, 0.3), 'prob': 0.4},
              'posterisation': {'values': (32, 64), 'prob': 0.5},
              'rotate': {'values': (-45, 45), 'prob': 0.4},
              'translate': {'values': ('RANDOM', -0.2, 0.2), 'prob': 0.2, 'use_replication': False},
              'zoom': {'values': (0.5, 1.5), 'prob': 0.9, 'use_replication': False}}

    augmentor = Augmentor(config, no_repetition=True)

    return augmentor

def generate_n_augmented_images(data_dirname: str, root_dirname: str, n=20) -> None:
    """
    Generate n new augmented images, where n is an input parameter
    :param data_dirname: The directory where the initial set of images are.
    :param root_dirname: The directory where to save the augmented set of images.
    :param n: The number of augmented images
    :return: None
    """
    input_image_dirname = os.path.join(data_dirname, 'images')
    input_mask_dirname = os.path.join(data_dirname, 'masks')

    augmentor = set_augmentor()
    preprocessing_fun = lambda *args: pre_processing_function(*args, augmentor=augmentor)
    generator = set_generator_segmentor(input_image_dirname, input_mask_dirname, preprocessing_fun,
                                        batch_size=1, number_of_images=n)
    generator.not_batch = True
    # NOTICE: This generator can be used for keras and pytorch in case that instead of saving images one desires to
    # augment images on the fly. Use a number larger than 1 for the batch size when training directly a CNN.

    image_dirname = os.path.join(root_dirname, 'images')
    mask_dirname = os.path.join(root_dirname, 'masks')
    if not os.path.isdir(image_dirname):
        os.makedirs(image_dirname)
    if not os.path.isdir(mask_dirname):
        os.makedirs(mask_dirname)
    # Save the new generated images
    counter_labels = {}
    for image, info in generator:
        mask = info[0]
        name = info[1]
        counter = counter_labels.get(name, 0)
        image_filename = name + '_' + str(counter).zfill(4) + '.jpg'
        mask_filename = name + '_' + str(counter).zfill(4) + '.png'

        filename = os.path.join(image_dirname, image_filename)
        imsave(filename, image.astype(np.uint8))

        filename = os.path.join(mask_dirname, mask_filename)
        imsave(filename, mask.astype(np.uint8))

        counter_labels[name] = counter + 1

    print(f'Finished image generation. The output images were saved in {root_dirname}')

if __name__ == '__main__':
    root = os.path.dirname(os.path.dirname(__file__))
    data_dirname = os.path.join(root, 'images', 'segmentation')
    root_dirname = os.path.join(root, 'images', 'output_data', 'segmentation')

    generate_n_augmented_images(data_dirname, root_dirname)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

#    This file is part of Vallenato.fr.
#
#    Vallenato.fr is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    Vallenato.fr is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with Vallenato.fr.  If not, see <http://www.gnu.org/licenses/>.

# Adapted from:
#   https://github.com/youtube/api-samples/blob/master/python/my_uploads.py
#   https://github.com/youtube/api-samples/blob/master/python/upload_video.py

import httplib2
import os
import logging
import sys

from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from oauth2client.file import Storage
from oauth2client.client import flow_from_clientsecrets
from oauth2client.tools import run_flow

# The CLIENT_SECRETS_FILE variable specifies the name of a file that contains
# the OAuth 2.0 information for this application, including its client_id and
# client_secret. You can acquire an OAuth 2.0 client ID and client secret from
# the {{ Google Cloud Console }} at
# {{ https://cloud.google.com/console }}.
# Please ensure that you have enabled the YouTube Data API for your project.
# For more information about using OAuth2 to access the YouTube Data API, see:
#   https://developers.google.com/youtube/v3/guides/authentication
# For more information about the client_secrets.json file format, see:
#   https://developers.google.com/api-client-library/python/guide/aaa_client_secrets
CLIENT_SECRETS_FILE = 'client_secret.json'

# This OAuth 2.0 access scope allows an application to view videos and playlists
SCOPES = ['https://www.googleapis.com/auth/youtube.readonly']
API_SERVICE_NAME = 'youtube'
API_VERSION = 'v3'

# This variable defines a message to display if the CLIENT_SECRETS_FILE is
# missing.
MISSING_CLIENT_SECRETS_MESSAGE = """
WARNING: Please configure OAuth 2.0

To make this sample run you will need to populate the client_secrets.json file
found at:
   %s
with information from the APIs Console
https://console.developers.google.com

For more information about the client_secrets.json file format, please visit:
https://developers.google.com/api-client-library/python/guide/aaa_client_secrets
""" % os.path.abspath(os.path.join(os.path.dirname(__file__), CLIENT_SECRETS_FILE))

# Authorize the request and store authorization credentials.
def yt_get_authenticated_service(args):
    flow = flow_from_clientsecrets(CLIENT_SECRETS_FILE, scope=SCOPES, message=MISSING_CLIENT_SECRETS_MESSAGE)
    storage = Storage("vallenato.fr-oauth2.json")
    credentials = storage.get()
    if credentials is None or credentials.invalid:
        credentials = run_flow(flow, storage, args)
    return build(API_SERVICE_NAME, API_VERSION, credentials = credentials, cache_discovery=False)

def yt_get_my_uploads_list(youtube):
    return "UU_8R235jg1ld6MCMOzz2khQ"
    # # Retrieve the contentDetails part of the channel resource for the
    # # authenticated user's channel.
    # channels_response = youtube.channels().list(
    #     mine=True,
    #     part='contentDetails'
    # ).execute()
    #
    # for channel in channels_response['items']:
    #     # From the API response, extract the playlist ID that identifies the list
    #     # of videos uploaded to the authenticated user's channel.
    #     return channel['contentDetails']['relatedPlaylists']['uploads']
    # return None

def yt_list_my_uploaded_videos(uploads_playlist_id, youtube):
    uploaded_videos = []
    # Retrieve the list of videos uploaded to the authenticated user's channel.
    playlistitems_list_request = youtube.playlistItems().list(
        playlistId=uploads_playlist_id,
        part='contentDetails'
    )

    logging.debug('Videos in list %s' % uploads_playlist_id)
    while playlistitems_list_request:
        playlistitems_list_response = playlistitems_list_request.execute()

        videoIds = ",".join(playlist_item['contentDetails']['videoId'] for playlist_item in playlistitems_list_response['items'])

        # To get the videos' tags, we need to do an extra query
        videos_list_request = youtube.videos().list(
            id=videoIds,
            part='snippet,status'
        )

        while videos_list_request:
            videos_list_response = videos_list_request.execute()

            for video in videos_list_response['items']:
                # Only keep Public videos
                if video['status']['privacyStatus'] != "public":
                    continue
                # Skip videos tagged as "no-website" or "Tutorial"
                if 'tags' in video['snippet'] and (
                    "no-website" in video['snippet']['tags'] or
                    "Tutorial" in video['snippet']['tags']):
                    continue
                vid = {}
                vid["id"] = video['id']
                vid["title"] = video['snippet']['title']
                vid["description"] = video['snippet']['description']
                try:
                    vid["tags"] = video['snippet']['tags']
                except KeyError:
                    vid["tags"] = []
                vid["publishedAt"] = video['snippet']['publishedAt']
                # Sizes: default, medium, high, standard, maxres
                vid["thumbnail"] = video['snippet']['thumbnails']['medium']

                uploaded_videos.append(vid)
                logging.debug(vid)
            videos_list_request = youtube.videos().list_next(videos_list_request, videos_list_response)

        playlistitems_list_request = youtube.playlistItems().list_next(playlistitems_list_request, playlistitems_list_response)
    return uploaded_videos


import itertools
n, w = map(int, input().split())
a = list(map(int, input().split()))

s = set()
for i in range(len(a)):
  if a[i] <= w:
    s.add(a[i])

l = list(itertools.combinations(a,2))
for i in range(len(l)):
  if sum(l[i]) <= w:
    s.add(sum(l[i]))

l2 = list(itertools.combinations(a,3))
for i in range(len(l2)):
  if sum(l2[i]) <= w:

    s.add(sum(l2[i]))

print(len(s))


"""This module tests the compat.py module."""

import numpy as np
from pathlib import Path
import pytest

from acmpy.compat import require_int, require_nonnegint, require_posint, \
    parse_line_float, readdata_float, \
    NDArrayFloat, is_nd_float, is_nd_vector, is_nd_matrix, is_nd_square

class TestRequireInt:
    """Tests require_int(name, value)."""

    def test_ok(self):
        require_int('x', -1)
        require_int('y', 0)
        require_int('z', 1)

    def test_type_error(self):
        with pytest.raises(TypeError):
            require_int('x', 1.0)  # type: ignore
        with pytest.raises(TypeError):
            require_int('y', '1')  # type: ignore

class TestRequireNonNegInt:
    """Tests require_nonnegint(name, value)."""

    def test_ok(self):
        require_nonnegint('y', 0)
        require_nonnegint('z', 1)

    def test_type_error(self):
        with pytest.raises(TypeError):
            require_nonnegint('x', 1.0)  # type: ignore
        with pytest.raises(TypeError):
            require_nonnegint('y', '1')  # type: ignore

    def test_value_error(self):
        with pytest.raises(ValueError):
            require_nonnegint('x', -1)

class TestRequirePosInt:
    """Tests require_posint(name, value)."""

    def test_ok(self):
        require_posint('z', 1)

    def test_type_error(self):
        with pytest.raises(TypeError):
            require_posint('x', 1.0)  # type: ignore
        with pytest.raises(TypeError):
            require_posint('y', '1')  # type: ignore

    def test_value_error(self):
        with pytest.raises(ValueError):
            require_posint('x', -1)
        with pytest.raises(ValueError):
            require_posint('y', 0)

class TestParseLineFloat:
    """Tests parse_line_float(line)."""

    def test_ok(self):
        line: str = ' +1.000000e+00      0    1    0      1    1    2      1    1    2'
        value: float = parse_line_float(line)
        assert value == 1.0

    def test_error(self):
        line: str = 'abc'
        with pytest.raises(ValueError):
            parse_line_float(line)

class TestReadDataFloat:
    """Tests readdata_float(filename)."""

    def test_ok(self, tmp_path: Path):
        filepath: Path = tmp_path / "data"
        filepath.write_text('1.0 1 1\n2.0 2 2\n3.0 3 3\n')
        filename: str = str(filepath)
        data: list[float] = readdata_float(filename)
        assert len(data) == 3
        assert data[0] == 1.0
        assert data[1] == 2.0
        assert data[2] == 3.0

class TestIs_nd_float:
    """Tests the is_nd_float() function."""

    @pytest.mark.parametrize(
        "a,b",
        [(np.array([1.0]), True),
         (np.array([1], dtype='int'), False)]
    )
    def test_ok(self, a: NDArrayFloat, b: bool):
        assert is_nd_float(a) == b

class TestIs_nd_vector:
    """Tests the is_nd_vector() function."""

    @pytest.mark.parametrize(
        "a,b",
        [(np.array([1.0]), True),
         (np.array([1.0, 2.0]), True),
         (np.array([1], dtype='int'), False),
         (np.array([[1.0], [1.0]]), False)]
    )
    def test_ok(self, a: NDArrayFloat, b: bool):
        assert is_nd_vector(a) == b

class TestIs_nd_matrix:
    """Tests the is_nd_matrix() function."""

    @pytest.mark.parametrize(
        "a,b",
        [(np.array([1.0]), False),
         (np.array([1.0, 2.0]), False),
         (np.array([1], dtype='int'), False),
         (np.array([[1.0], [1.0]]), True)]
    )
    def test_ok(self, a: NDArrayFloat, b: bool):
        assert is_nd_matrix(a) == b

class TestIs_nd_square:
    """Tests the is_nd_square() function."""

    @pytest.mark.parametrize(
        "a,b",
        [(np.array([[1, 0], [0, 1]], dtype='int'), False),
         (np.array([[1.0, 0.0], [0.0, 1.0]]), True),
         (np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]]), False)]
    )
    def test_ok(self, a: NDArrayFloat, b: bool):
        assert is_nd_square(a) == b


from scraper_classes.chrome_scraper import ChromeScraper
from scrapers.logger import get_logger
from bs4 import BeautifulSoup
import csv

class GlobalDairyTrade(ChromeScraper):

    logger = get_logger("GlobalDairyTrade")

    def scrape(self):
        self.logger.info("Starting")
        self.url = "https://www.globaldairytrade.info/en/product-results/"
        self.driver.get(self.url)

        soup = BeautifulSoup(self.driver.page_source, "lxml")

        # List for use in finding event_title
        event_title_tags = soup.find("div", {"class": "moduleEventDetails moduleEventDetailsSummary nextEventDate"})
        # list for use in finding change_in_gdt and average_price. This list should always be len(2)
        gdt_price_index = soup.find_all("div", {"class": "moduleChangePriceIndex"})

        event_title = " ".join([x.text.strip() for x in event_title_tags if "\n" not in x])
        change_in_gdt = (gdt_price_index[0].h4.text,
                         "".join([x.text.strip() for x in gdt_price_index[0].find_all("td") if "\n" not in x]))

        average_price = (gdt_price_index[1].h4.text,
                         "".join([x.text.strip() for x in gdt_price_index[1].div if "\n" not in x]))

        products_tags = soup.find("div", {"class": "productKnobs"})

        # This list contains <div> of Products (Anhydrous Milk Fat, Butter, Butter Milk Powder, etc.)
        products = products_tags.div.find_all("div", {"class", "productFloat"})
        filtered_products = [x.find_all("span") for x in products]

        write_list = []
        write_list.append(["Product", average_price[0], change_in_gdt[0]])
        write_list.append([event_title, average_price[1], change_in_gdt[1]])

        unwanted = ["not offered at this event", "information currently unavailable"]
        # Append final data to write_list
        for product in filtered_products:
            # ['Anhydrous Milk Fat', '$5,530', '-3.3', '%', '-3.3%$5530']
            row = [x.text.strip() for x in product if x.text.strip().lower() not in unwanted]
            if self.is_number(row[1]):
                write_list.append([row[0], row[1], row[2] + row[3]])
            else:
                write_list.append([row[0], "na", "na"])

        # Write write_list to file
        with open(self.CSV_FILE_PATH, "w+", newline="") as file:
            writer = csv.writer(file, quoting=csv.QUOTE_ALL)
            for row in write_list:
                writer.writerow(row)
        self.logger.info("Finished")

    def is_number(self, x):
        """Method checks whether a string price (e.g. $4,700) contains a valid number."""
        symbols = ["$", "€", "£", "¥"]
        if "n" not in x and "a" not in x:
            if len(x) > 1 and x[0] in symbols:
                x = x[1:]
                if x.isdigit() is True or self.is_decimal(x) is True:
                    return True
                elif "," in x:
                    x = x.split(",")
                    if x[0].isdigit() and x[1].isdigit():
                        return True
        return False

    def is_decimal(self, n):
        """Checks whether a number is a decimal number or not. Method takes a string."""
        n = n.split(".")
        if len(n) == 2:
            if n[0].isdigit() and n[1].isdigit():
                return True
            elif "-" in n[0] and n[0][1:].isdigit():
                return True
        return False

if __name__ == "__main__":
    GlobalDairyTrade().scrape()


from argparse import ArgumentParser
import logging
import config
from requests import get
from os import path, makedirs
from bs4 import BeautifulSoup
from datetime import datetime

def download_file(url, outputfolder, logger):
    """
    helper file to download any file to specified folder
    :param url: url of the file
    :param outputfolder: folder to save the file to
    :param logger:
    :return:
    """
    logger.info('download actual dump')
    filename = url.split('/')[-1]
    local_filename = path.join(outputfolder, filename)

    r = get(url, stream=True)
    download_size = int(r.headers['Content-Length'])

    # check if file is already downloaded
    if path.isfile(local_filename):
        local_size = path.getsize(local_filename)
        if local_size == download_size:
            logger.info('skip: ' + filename)
            return local_filename

    logger.info('{} ({:,d} Bytes)'.format(url, download_size))
    with open(local_filename, 'wb') as f:
        size_to_log = download_size / 20  # 5% steps
        downloaded = 0
        last_time = datetime.now()
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks
                downloaded += 1024
                if downloaded % (size_to_log) < 1024:
                    actual_time = datetime.now()
                    delta = actual_time - last_time
                    logger.info(
                        '{done:>5,d} MB ({done_percent:>6.{digits}f}%, {speed:>10.{digits}f} KB/s)'
                        .format(
                            done=downloaded / (1048576),
                            done_percent=100 * float(downloaded) / download_size,
                            speed=(size_to_log / (delta.seconds + delta.microseconds / 1E6)) / 1024,
                            digits=2,
                        )
                    )
                    last_time = actual_time
                f.write(chunk)
                f.flush()
    logger.info('finished downloading')
    return local_filename

def download_wd_dump(outputfolder, logger):
    """
    downloads the latest wikidata json dump
    :param outputfolder:
    :param logger:
    :return:
    """
    if not path.exists(outputfolder):
        logger.info('create outputfolder')
        makedirs(outputfolder)

    resp = get(config.WIKIDATA_DUMPS_URL.format(''))
    soup = BeautifulSoup(resp.text)
    all_dumps = set()
    for a in soup.find_all('a'):
        href = a.attrs['href'].strip()
        if not href == '../':
            all_dumps.add(href)

    latest_dump = sorted(all_dumps)[-1]
    download_file(config.WIKIDATA_DUMPS_URL.format(latest_dump), outputfolder, logger)

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument("-f", "--folder", default='dumps',
                        help="folder where the wikidata dumps are getting downloaded to")
    args = parser.parse_args()

    # set up logging to file
    logging.basicConfig(level=logging.DEBUG,
                        format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                        datefmt='%m-%d %H:%M',
                        filename='wd_download.log',
                        filemode='a')

    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s', "%H:%M:%S")
    console.setFormatter(formatter)
    logging.getLogger('').addHandler(console)

    download_wd_dump(args.folder, logging.getLogger('download'))


import argparse
import datetime
from enum import Enum
from pathlib import Path
from typing import Iterable, Dict
import pandas as pd
import requests

BEGINNING_OF_NBA_STAT_COLLECTION = 1996

HEADERS = {
    'Host': 'stats.nba.com',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:72.0) Gecko/20100101 Firefox/72.0',
    'Accept': 'application/json, text/plain, */*',
    'Accept-Language': 'en-US,en;q=0.5',
    'Accept-Encoding': 'gzip, deflate, br',
    'x-nba-stats-origin': 'stats',
    'x-nba-stats-token': 'true',
    'Connection': 'keep-alive',
    'Referer': 'https://stats.nba.com/',
    'Pragma': 'no-cache',
    'Cache-Control': 'no-cache',
}

class TargetNBAObject(str, Enum):
    Players = 'players'
    Teams = 'teams'

TARGET_NBA_OBJECT_URL_MAP = {
    TargetNBAObject.Players: lambda year:
    f"https://stats.nba.com/stats/leaguedashplayerstats?College=&Conference=&Country=&DateFrom=&DateTo=&Division=&DraftPick=&DraftYear=&GameScope=&GameSegment=&Height=&LastNGames=0&LeagueID=00&Location=&MeasureType=Base&Month=0&OpponentTeamID=0&Outcome=&PORound=0&PaceAdjust=N&PerMode=Totals&Period=0&PlayerExperience=&PlayerPosition=&PlusMinus=N&Rank=N&Season={year}-{str(year + 1)[-2:]}&SeasonSegment=&SeasonType=Regular%20Season&ShotClockRange=&StarterBench=&TeamID=0&VsConference=&VsDivision=&Weight=",
    TargetNBAObject.Teams: lambda year:
    f"https://stats.nba.com/stats/leaguedashteamstats?Conference=&DateFrom=&DateTo=&Division=&GameScope=&GameSegment=&Height=&LastNGames=0&LeagueID=00&Location=&MeasureType=Base&Month=0&OpponentTeamID=0&Outcome=&PORound=0&PaceAdjust=N&PerMode=Totals&Period=0&PlayerExperience=&PlayerPosition=&PlusMinus=N&Rank=N&Season={year}-{str(year + 1)[-2:]}&SeasonSegment=&SeasonType=Regular%20Season&ShotClockRange=&StarterBench=&TeamID=0&TwoWay=0&VsConference=&VsDivision="
}

def valid_nba_year(year: str) -> int:
    year_as_int = int(year)
    assert year_as_int >= BEGINNING_OF_NBA_STAT_COLLECTION
    assert year_as_int <= datetime.date.today().year
    return year_as_int

def get_stats_from_nba_website(years: Iterable[int], target_nba_object: TargetNBAObject) -> Dict[int, pd.DataFrame]:
    years_stats_dfs_dict = {}
    for year in years:
        year_data = requests.get(TARGET_NBA_OBJECT_URL_MAP[target_nba_object](year),
                                 headers=HEADERS
                                 ).json()
        rows = year_data['resultSets'][0]['rowSet']
        headers = year_data['resultSets'][0]['headers']
        year_df = pd.DataFrame(rows, columns=headers)
        years_stats_dfs_dict[year] = year_df

    return years_stats_dfs_dict

def combine_years_stats(years_stats_dfs_dict: Dict[int, pd.DataFrame]) -> pd.DataFrame:
    seasons_stats_dict = {}
    for year, year_df in years_stats_dfs_dict.items():
        year_df['SEASON'] = year
        seasons_stats_dict[year] = year_df

    all_seasons_df = pd.concat(seasons_stats_dict.values())
    return all_seasons_df

def get_and_format_nba_stats(years: Iterable[int], target_nba_object: TargetNBAObject, output_file_path: Path):
    years_stats_dfs_dict: Dict[int, pd.DataFrame] = get_stats_from_nba_website(years, target_nba_object)
    combined_stats: pd.DataFrame = combine_years_stats(years_stats_dfs_dict)
    combined_stats.to_csv(output_file_path, index=False)

def main():
    parser = argparse.ArgumentParser(
        prog='obtain_nba_data',
        description='',
    )

    parser.add_argument('beginning_year', type=valid_nba_year)
    parser.add_argument('end_year', type=valid_nba_year)
    parser.add_argument('target_nba_object', choices=[target_object.value for target_object in TargetNBAObject])
    parser.add_argument('output_file_path', type=Path)

    args = parser.parse_args()

    years_range = range(args.beginning_year, args.end_year + 1)
    target_nba_object = args.target_nba_object
    output_file_path = args.output_file_path

    get_and_format_nba_stats(years_range,
                             target_nba_object,
                             output_file_path)
    print("Success!")

if __name__ == '__main__':
    main()


# coding:utf-8
import json
import thread
import time
import requests
import os
from unittest import TestCase
from flask import Flask, request

app = Flask(__name__)
#__test_case = TestCase()
#http://0.0.0.0:9022/api/v1/daemons
__base_url = '/api/v1/'
__agent_nic1 = 'ens33'
__agent_nic2 = 'ens32'
__daemon = {}
__sync_cnts = 0
__remote_ip1 = '147.147.147.147'
__remote_ip2 = '146.146.146.146'
__fp = open('err.log', 'a+')

def resp_with_error(sc, code, message):
    rep = {
        'code': code,
        'message': message
    }
    rep_js = json.dumps(rep)
    return rep_js, sc

def add_agent(req, id, key, filter):
    global __daemon, __remote_ip1, __remote_ip2, __agent_nic1, __agent_nic2

    req['agent'] = {
        "id": id,
        "agentType": "PA",
        "cpuLimit": 0.1, #10%cpu
        "memLimit": 40, #40M
        "packetAgentStrategies": [{
            "serviceTag": 1,
            "startup": "",
            'networkInterface': {
                "name" : __agent_nic1
            },
            "bpf": filter,
            "packetChannelType": "GRE",
            "address": __remote_ip1
        },
        {
            "serviceTag": 1,
            "startup": "",
            'networkInterface': {
                "name" : __agent_nic2
            },
            "bpf": filter,
            "packetChannelType": "VXLAN",
            "address": __remote_ip2
        }]
    }

@app.route(__base_url + 'daemons', methods=['POST'])
def post_daemons():
    global __daemon, __test_case, __fp

    req_data = request.get_data()
    req = json.loads(req_data.decode("utf-8"))
    if req['syncMode'] != 'pull':
        __fp.write("Err: mode is not pull in REG\n")
        return resp_with_error(404, 404, "syncMode not pull")

    if not req.has_key('networkInterfaces'):
        __fp.write("Err: no networkInterfaces in REG\n")
        return resp_with_error(404, 404, "no networkInterfaces in REG")

    rep_js = json.dumps(req)
    return rep_js, 200

@app.route(__base_url + 'daemons/sync', methods=['POST'])
def post_daemons_sync():
    global __daemon, __sync_cnts, __test_case, __fp

    req_data = request.get_data()
    req = json.loads(req_data.decode("utf-8"))

    hasNic1 = False
    hasNic2 = False
    for interface in req['networkInterfaces']:
        if interface['name'] == __agent_nic1:
            hasNic1 = True
        elif interface['name'] == __agent_nic2:
            hasNic2 = True
    if hasNic1 == False or hasNic2 == False:
        __fp.write("Err: no nic1 or nic 2 in sync\n")
        return resp_with_error(404, 404, "no nic1 or nic2 in REG")

    if __sync_cnts == 0:
        add_agent(req, 0, 29, 'host 10.1.1.10') #82pkts

    pro=os.system("ps -ef|grep pktminerg|grep -v grep")
    if pro == 0x100:
        __fp.write("Err: no pktminerg running\n")
        return resp_with_error(404, 404, "no pktminerg running")

    __daemon = req
    __sync_cnts = __sync_cnts + 1

    rep_js = json.dumps(req)
    if __sync_cnts == 1:
        __sync_cnts
        return rep_js, 200
    else:
        return rep_js, 304

if __name__ == '__main__':
    app.run(debug=True, host='127.0.0.1', port=9090)


import os
from PIL import Image
import json
import numpy as np
import pickle
from pprint import pprint

from keras.utils import to_categorical
from keras.preprocessing.text import Tokenizer, text_to_word_sequence
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.models import load_model
from keras.layers import Input, Dense, Flatten, Conv1D, MaxPooling1D, Embedding

###
# Constants
###

MAX_HEADLINE_WORDS = 12

LETTERS = 'qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM 1234567890.-|/'
EMBEDDING_DIM = len(LETTERS)+1
INPUT_LEN = 20 + 24

VALIDATION_SPLIT = 0.2

class Model(object):
    def __init__(self, outdir='model'):
        self._labels_index = {}
        self._outdir = outdir

        if not os.path.isdir(self._outdir):
            # 1st time
            os.mkdir(self._outdir)
            self._init_model()
        else:
            # Load model
            self._model = load_model(os.path.join(self._outdir, 'model.hdf5'))

    def train(self, epochs):
        x, y = self._load_training_data('data/small.jl')
        x_val, x_train, y_val, y_train = self._process_training_data(
            x, y)
        self._model.fit(
            x_train, y_train,
            batch_size=128,
            epochs=epochs,
            validation_data=(x_val, y_val))
        self._save()

    def _save(self):
        self._model.save(os.path.join(self._outdir, 'model.hdf5'))

    def _prepare_input(self, domain, title):
        string = '{:20.20}{:24.24}'.format(domain, title)
        return [LETTERS.index(c) if c in LETTERS else len(LETTERS) for c in string]

    def _load_training_data(self, fp):
        X = []
        Y = []
        with open(fp) as f:
            for line in f:
                domain, title, y = json.loads(line)
                Y.append(y)
                X.append(self._prepare_input(domain, title))
        return X, Y

    def _process_training_data(self, x_raw, y_raw):
        # x and y as if we have the function that y = f(x)
        # where y is result, x is input
        x_all = np.array(x_raw)
        y_all = np.array(y_raw, 'float32') / 255

        # Split into training and validation
        rshuffle = np.arange(x_all.shape[0])  # np.arange ~= a (python) range
        np.random.shuffle(rshuffle)
        x_all = x_all[rshuffle]
        y_all = y_all[rshuffle]

        split_point = int(VALIDATION_SPLIT * x_all.shape[0])
        x_val = x_all[:split_point]
        x_train = x_all[split_point:]
        assert x_val.shape[0] + x_train.shape[0] == x_all.shape[0]
        y_val = y_all[:split_point]
        y_train = y_all[split_point:]
        assert y_val.shape[0] + y_train.shape[0] == y_all.shape[0]

        return x_val, x_train, y_val, y_train

    def _init_model(self):
        self._model = Sequential([
            # Input is implied because we are using the Sequential API
            # Input(shape=(MAX_HEADLINE_WORDS,), dtype='int32'),
            Embedding(len(LETTERS)+1,
                      EMBEDDING_DIM,
                      input_length=INPUT_LEN),
            Flatten(),
            Dense(256),
            Dense(128),
            Dense(256),
            Dense((16*16)*3, activation='softmax'),
        ])
        self._model.summary()
        self._model.compile(
            loss='mean_squared_error',
            optimizer='sgd')

    def predict(self, domain, title):
        x = self._prepare_input(domain, title)
        pred = self._model.predict(np.array([x]))[0]
        return pred

model = Model()
epochs = int(input('How many epochs to train for?  ') or '0')
if epochs:
    model.train(epochs)

while True:
    domain = input('Domain:  ')
    title = input('Title:  ')

    ret = model.predict(domain, title)
    pprint(ret)
    ret = np.array(ret)
    ret *= 255
    ret = ret.reshape((16, 16, 3))
    im = Image.fromarray(ret, mode='RGB')
    im.show()


from data_read import *

import tkinter as tk
from tkinter import Tk, ttk, tix, Frame, Label, filedialog
import configparser
from collections import deque

import matplotlib.pyplot as plt
import matplotlib.pylab as plb

import time

import multiprocessing as mp

import scipy
from scipy import ndimage
import swertka

class Graphica():
    def __init__(self, is_array, is_var):
        # Chart preparing
        plt.ion()
        x = range(5)
        y = range(5)
        # chart lines for arrays and variables
        self.line_a = []
        self.line_v = []

        # max & min for axes
        self.a_minx = 0
        self.a_miny = 0
        self.a_maxx = 1
        self.a_maxy = 1
        self.v_minx = 0
        self.v_miny = 0
        self.v_maxx = 1
        self.v_maxy = 1

    def Draw(self):
        data = q.get()
        arr_data = data[2]
        arr_var = data[3]
        x = range(2)
        y = range(2)

        # need show arrays
        if data[0] == 1:
            # create figure if need
            if plt.get_fignums().count(1) == 0:
                self.fig = plt.figure(num = 1)
                self.line_a.clear()

            # create lines if need
            if len(self.line_a) < len(arr_data):
                for i in range(len(arr_data) - len(self.line_a)):
                    line, = plt.plot(x, y, figure = plt.figure(num = 1))
                    self.line_a.append(line)

            # overgrow protection
            if self.a_maxx > 1000000:
                self.a_maxx = 1;
            if self.a_maxy > 1000000:
                self.a_maxy = 1;
            if self.a_miny < -1000000:
                self.a_miny = 0;

            # match line length and draw lines
            for line in self.line_a:
                if len(line.get_xdata()) != len(arr_data[self.line_a.index(line)]):
                    line.set_xdata(range(len(arr_data[self.line_a.index(line)])))
                line.set_ydata(arr_data[self.line_a.index(line)])

                if self.a_minx > min(line.get_xdata()):
                    self.a_minx = min(line.get_xdata())
                if self.a_maxx < max(line.get_xdata()):
                    self.a_maxx = max(line.get_xdata())

                if self.a_miny > min(line.get_ydata()):
                    self.a_miny = min(line.get_ydata())
                if self.a_maxy < max(line.get_ydata()):
                    self.a_maxy = max(line.get_ydata())
            # set axes
            self.line_a[0].get_axes().axis([self.a_minx, self.a_maxx,
                                                self.a_miny, self.a_maxy])
            self.fig.canvas.draw()
            self.fig.canvas.flush_events()

        # need show variables
        if data[1] == 1:
            if plt.get_fignums().count(2) == 0:
                self.fig_var = plt.figure(num = 2)
                self.line_v.clear()

            if len(self.line_v) < len(arr_var):
                for i in range(len(arr_var) - len(self.line_v)):
                    line, = plt.plot(x, y, figure = plt.figure(num = 2))
                    self.line_v.append(line)

            for line in self.line_v:
                if len(line.get_xdata()) != len(arr_var[self.line_v.index(line)]):
                    line.set_xdata(range(len(arr_var[self.line_v.index(line)])))
                line.set_ydata(arr_var[self.line_v.index(line)])

                if self.v_minx > min(line.get_xdata()):
                    self.v_minx = min(line.get_xdata())
                if self.v_maxx < max(line.get_xdata()):
                    self.v_maxx = max(line.get_xdata())

                if self.v_miny > min(line.get_ydata()):
                    self.v_miny = min(line.get_ydata())
                if self.v_maxy < max(line.get_ydata()):
                    self.v_maxy = max(line.get_ydata())

            self.line_v[0].get_axes().axis([self.v_minx, self.v_maxx,
                                                self.v_miny, self.v_maxy])

            self.fig_var.canvas.draw()
            self.fig_var.canvas.flush_events()

class Application(tk.Frame):
    def __init__(self, master=None):
        self.ini_file = 'setting.ini'
        self.config = configparser.ConfigParser()
        self.config.read(self.ini_file)
        self.stop = False

        tk.Frame.__init__(self, master)
        self.pack()
        self.createWidgets()
        self.setWindowPosition()
        self.master.bind("<Destroy>", self.onDestroy)

        self.gr = Graphica(self.is_show_graph.get(), self.is_show_var_graph.get())

        self.time_queue = deque([])
        self.time_queue_max = 100

    def onDestroy(self, event):
        self.save_geometry()
        plt.close('all')

    def setWindowPosition(self):
        if self.config.has_option('POS', 'geom'):
            g = self.config.get('POS', 'geom').split('+')
            self.master.geometry('+%s+%s' % (g[1], g[2]))

    def createWidgets(self):
        if self.config.has_option('FILES', 'config'):
            self.config_file_name = self.config.get('FILES', 'config')
        else:
            self.config_file_name = ''
        if self.config.has_option('FILES', 'data'):
            self.data_file_name = self.config.get('FILES', 'data')
        else:
            self.data_file_name = ''
        if self.config.has_option('FILES', 'result'):
            self.result_file_name = self.config.get('FILES', 'result')
        else:
            self.result_file_name = ''

        self.ini_var = []
        if self.config.has_section('SELVAR'):
            ini_var_opt = self.config.options('SELVAR')
            self.ini_var = [self.config.get('SELVAR', opt) for opt in self.config.options('SELVAR')]

        # binded variables preparing
        self.is_show_graph = tk.IntVar()
        self.is_show_var_graph = tk.IntVar()
        self.is_not_thinned = tk.IntVar()
        self.is_use_skip = tk.IntVar()
        self.str_skip_b = tk.StringVar()
        self.str_skip_e = tk.StringVar()
        self.str_skip_b.set("0")
        self.str_skip_e.set("0")
        if self.config.has_option('OPTIONS', 'graph'):
            self.is_show_graph.set(self.config.getint('OPTIONS', 'graph'))
        if self.config.has_option('OPTIONS', 'use_skip'):
            self.is_use_skip.set(self.config.getint('OPTIONS', 'use_skip'))
        if self.config.has_option('OPTIONS', 'skip_begin'):
            self.str_skip_b.set(self.config.getint('OPTIONS', 'skip_begin'))
        if self.config.has_option('OPTIONS', 'skip_end'):
            self.str_skip_e.set(self.config.getint('OPTIONS', 'skip_end'))
        if self.config.has_option('OPTIONS', 'var_graph'):
            self.is_show_var_graph.set(self.config.getint('OPTIONS', 'var_graph'))
        if self.config.has_option('OPTIONS', 'not_thinned'):
            self.is_not_thinned.set(self.config.getint('OPTIONS', 'not_thinned'))

        # for variable selecting
        self.varSelVar = tk.StringVar(self)
        self.varSelVar.set("---")

        self.sel_var_list = {}
        self.sel_varframe_list = {}

        # Files group
        g_files = ttk.LabelFrame(self, text = "Files", padding = 5)
        g_files.pack(fill = 'x', expand = 1)
        Label(g_files, text = 'config: ').grid(row = 0)
        Label(g_files, text = 'data: ').grid(row = 1)
        self.l_config = Label(g_files, text = self.config_file_name)
        self.l_data = Label(g_files, text = self.data_file_name)
        self.l_config.grid(row = 0, column = 1)
        self.l_data.grid(row = 1, column = 1)

        self.bt_config = tk.Button(g_files, text = 'select...')
        self.bt_data = tk.Button(g_files, text = 'select...')
        self.bt_config.grid(row = 0, column = 2)
        self.bt_data.grid(row = 1, column = 2)
        self.bt_config['command'] = lambda : self.get_fname('config')
        self.bt_data['command'] = lambda : self.get_fname('data')

        Label(g_files, text = 'result: ').grid(row = 2)
        self.l_result = Label(g_files, text = self.result_file_name)
        self.l_result.grid(row = 2, column = 1)
        self.bt_result = tk.Button(g_files, text = 'select...')
        self.bt_result.grid(row = 2, column = 2)
        self.bt_result['command'] = self.get_result_fname

        # Options group
        g_options = ttk.LabelFrame(self, text = "Options", padding = 5)
        g_options.pack(fill = 'x', expand = 1)

        g_graph_opt = ttk.Frame(g_options)
        g_graph_opt.pack(fill = 'x', expand = 1, ipadx = 5)

        tk.Checkbutton(g_graph_opt, text = 'Show array graph',
                                    variable = self.is_show_graph).pack(side = 'left')
        tk.Checkbutton(g_graph_opt, text = 'Show variable graph',
                                    variable = self.is_show_var_graph).pack(side = 'left')
        tk.Checkbutton(g_graph_opt, text = 'Not thinning data (slowly)',
                                    variable = self.is_not_thinned).pack(side = 'left')

        g_skip = ttk.Frame(g_options)
        g_skip.pack(fill = 'x', expand = 1, ipadx = 5)
        tk.Checkbutton(g_skip, text = 'Use skip',
                            variable = self.is_use_skip).pack(side = 'left')
        Label(g_skip, text = "from begin=").pack(side = 'left')
        e_skip_b = ttk.Entry(g_skip, textvariable = self.str_skip_b)
        e_skip_b.pack(side = 'left')
        Label(g_skip, text = ", to end=").pack(side = 'left')
        e_skip_e = ttk.Entry(g_skip, textvariable = self.str_skip_e)
        e_skip_e.pack(side = 'left')

        # Info group
        g_info = ttk.LabelFrame(self, text = "Info")
        g_info.pack(fill = 'x', expand = 1)

        g_time_info = ttk.Frame(g_info)
        g_time_info.pack(anchor = 'ne', fill = 'x', side = 'left', padx = 10)

        g_sel_var = ttk.Frame(g_info)
        g_sel_var.pack(anchor = 'ne', fill = 'x', side = 'left', padx = 10)

        self.g_vars = ttk.Frame(g_info)
        self.g_vars.pack(anchor = 'ne', fill = 'x', side = 'left', padx = 10)

        Label(g_time_info, text = 'packets: ').grid(row = 0)
        self.l_packets = Label(g_time_info, text = '-')
        self.l_packets.grid(row = 0, column = 1)

        Label(g_time_info, text = 'packet #: ').grid(row = 1)
        self.l_packet = Label(g_time_info, text = '0')
        self.l_packet.grid(row = 1, column = 1)

        Label(g_time_info, text = 'Elapsed time: ').grid(row = 2)
        self.l_time = Label(g_time_info, text = '-')
        self.l_time.grid(row = 2, column = 1)

        Label(g_time_info, text = 'Remained time: ').grid(row = 3)
        self.l_rem_time = Label(g_time_info, text = '-')
        self.l_rem_time.grid(row = 3, column = 1)

        Label(g_time_info, text = 'Expected time: ').grid(row = 4)
        self.l_exp_time = Label(g_time_info, text = '-')
        self.l_exp_time.grid(row = 4, column = 1)

        self.cbSelVar = tk.ttk.Combobox(g_sel_var, textvariable = self.varSelVar)
        self.cbSelVar.grid(row = 0, column = 3)
        self.btAddVarInfo = tk.Button(g_sel_var, text = '+', command = self.add_var_info)
        self.btAddVarInfo.grid(row = 0, column = 4)
        for var_name in self.ini_var:
            self.create_var_info_frame(var_name)

        # Buttons
        self.bt_open = tk.Button(self)
        self.bt_open["text"] = "Open"
        self.bt_open["command"] = self.open_data
        self.bt_open.pack(pady=5)

        self.bt_process = tk.Button(self)
        self.bt_process["text"] = "Process"
        self.bt_process["command"] = self.process_data
        self.bt_process.pack(pady = 5)

        self.bt_stop = tk.Button(self)
        self.bt_stop["text"] = "Stop"
        self.bt_stop["command"] = self.set_stop
        self.bt_stop.pack(pady = 5)

        self.QUIT = tk.Button(self, text="QUIT", fg="red", command=root.destroy)
        self.QUIT.pack(side="bottom", pady = 5)

        self.pb = tk.ttk.Progressbar(self, maximum = 10, )
        self.pb.pack(fill = 'x', expand = 1, pady = 5)
        self.pb["maximum"] = 5      #self.pb.config(maximum=5)

    def get_fname(self, what_file):
        if what_file == 'config':
            ftypes = [('config files', '*.txt'), ('All files', '*')]
        else:
            ftypes = [('data files', '*.bin'), ('All files', '*')]
        dlg = tk.filedialog.Open(self, filetypes = ftypes)
        fname = dlg.show()
        if fname != '':
            if what_file == 'config':
                self.config_file_name = fname
                self.l_config['text'] = self.config_file_name
            else:
                self.data_file_name = fname
                self.l_data['text'] = self.data_file_name

    def get_result_fname(self):
        ftypes = [('text files', '*.txt'), ('All files', '*')]
        dlg = tk.filedialog.SaveAs(self, filetypes = ftypes)
        fname = dlg.show()
        self.result_file_name = fname
        self.l_result['text'] = self.result_file_name

    def open_data(self):
        self.dr = DataRead(self.data_file_name, self.config_file_name)
        self.l_packets["text"] = '%d' % self.dr.block_num

        options = tuple([opt[0] for opt in self.dr.dconf.config if opt[2] == 1])
        self.varSelVar.set(options[0])
        self.cbSelVar['values'] = options

    def set_stop(self):
        self.stop = True

    def add_var_info(self):
        var_name = self.varSelVar.get()
        if list(self.sel_var_list.keys()).count(var_name) == 0:
            self.create_var_info_frame(var_name)

    def create_var_info_frame(self, name):
         _g_ = ttk.Frame(self.g_vars)
         _g_.pack(anchor = 'ne')
         _l_ = tk.Label(_g_, text = '{} = '.format(name))
         _l_.grid(row = 0, column = 0)
         _l_data = tk.Label(_g_, text = '-')
         _l_data.grid(row = 0, column = 1)
         _b_ = tk.Button(_g_, text = 'x')
         _b_.grid(row = 0, column = 2)
         _b_['command'] = lambda : self.del_sel_var(name)
         self.sel_var_list.update({name:_l_data})
         self.sel_varframe_list.update({name:_g_})

    def del_sel_var(self, name):
        _g_ = self.sel_varframe_list[name]
        _g_.destroy()
        del self.sel_var_list[name]
        del self.sel_varframe_list[name]

    def process_data(self):
        self.bt_process["state"] = 'disabled'

        # get skip values
        if self.is_use_skip.get() == 1:
            skip_b = int(self.str_skip_b.get())
            skip_e = int(self.str_skip_e.get())
        # for timer
        t1 = time.perf_counter()
        # skip count
        skip = 0
        if self.is_use_skip.get() == 1:
            skip = skip_b + skip_e
        pack_cnt = skip
        self.pb["maximum"] = self.dr.block_num - skip
        self.pb["value"] = 0

        # open data file
        data_file = open(self.data_file_name, "rb")

        start_h = 11
        start_m = 27
        start_s = 22
        start_t = start_h * 60 * 60 + start_m * 60 + start_s
        # get time of first block
        i = self.dr.get_vars(data_file, 1).__next__()
        self.start_time = i['Sys_t_']
        data_file.seek(0)
        if self.is_use_skip.get() == 1:
            data_file.seek(self.dr.dconf.packet_length * skip_b)
        # open result file
        result_file = open(self.result_file_name, "w", 1)

        # assembling data names to out
        out_vars = ("Time", "Hi", "Sys_t")
        # write the header to the result file
        res_header = '\t'.join(out_vars)
        result_file.writelines('{}{}'.format(res_header, '\n'))

        # data preparing for var chart
        hi_ = []
        sys_dt_ = []
        old_sys_t = 0

        # subprocess initialization
        q.put([0, 0,  0, 0])
        pp = mp.Process(target = self.gr.Draw())

        now_t = time.perf_counter()
        # data processing loop
        for i in self.dr.get_vars(data_file, self.dr.block_num - skip):
            if self.stop:
                self.stop = False
                break
            # ============== Data processing and indication here ===============
            result = swertka.get_swertka(i['CodNonius'], i['Num_Swr'],
                                 i['Num_Div'], i['Diapazon'], i['Srez'])

            # prepare data for file
            curr_t = (i['Sys_t_'] - self.start_time) * 244.15e-6  + start_t
            out_data = {}
            out_data.update({"Time": '%f' % curr_t})
            out_data.update({"Hi": '%f' % (i['Hi'] / 8 - 32)})
            out_data.update({"Sys_t": '%d' % i['Sys_t_']})

            # write data to file
            out_data_str = [out_data[ind] for ind in out_vars]
            result_file.writelines('{}{}'.format('\t'.join(out_data_str), '\n'))

            # data for charts
            hi_.append(i['Hi'] / 8 - 32)

            if (self.is_show_graph.get() == 1) or (self.is_show_var_graph.get() == 1):
                arr_data = []
                arr_data.append(result)
                arr_data.append(i['Swertka'][0:i['Num_Swr']])
                arr_var = [hi_]

                if self.is_not_thinned.get():
                    q.put([self.is_show_graph.get(),
                                self.is_show_var_graph.get(), arr_data, arr_var])
                    self.gr.Draw()
                else:
                    if not pp.is_alive():
                        q.put([self.is_show_graph.get(),
                                self.is_show_var_graph.get(), arr_data, arr_var])
                        pp = mp.Process(target = self.gr.Draw())
                        pp.start()

            # show variables on GUI
            for var in self.sel_var_list.keys():
                if list(i.keys()).count(var) > 0:
                    self.sel_var_list[var]["text"] = str(i[var])
                if list(out_data.keys()).count(var) > 0:
                    self.sel_var_list[var]["text"] = str(out_data[var])
            # ==================================================================
            self.l_packet["text"] = str(pack_cnt)
            pack_cnt = pack_cnt + 1

            # progress bar and timer
            self.pb.step()
            if self.pb["value"] > 0:
                old_t = now_t
                now_t = time.perf_counter()
                now_dt = now_t - old_t
                self.time_queue.append(now_dt)

                if len(self.time_queue) > self.time_queue_max:
                    self.time_queue.popleft()

                #t_aver = (now_t - self.time_queue[0]) / len(self.time_queue)
                if len(self.time_queue) > 0:
                    t_aver = sum(self.time_queue) / len(self.time_queue)
                else:
                    t_aver = 0

                t2 = now_t - t1
                self.l_time["text"] = self.time_to_text(t2)
                #t_expected = (t2 / self.pb["value"]) * self.pb["maximum"]
                t_expected = (t_aver) * self.pb["maximum"]
                self.l_exp_time["text"] = self.time_to_text(t_expected)
                #t_remained = t_expected - t2
                #self.l_rem_time["text"] = self.time_to_text(t_remained)
                t_remained = t_aver * (self.pb["maximum"] - self.pb["value"])
                self.l_rem_time["text"] = self.time_to_text(t_remained)

            self.update()

        data_file.close()
        result_file.close()
        self.bt_process["state"] = 'normal'

    def time_to_text(self, t):
        Secs = t % 60
        Mins = (t/60) % 60
        Hrs = (t/3600) % 60
        return '%.2d:%.2d:%.2d' % (Hrs, Mins, Secs)

    def save_app_config(self):
        if not self.config.has_section('FILES'):
            self.config.add_section('FILES')
        self.config.set('FILES', 'config', self.config_file_name)
        self.config.set('FILES', 'data', self.data_file_name)
        self.config.set('FILES', 'result', self.result_file_name)
        if not self.config.has_section('OPTIONS'):
            self.config.add_section('OPTIONS')
        self.config.set('OPTIONS', 'graph', str(self.is_show_graph.get()))
        self.config.set('OPTIONS', 'use_skip', str(self.is_use_skip.get()))
        self.config.set('OPTIONS', 'skip_begin', self.str_skip_b.get())
        self.config.set('OPTIONS', 'skip_end', self.str_skip_e.get())
        self.config.set('OPTIONS', 'var_graph', str(self.is_show_var_graph.get()))
        self.config.set('OPTIONS', 'not_thinned', str(self.is_not_thinned.get()))

        if self.config.has_section('SELVAR'):
            if len(self.config.options('SELVAR')) > 0:
                self.config.remove_section('SELVAR')
        if not self.config.has_section('SELVAR'):
            self.config.add_section('SELVAR')
        for var in self.sel_var_list.keys():
            self.config.set('SELVAR', var, var)

        with open(self.ini_file, 'w') as configfile:
            self.config.write(configfile)

    def save_geometry(self):
        if not self.config.has_section('POS'):
            self.config.add_section('POS')
        self.config.set('POS', 'geom', self.master.geometry(None))

if __name__ == '__main__':
    root = tk.Tk()
    q = mp.Queue()
    app = Application(master=root)
    app.mainloop()
    app.save_app_config()