import autosar.package
import autosar.parser.package_parser
import autosar.writer
from autosar.base import (parseXMLFile, getXMLNamespace, removeNamespace, parseAutosarVersionAndSchema, prepareFilter, parseVersionString)
import json
import os
import ntpath
import collections
import re
from autosar.parser.datatype_parser import (DataTypeParser, DataTypeSemanticsParser, DataTypeUnitsParser)
from autosar.parser.portinterface_parser import (PortInterfacePackageParser,SoftwareAddressMethodParser,ModeDeclarationGroupPackageParser)
#default parsers
from autosar.parser.constant_parser import ConstantParser
from autosar.parser.behavior_parser import BehaviorParser
from autosar.parser.component_parser import ComponentTypeParser
from autosar.parser.system_parser import SystemParser
from autosar.parser.signal_parser import SignalParser
#default writers
from autosar.writer.datatype_writer import XMLDataTypeWriter, CodeDataTypeWriter
from autosar.writer.constant_writer import XMLConstantWriter, CodeConstantWriter
from autosar.writer.portinterface_writer import XMLPortInterfaceWriter, CodePortInterfaceWriter
from autosar.writer.component_writer import XMLComponentTypeWriter, CodeComponentTypeWriter
from autosar.writer.behavior_writer import XMLBehaviorWriter, CodeBehaviorWriter
from autosar.writer.signal_writer import SignalWriter

_validWSRoles = ['DataType', 'Constant', 'PortInterface', 'ComponentType', 'ModeDclrGroup', 'CompuMethod', 'Unit',
                 'BaseType', 'DataConstraint']

class Workspace(object):
    def __init__(self, version, patch, schema, attributes = None, useDefaultWriters=True):
        self.packages = []
        if isinstance(version, str):
            (major, minor, patch) = parseVersionString(version)
            self._version=float("%s.%s"%(major, minor))
            self.patch=patch
        elif isinstance(version, float):
            self._version=version
            self.patch=int(patch)
        self.schema=schema
        self.packageParser=None
        self.packageWriter=None
        self.xmlroot = None
        self.attributes = attributes
        self.useDefaultWriters = bool(useDefaultWriters)
        self.roles = {'DataType': None,
                      'Constant': None,
                      'PortInterface': None,
                      'ModeDclrGroup': None,
                      'ComponentType': None,
                      'CompuMethod': None,
                      'Unit': None,
                      'BaseType': None,        #AUTOSAR 4 only
                      'DataConstraint': None }  #AUTOSAR 4 only
        self.map = {'packages': {}}
        self.errorHandlingOpt = False
        self.defaultPackages = {'DataType':'DataType',
                                  'CompuMethod':'DataTypeSemantics',
                                  'Unit':'DataTypeUnits',
                                'Constant':'Constant',
                                'PortInterface':'PortInterface',
                                'ModeDclrGroup':'ModeDclrGroup',
                                'ComponentType':'ComponentType',
                                }

    @property
    def version(self):
        return self._version

    @version.setter
    def version(self, version):
        if isinstance(version, str):
            (major, minor, patch) = parseVersionString(version)
            self._version=float("%s.%s"%(major, minor))
            self.patch=patch
        elif isinstance(version, float):
            self._version=version

    def __getitem__(self,key):
        if isinstance(key,str):
            return self.find(key)
        else:
            raise ValueError('expected string')

    def _adjustFileRef(self,fileRef,basedir):
        basename = ntpath.basename(fileRef['path'])
        dirname=ntpath.normpath(ntpath.join(basedir,ntpath.dirname(fileRef['path'])))
        retval=ntpath.join(dirname,basename)
        if os.path.sep == '/': #are we running in cygwin/Linux?
            retval = retval.replace(r'\\','/')
        return retval

    def getRole(self, role):
        return self.roles[role]

    def setRole(self, ref, role):
        if (role is not None) and (role not in _validWSRoles):
            raise ValueError('invalid role name: '+role)
        if ref is None:
            self.roles[role]=None
        else:
            package = self.find(ref)
            if package is None:
                raise ValueError('invalid reference: '+ref)
            if not isinstance(package, autosar.package.Package):
                raise ValueError('invalid type "%s"for reference "%s", expected Package type'%(str(type(package)),ref))
            package.role=role
            self.roles[role]=package.ref

    def openXML(self,filename):
        xmlroot = parseXMLFile(filename)
        namespace = getXMLNamespace(xmlroot)

        assert (namespace is not None)
        (major, minor, patch, schema) = parseAutosarVersionAndSchema(xmlroot)
        removeNamespace(xmlroot,namespace)
        self.version=float('%s.%s'%(major,minor))
        self.major=major
        self.minor=minor
        self.patch=patch
        self.schema=schema
        self.xmlroot = xmlroot
        if self.packageParser is None:
            self.packageParser = autosar.parser.package_parser.PackageParser(self.version)
        self._registerDefaultElementParsers(self.packageParser)

    def loadXML(self, filename, roles=None):
        global _validWSRoles
        self.openXML(filename)
        self.loadPackage('*')
        if roles is not None:
            if not isinstance(roles, collections.Mapping):
                raise ValueError('roles parameter must be a dictionary or Mapping')
            for ref,role in roles.items():
                self.setRole(ref,role)

    def loadPackage(self, packagename, role=None):
        found=False
        result=[]
        if self.xmlroot is None:
            raise ValueError("xmlroot is None, did you call loadXML() or openXML()?")
        if self.version >= 3.0 and self.version < 4.0:
            if self.xmlroot.find('TOP-LEVEL-PACKAGES'):
                for xmlPackage in self.xmlroot.findall('./TOP-LEVEL-PACKAGES/AR-PACKAGE'):
                    if self._loadPackageInternal(result, xmlPackage, packagename, role):
                        found = True

        elif self.version>=4.0:
            if self.xmlroot.find('AR-PACKAGES'):
                for xmlPackage in self.xmlroot.findall('.AR-PACKAGES/AR-PACKAGE'):
                    if self._loadPackageInternal(result, xmlPackage, packagename, role):
                        found = True

        else:
            raise NotImplementedError('Version %s of ARXML not supported'%version)
        if found==False:
            raise KeyError('package not found: '+packagename)
        return result

    def _loadPackageInternal(self, result, xmlPackage, packagename, role):
        name = xmlPackage.find("./SHORT-NAME").text
        found = False
        if packagename=='*' or packagename==name:
            found=True
            package = self.find(name)
            if package is None:
                package = autosar.package.Package(name, parent=self)
                self.packages.append(package)
                result.append(package)
                self.map['packages'][name] = package
            self.packageParser.loadXML(package,xmlPackage)
            if (packagename==name) and (role is not None):
                self.setRole(package.ref, role)
        return found

#BEGIN DEPRECATED
    # def loadJSON(self, filename):
    #    with open(filename) as fp:
    #       basedir = ntpath.dirname(filename)
    #       data = json.load(fp)
    #       if data is not None:
    #          for item in data:
    #             if item['type']=='fileRef':
    #                adjustedPath = self._adjustFileRef(item, basedir)
    #                if adjustedPath.endswith('.arxml'):
    #                   self.loadXML(adjustedPath)
    #                else:
    #                   raise NotImplementedError(adjustedPath)
    #             else:
    #                raise ValueError('Unknown type: %s'%item['type'])
#END DEPRECATED

    def find(self, ref, role=None):
        global _validWSRoles
        if ref is None: return None
        if (role is not None) and ( ref[0] != '/'):
            if role not in _validWSRoles:
                raise ValueError("unknown role name: "+role)
            if self.roles[role] is not None:
                ref=self.roles[role]+'/'+ref #appends the role packet name in front of ref

        if ref[0]=='/': ref=ref[1:] #removes initial '/' if it exists
        ref = ref.partition('/')
        if ref[0] in self.map['packages']:
            pkg = self.map['packages'][ref[0]]
            if len(ref[2])>0:
                return pkg.find(ref[2])
            return pkg
        return None

    def findall(self,ref):
        """
        experimental find-method that has some rudimentary support for globs.
        """
        if ref is None: return None
        if ref[0]=='/': ref=ref[1:] #removes initial '/' if it exists
        ref = ref.partition('/')
        if ref[0]=='*' and len(ref[2])==0:
            result=list(self.packages)
        else:
            result=[]
            for pkg in self.packages:
                if pkg.name == ref[0] or ref[0]=='*':
                    if len(ref[2])>0:
                        result.extend(pkg.findall(ref[2]))
                    else:
                        result.append(pkg)
        return result

    def asdict(self,packages=None):
        retval = {'type': self.__class__.__name__, 'packages':[]}
        for package in self.packages:
            if (isinstance(packages,list) and package.name in packages) or (packages is None):
                retval['packages'].append(package.asdict())
        return retval

    def findRolePackage(self,roleName):
        """
        finds a package with role set to roleName
        """
        if roleName is None: return None
        for pkg in self.packages:
            if pkg.role == roleName:
                return pkg
            elif len(pkg.subPackages)>0:
                for childPkg in pkg.subPackages:
                    if childPkg.role == roleName:
                        return childPkg
        return None

    def createPackage(self,name,role=None):
        if name not in self.map['packages']:
            package = autosar.package.Package(name,self)
            self.packages.append(package)
            self.map['packages'][name] = package
            if role is not None:
                self.setRole(package.ref, role)
            return package
        else:
            return self.map['packages'][name]

    def dir(self,ref=None,_prefix='/'):
        if ref is None:
            return [x.name for x in self.packages]
        else:
            if ref[0]=='/':
                ref=ref[1:]
            ref = ref.partition('/')
            result=self.find(ref[0])
            if result is not None:
                return result.dir(ref[2] if len(ref[2])>0 else None,_prefix+ref[0]+'/')
            else:
                return None

    def findWS(self):
        return self

    def rootWS(self):
        return self

    def saveXML(self, filename, filters=None, packages=None, ignore=None, version=None, patch=None, schema=None):
        if version is None:
            version = self.version
        if patch is None:
            patch = self.patch
        if schema is None:
            schema = self.schema
        if self.packageWriter is None:
            self.packageWriter = autosar.writer.package_writer.PackageWriter(version, patch)
            if self.useDefaultWriters:
                self._registerDefaultElementWriters(self.packageWriter)
        writer=autosar.writer.WorkspaceWriter(version, patch, schema, self.packageWriter)
        with open(filename, 'w', encoding="utf-8") as fp:
            if isinstance(filters,str): filters=[filters]
            if isinstance(packages,str): packages=[packages]
            if isinstance(ignore,str): filters=[ignore]
            if packages is not None:
                if filters is None:
                    filters = []
                for package in packages:
                    if package[-1]=='/':
                        filters.append(package+'*')
                    else:
                        filters.append(package+'/*')
            if filters is not None:
                filters = [prepareFilter(x) for x in filters]
            writer.saveXML(self, fp, filters, ignore)

    def toXML(self, filters=None, packages=None, ignore=None, version=None, patch=None, schema=None):
        if version is None:
            version = self.version
        if patch is None:
            patch = self.patch
        if schema is None:
            schema = self.schema
        if self.packageWriter is None:
            self.packageWriter = autosar.writer.package_writer.PackageWriter(version, patch)
            if self.useDefaultWriters:
                self._registerDefaultElementWriters(self.packageWriter)
        writer=autosar.writer.WorkspaceWriter(version, patch, schema, self.packageWriter)
        if isinstance(filters,str): filters=[filters]
        if isinstance(packages,str): packages=[packages]
        if isinstance(ignore,str): filters=[ignore]
        if packages is not None:
            if filters is None:
                filters = []
            for package in packages:
                if package[-1]=='/':
                    filters.append(package+'*')
                else:
                    filters.append(package+'/*')
        if filters is not None:
            filters = [prepareFilter(x) for x in filters]
        return writer.toXML(self, filters, ignore)

    def append(self,elem):
        if isinstance(elem,autosar.package.Package):
            self.packages.append(elem)
            elem.parent=self
        else:
            raise ValueError(type(elem))

    # def toJSON(self,packages=None,indent=3):
    #    data=ws.asdict(packages)
    #    return json.dumps(data,indent=indent)
    #
    # def saveJSON(self,filename,packages=None,indent=3):
    #    data=self.asdict(packages)
    #    with open(filename,'w') as fp:
    #       json.dump(data,fp,indent=indent)

    def toCode(self, filters=None, packages=None, header=None, version=None, patch=None):
        if version is None:
            version = self.version
        if patch is None:
            patch = self.patch
        writer=autosar.writer.WorkspaceWriter(version, patch, None, self.packageWriter)
        if isinstance(filters,str): filters=[filters]
        if isinstance(packages,str): packages=[packages]
        if packages is not None:
            if filters is None:
                filters = []
            for package in packages:
                if package[-1]=='/':
                    filters.append(package+'*')
                else:
                    filters.append(package+'/*')
        if filters is not None:
            filters = [prepareFilter(x) for x in filters]
        return writer.toCode(self, filters ,str(header), ws.noDefault)

    def saveCode(self, filename, filters=None, packages=None, ignore=None, head=None, tail=None, module=False, template=False, version=None, patch=None):
        """
        saves the workspace as python code so it can be recreated later
        """
        if version is None:
            version = self.version
        if patch is None:
            patch = self.patch
        if self.packageWriter is None:
            self.packageWriter = autosar.writer.package_writer.PackageWriter(version, patch)
            if self.useDefaultWriters:
                self._registerDefaultElementWriters(self.packageWriter)
        writer=autosar.writer.WorkspaceWriter(version, patch, None, self.packageWriter)
        if isinstance(packages,str): packages=[packages]
        if isinstance(filters,str): filters=[filters]
        if isinstance(ignore,str): ignore=[ignore]
        if packages is not None:
            if filters is None:
                filters = []
            for package in packages:
                if package[-1]=='/':
                    filters.append(package+'*')
                else:
                    filters.append(package+'/*')
        if filters is not None:
            filters = [prepareFilter(x) for x in filters]

        with open(filename,'w', encoding="utf-8") as fp:
            writer.saveCode(self, fp, filters, ignore, head, tail, module, template)

    @property
    def ref(self):
        return ''

    def listPackages(self):
        """returns a list of strings containg the package names of the opened XML file"""
        packageList=[]
        if self.xmlroot is None:
            raise ValueError("xmlroot is None, did you call loadXML() or openXML()?")
        if self.version >= 3.0 and self.version < 4.0:
            if self.xmlroot.find('TOP-LEVEL-PACKAGES'):
                for xmlPackage in self.xmlroot.findall('./TOP-LEVEL-PACKAGES/AR-PACKAGE'):
                    packageList.append(xmlPackage.find("./SHORT-NAME").text)
        elif self.version>=4.0:
            if self.xmlroot.find('AR-PACKAGES'):
                for xmlPackage in self.xmlroot.findall('.AR-PACKAGES/AR-PACKAGE'):
                    packageList.append(xmlPackage.find("./SHORT-NAME").text)
        else:
            raise NotImplementedError('Version %s of ARXML not supported'%version)
        return packageList

    def delete(self, ref):
        if ref is None: return
        if ref[0]=='/': ref=ref[1:] #removes initial '/' if it exists
        ref = ref.partition('/')
        for i,pkg in enumerate(self.packages):
            if pkg.name == ref[0]:
                if len(ref[2])>0:
                    return pkg.delete(ref[2])
                else:
                    del self.packages[i]
                    del self.map['packages'][ref[0]]
                    break

    def createAdminData(self, data):
        return autosar.base.createAdminData(data)

    # def fromDict(self, data):
    #    for item in data:
    #       if item['type'] == 'FileRef':
    #          if os.path.isfile(item['path']):
    #             roles = item.get('roles',None)
    #             self.loadXML(item['path'],roles=roles)
    #          else:
    #             raise ValueError('invalid file path "%s"'%item['path'])

    def apply(self, template, **kwargs):
        """
        Applies template to this workspace
        """
        if len(kwargs) == 0:
            rv = template.apply(self)
        else:
            rv = template.apply(self, **kwargs)
        template.usageCount+=1
        return rv

    #---DEPRECATED CODE, TO BE REMOVED ---#
    @classmethod
    def _createDefaultDataTypes(cls, package):
        package.createBooleanDataType('Boolean')
        package.createIntegerDataType('SInt8', -128, 127)
        package.createIntegerDataType('SInt16', -32768, 32767)
        package.createIntegerDataType('SInt32', -2147483648, 2147483647)
        package.createIntegerDataType('UInt8', 0, 255)
        package.createIntegerDataType('UInt16', 0, 65535)
        package.createIntegerDataType('UInt32', 0, 4294967295)
        package.createRealDataType('Float', None, None, minValType='INFINITE', maxValType='INFINITE')
        package.createRealDataType('Double', None, None, minValType='INFINITE', maxValType='INFINITE', hasNaN=True, encoding='DOUBLE')

    def getDataTypePackage(self):
        """
        Returns the current data type package from the workspace. If the workspace doesn't yet have such package a default package will be created and returned.
        """
        package = self.find(self.defaultPackages["DataType"])
        if package is None:
            package=self.createPackage(self.defaultPackages["DataType"], role="DataType")
            package.createSubPackage(self.defaultPackages["CompuMethod"], role="CompuMethod")
            package.createSubPackage(self.defaultPackages["Unit"], role="Unit")
            Workspace._createDefaultDataTypes(package)
        return package

    def getPortInterfacePackage(self):
        """
        Returns the current port interface package from the workspace. If the workspace doesn't yet have such package a default package will be created and returned.
        """
        package = self.find(self.defaultPackages["PortInterface"])
        if package is None:
            package=self.createPackage(self.defaultPackages["PortInterface"], role="PortInterface")
        return package

    def getConstantPackage(self):
        """
        Returns the current constant package from the workspace. If the workspace doesn't yet have such package, a default package will be created and returned.
        """
        package = self.find(self.defaultPackages["Constant"])
        if package is None:
            package=self.createPackage(self.defaultPackages["Constant"], role="Constant")
        return package

    def getModeDclrGroupPackage(self):
        """
        Returns the current mode declaration group package from the workspace. If the workspace doesn't yet have such package, a default package will be created and returned.
        """
        package = self.find(self.defaultPackages["ModeDclrGroup"])
        if package is None:
            package=self.createPackage(self.defaultPackages["ModeDclrGroup"], role="ModeDclrGroup")
        return package

    def getComponentTypePackage(self):
        """
        Returns the current component type package from the workspace. If the workspace doesn't yet have such package, a default package will be created and returned.
        """
        if self.roles["ComponentType"] is not None:
            packageName = self.roles["ComponentType"]
        else:
            packageName = self.defaultPackages["ComponentType"]
        package = self.find(packageName)
        if package is None:
            package=self.createPackage(packageName, role="ComponentType")
        return package

    #--- END DEPCRECATED CODE ---#

    def registerElementParser(self, elementParser):
        """
        Registers a custom element parser object
        """
        if self.packageParser is None:
            self.packageParser = autosar.parser.package_parser.PackageParser(self.version)
            self._registerDefaultElementParsers(self.packageParser)
        self.packageParser.registerElementParser(elementParser)

    def registerElementWriter(self, elementWriter):
        """
        Registers a custom element parser object
        """
        if self.packageWriter is None:
            self.packageWriter = autosar.writer.package_writer.PackageWriter(self.version, self.patch)
            if self.useDefaultWriters:
                self._registerDefaultElementWriters(self.packageWriter)
        self.packageWriter.registerElementWriter(elementWriter)

    def _registerDefaultElementParsers(self, parser):
        parser.registerElementParser(DataTypeParser(self.version))
        parser.registerElementParser(DataTypeSemanticsParser(self.version))
        parser.registerElementParser(DataTypeUnitsParser(self.version))
        parser.registerElementParser(PortInterfacePackageParser(self.version))
        parser.registerElementParser(SoftwareAddressMethodParser(self.version))
        parser.registerElementParser(ModeDeclarationGroupPackageParser(self.version))
        parser.registerElementParser(ConstantParser(self.version))
        parser.registerElementParser(ComponentTypeParser(self.version))
        parser.registerElementParser(BehaviorParser(self.version))
        parser.registerElementParser(SystemParser(self.version))
        parser.registerElementParser(SignalParser(self.version))

    def _registerDefaultElementWriters(self, writer):
        writer.registerElementWriter(XMLDataTypeWriter(self.version, self.patch))
        writer.registerElementWriter(XMLConstantWriter(self.version, self.patch))
        writer.registerElementWriter(XMLPortInterfaceWriter(self.version, self.patch))
        writer.registerElementWriter(XMLComponentTypeWriter(self.version, self.patch))
        writer.registerElementWriter(XMLBehaviorWriter(self.version, self.patch))
        writer.registerElementWriter(CodeDataTypeWriter(self.version, self.patch))
        writer.registerElementWriter(CodeConstantWriter(self.version, self.patch))
        writer.registerElementWriter(CodePortInterfaceWriter(self.version, self.patch))
        writer.registerElementWriter(CodeComponentTypeWriter(self.version, self.patch))
        writer.registerElementWriter(CodeBehaviorWriter(self.version, self.patch))
        writer.registerElementWriter(SignalWriter(self.version, self.patch))


"""
Run from "make_plots" directory with
$ PYTHONPATH=/path/to/uq4pk python3 run_computations.py
"""

from pathlib import Path

from src.blob_detection import compute_blob_detection
from src.fci import compute_fci
from src.svd_mcmc import compute_svd_mcmc
from src.m54 import compute_m54

MODE = "final"
# "test" just for checking that code runs, "base" for faster computations, "final" for final computations.

if MODE == "test":
    OUT = Path("out/out_test")
elif MODE == "base":
    OUT = Path("out/out_base")
elif MODE == "final":
    OUT = Path("out/out_final")
else:
    raise ValueError("Only available modes are 'test', 'base' and 'final'.")

print("---------- COMPUTING: BLOB DETECTION ----------")
compute_blob_detection(mode=MODE, out=OUT)

print("---------- COMPUTING: FCI ----------")
compute_fci(mode=MODE, out=OUT)

print("---------- COMPUTING: M54----------")
compute_m54(mode=MODE, out=OUT)

print("---------- COMPUTING: SVD-MCMC ----------")
compute_svd_mcmc(mode=MODE, out=OUT)


# -*- coding: utf-8 -*-
"""
Created on Sun Nov 12 09:04:20 2017

@author: Nguyen
"""

#Data Frames
import pandas as pd

#Linear Algebra
import numpy as np

#Visualization
import matplotlib.pyplot as plt
import seaborn as sns

#Image Manipulation
from scipy import signal

def show_images(arrays, kernel=None):
    fig = plt.figure(1, figsize=(20,20))
    for i in range(9):
        ax = fig.add_subplot(3,3,i+1)
        arr = signal.convolve2d(np.reshape(np.array(arrays.iloc[i,0]), (75,75)), kernel, mode='valid')
        ax.imshow(arr, cmap = 'inferno')

if __name__ == '__main__':
    train = pd.read_json('../Data/train.json')
    train['inc_angle'] = pd.to_numeric(train['inc_angle'], errors = 'coerce')
    print(train.head())
    #Some practice from https://www.kaggle.com/muonneutrino/exploration-transforming-images-in-python
    icebergs = train[train.is_iceberg==1].sample(n=9, random_state = 456)
    not_icebergs = train[train.is_iceberg==0].sample(n=9, random_state = 123)

    #Image kernels (i.e. edge detectors like Sobel)
    xder = np.array([[-1,0,1], [-2,0,2],[-1,0,1]])
    yder = np.array([[1,2,1],[0,0,0],[-1,-2,-1]])
    smooth = np.array([[1,1,1],[1,10,1],[1,1,1]])
    xder2 = np.array([[-1,2,-1],[-3,6,-3],[-1,2,-1]])
    yder2 = np.array([[-1,-3,-1],[2,6,2],[-1,-3,-1]])
    show_images(icebergs,smooth)


# UFRJ
# Bruno
# UFRJ Nautilus
# Maratona maker

import cv2
import numpy as np
import zbar
import Image

def SegmentacaoAma(imagem):
    hsv = cv2.cvtColor(imagem, cv2.COLOR_BGR2HSV)
    lower = np.array([15, 35, 0])
    upper = np.array([40, 360, 360])
    mask = cv2.inRange(hsv, lower, upper)
    return mask

def SegmentacaoVer(imagem):
    hsv = cv2.cvtColor(imagem, cv2.COLOR_BGR2HSV)
    lower = np.array([150, 70, 140])
    upper = np.array([360, 360, 360])
    mask = cv2.inRange(hsv, lower, upper)
    return mask

def SegmentacaoAzu(imagem):
    lower = np.array([65, 0, 0])
    upper = np.array([215, 175, 113])
    mask = cv2.inRange(imagem, lower, upper)
    return mask

def Blur(imagem):
    imagem = cv2.pyrDown(imagem)
    imagem = cv2.pyrUp(imagem)
    return locals

def reconhecimento(imagem):
#    endereco = "/home/zarjitsky/Imagens/reconhecimentoGeladeira/"
    imagem = cv2.imread(imagem)
    salvo = imagem
    cv2.medianBlur(imagem, 5)
    amarelo = SegmentacaoAma(imagem)
    contoursAM = cv2.findContours(amarelo, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[1]
    for i in range(len(contoursAM)):
        area = cv2.contourArea(contoursAM[i])
        x, y, w, z = cv2.boundingRect(contoursAM[i])
        if area >= 3000 and area <= 8000:
            cv2.rectangle(salvo, (x, y), (x + w, y + z), (0, 0, 0), 2)
            cv2.putText(salvo, "Leite", (x + 3, y - 6), 1, 1, (0, 0, 0))
    vermelho = SegmentacaoVer(imagem)
    contoursVE = cv2.findContours(vermelho, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[1]
    for i in range(len(contoursVE)):
        area = cv2.contourArea(contoursVE[i])
        x, y, w, z = cv2.boundingRect(contoursVE[i])
        if area >= 3000 and area <= 8000:
            cv2.rectangle(salvo, (x, y), (x + w, y + z), (0, 0, 0), 2)
            cv2.putText(salvo, "Coca", (x + 3, y - 6), 1, 1, (0, 0, 0))
    azul = SegmentacaoAzu(imagem)
    contoursAZ = cv2.findContours(azul, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[1]
    for i in range(len(contoursAZ)):
        area = cv2.contourArea(contoursAZ[i])
        x, y, w, z = cv2.boundingRect(contoursAZ[i])
        if area >= 1000 and area <= 8000:
            cv2.rectangle(salvo, (x, y), (x + w, y + z), (0, 0, 0), 2)
            cv2.putText(salvo, "Requeijao", (x + 3, y - 6), 1, 1, (0, 0, 0))
    imagem = vermelho + azul + amarelo
    cv2.namedWindow("frame", cv2.WINDOW_NORMAL)
    cv2.imshow("frame", salvo)

    cv2.waitKey(0)
    cv2.destroyAllWindows()

lista = ["geladeiraRequeijaoLeite.jpg", "geladeiraRequeijao.jpg", "geladeiraRequeijaoLeite.jpg", "geladeiraLeite.jpg", \
         "geladeiraCoca.jpg", "geladeiraCheia.jpg", "geladeiraCocaRequeijao.jpg", \
         "geladeiraCocaLeite.jpg", "geladeiraCheia2.jpg", "geladeiraTeste.jpg"]
if __name__ == "__main__":
    for i in lista:
        reconhecimento(i)


from rest_framework.routers import SimpleRouter
from django.urls import include, path
from rest_framework.authtoken import views
from rest_framework import permissions
from drf_yasg.views import get_schema_view
from drf_yasg import openapi
from django.conf.urls import url

from .views import RequestViewSet, MessageViewSet, ServiceViewSet, SetRatingViewSet

router_v1 = SimpleRouter()
router_v1.register('requests', RequestViewSet)
router_v1.register(r'requests/(?P<request_id>\d+)/messages', MessageViewSet, basename='message')
router_v1.register('services', ServiceViewSet)
router_v1.register('requests/set-rating', SetRatingViewSet)

urlpatterns = [
    path('v1/', include(router_v1.urls)),
    path('v1/auth/', include('djoser.urls')),  # аутентификация jwt. работа с пользователями
    path('v1/auth/', include('djoser.urls.jwt')),  # аутентификация jwt. работа с токенами
    path('v1/auth/', include('rest_framework.urls')),  # ссылки на логин/логаут при работе с api через web.
    path('v1/auth/api-token-auth/', views.obtain_auth_token),  # аутентификация токеном.
]

# Настраиваем автоматическую документацию для API
schema_view = get_schema_view(
   openapi.Info(
       title="HelpDesk API",
       default_version='v1',
       description="Документация для приложения HelpDesk",
   ),
   public=True,
   permission_classes=(permissions.AllowAny,),
)

# Подключаем автоматическую документацию для API
urlpatterns += [
   url(r'^swagger(?P<format>\.json|\.yaml)$',
       schema_view.without_ui(cache_timeout=0), name='schema-json'),
   url(r'^swagger/$', schema_view.with_ui('swagger', cache_timeout=0),
       name='schema-swagger-ui'),
   url(r'^redoc/$', schema_view.with_ui('redoc', cache_timeout=0),
       name='schema-redoc'),
]


from flask import render_template, url_for, flash, redirect, request, abort, Blueprint
from flask_login import login_required, current_user
from flaskbook.modelss import Post
from flaskbook import db
from flaskbook.posts.forms import PostForm

posts = Blueprint('posts', __name__)

@posts.route("/posts/new", methods=['GET', 'POST'])
@login_required
def new_post():
    formation = PostForm()
    if formation.validate_on_submit():
        poster = Post(titlee=formation.title.data, contentt=formation.content.data, author=current_user)
        db.session.add(poster)
        db.session.commit()
        flash('Your Post has been created', 'success')
        return redirect(url_for('main.home'))
    return render_template('create_post.html', title='New Post', legend='New Post', form=formation)

@posts.route("/posts/<int:post_idi>")
def postff(post_idi):
    postff = Post.query.get_or_404(post_idi)
    return render_template('every_post.html', title=postff.titlee, post=postff)

@posts.route("/posts/<int:post_idi>/update", methods=['GET', 'POST'])
@login_required
def update_post(post_idi):
    postff = Post.query.get_or_404(post_idi)
    if postff.author != current_user:
        abort(403)
    formation = PostForm()
    if formation.validate_on_submit():
        postff.titlee = formation.title.data
        postff.contentt = formation.content.data
        db.session.commit()
        flash("Your post has been updated", 'success')
        return redirect(url_for('posts.postff', post_idi=postff.idd)) #obrati paznju da ovde se za post_idi = koristi postff.idd, jer je definisan sa postff = Post.query.get_or_404(post_idi), dok u html na stranici every_post se poziva post.idd direktno iz modelss.py
    elif request.method == 'GET':
        formation.title.data = postff.titlee
        formation.content.data = postff.contentt
    return render_template('create_post.html', title='Update Post', legend='Update Post', form=formation)

@posts.route("/posts/<int:post_idi>/delete", methods=['POST'])
@login_required
def delete_post(post_idi):
    postic = Post.query.get_or_404(post_idi)
    if postic.author != current_user:
        abort(403)
    db.session.delete(postic)
    db.session.commit()
    flash('Your post has been deleted', 'success')
    return redirect(url_for('main.home'))


import torch
import torch.nn as nn
import torch.nn.functional as F
from networks.cell2d import Cell, ConvBR

# Parameter-optimizable classifier used by neural archatecture search
# Classifier built from networks.cell2d Cell
# Cascade cells downsizing image size and increasing features finally connecting to fully connected layers
# NAS Cell parameters: out_channels - output chanels, steps - number of convolution interations
# NAS model parameters: nn.Linear output chanels
# Oblation study parameters: cell.residual, ConvBR.batch_norm, ConvBR.padding_mode
class AutoClassifier(nn.Module):
    def __init__(self, num_cells, cell=Cell):
        super(AutoClassifier, self).__init__()
        print('{}.{}'.format(__class__.__name__, __name__))

        self.num_cells = num_cells
        self.cell_dfn = cell
        self.cells = nn.ModuleList()
        self.parameters = []

        self.parameters.append({'name':'channels','tensor':(1e-3 * torch.randn(self.num_cells)).clone().detach().requires_grad_(True)})
        self.parameters.append({'name':'steps','tensor':(1e-3 * torch.randn(self.num_cells)).clone().detach().requires_grad_(True)})

        self.initialize()

    def forward(self, x):
        print('{}.{}'.format(__class__.__name__, __name__))
        return x

    def initialize(self):
        print('{}.{}'.format(__class__.__name__, __name__))

        # Registers neural architecture searchable parameters to PyTorch
        for i in range(self.parameters):
            self.register_parameter(self.parameters[i]['name'], torch.nn.Parameter(self.parameters[i]['tensor']))


from rest_framework.serializers import ModelSerializer, DateTimeField, ListField, \
    PrimaryKeyRelatedField, HyperlinkedRelatedField, BooleanField
from .models import Listing
from .models import Application

class BaseListingSerializer(ModelSerializer):
    available = BooleanField(read_only=True, default=True)

    class Meta:
        model = Listing
        fields = '__all__'

class SimpleListingSerializer(BaseListingSerializer):
    class Meta:
        model = Listing
        fields = ['name', 'gender', 'breed', 'picture1', 'available']

class ApplicationSerializer(ModelSerializer):
    class Meta:
        model = Application
        fields = '__all__'

class ApplicationUpdateSerializer(ModelSerializer):
    class Meta:
        model = Application
        fields = '__all__'
        read_only_fields = ['first_name', 'last_name', 'email', 'notes', 'listing', 'created_at', 'last_updated', 'phone', 'address']


from typing import Callable

import tensorflow as tf
from tensorflow.python.ops.image_ops_impl import ResizeMethod

def load_grayscale_images(dataset_path: str, batch_size: int, images_height: int,
                          images_width: int, epochs: int = 1) -> tf.data.Dataset:

    # list all files in folders subdirs
    paths_dataset = tf.data.Dataset.list_files(f"{dataset_path}/**/*", shuffle=True)

    # map each path to a tuple (path, class_name)
    paths_and_label_dataset = paths_dataset.map(_extract_label, num_parallel_calls=tf.data.AUTOTUNE)

    image_loader = _make_image_loader(images_height, images_width)
    images_dataset = paths_and_label_dataset.map(image_loader, num_parallel_calls=tf.data.AUTOTUNE)

    return images_dataset.repeat(epochs).prefetch(tf.data.AUTOTUNE).batch(batch_size).cache()

def _extract_label(path: tf.Tensor) -> (tf.Tensor, tf.Tensor):
    class_name = tf.strings.split(path, "/")[-2]
    class_index = tf.strings.to_number(class_name, out_type=tf.dtypes.int32)
    return path, class_index

def _make_image_loader(images_height: int, images_width: int) -> Callable:
    image_dimensions = (images_height, images_width)

    def loader(path: tf.Tensor, label: tf.Tensor) -> (tf.Tensor, tf.Tensor):
        # Load raw data from path
        raw = tf.io.read_file(path)

        # Decode image from raw data as a tensor of tf.float32
        image = tf.io.decode_image(raw, dtype=tf.float32, expand_animations=False, channels=1)

        # Resize the tensor image to the expected dimensions
        image = tf.image.resize(image, size=image_dimensions, antialias=True, preserve_aspect_ratio=False,
                                method=ResizeMethod.BILINEAR)
        image = _rescale(image)

        return image, label

    return loader

def _rescale(image: tf.Tensor) -> tf.Tensor:
    return tf.divide(image, 255.0)


# -*- coding: utf-8 -*-

from setuptools import setup, find_packages

with open('README.md') as f:
    readme = f.read()

with open('LICENSE') as f:
    license = f.read()

setup(
    name='tocrify',
    version='0.0.1',
    description='Pimp OCR with structural information from METS',
    long_description=readme,
    author='Kay-Michael Würzner, Matthias Boenig',
    author_email='wuerzner@bbaw.de, boenig@bbaw.de',
    license=license,
    packages=find_packages(exclude=('tests', 'docs')),
    package_data={'tocrify' : ['data/mets2hocr.yml']},
    install_requires=[
    ],
    entry_points={
          'console_scripts': [
              'tocrify=tocrify.scripts.tocrify:cli',
          ]
    },
)


from models.applicationSeason import ApplicationSeason
from models.user import User
from models.application import Application
from datetime import datetime, timedelta
from utils.enums import Rank, ApplicationStatus

def createBasicSeason(db_session=None):
    date1 = datetime.now().replace(microsecond=0)
    date2 = datetime.now().replace(microsecond=0) + timedelta(days=365)
    date3 = datetime.now().replace(microsecond=0) + timedelta(days=7)
    date4 = datetime.now().replace(microsecond=0) + timedelta(days=21)
    season = ApplicationSeason(date1, date2, date3, date4)
    if(db_session):
        db_session.add(season)
        db_session.commit()
    return season

def createApplication(db_session=None):
    season = createBasicSeason(db_session)
    user = createUser(db_session)
    application = Application(
        needs="needs",
        comments="comments",
        user=user,
        partnerUsername="Jar Jar Binks",
        preferredRoom="d1",
        seatRollover=True,
        status=ApplicationStatus.SUBMITTED,
        rank=Rank.WRITING_MASTER,
        applicationSeason=season,
    )
    if(db_session):
        db_session.add(application)
        db_session.commit()
    return application

def createUser(db_session=None):
    user = User(username="username",
                sub="55de7d71-4a25-4103-8e43-35df8c2d472a",
                email="email",
                fullname="fullname")
    if(db_session):
        db_session.add(user)
        db_session.commit()
    return user


import argparse
import sys
import os
import pickle

import numpy as np
import tensorflow as tf

import pygame
from pygame.locals import *

import enviroment
from enviroment import Enviroment

from agent import Agent
from replayMemory import ReplayMemory

import matplotlib.pyplot as plt

from datetime import datetime
import time

FLAGS = None
frame_by_frame_mode = False
next_frame = True

def checkEventQueue(enviroment):
    global frame_by_frame_mode
    global next_frame

    # Check the event que
    for event in pygame.event.get():
        if event.type == QUIT or (event.type == KEYDOWN and event.key == K_ESCAPE):
            # The user closed the window or pressed escape
            enviroment.running = False
        if event.type == KEYDOWN:
            if event.key == K_g:
                enviroment.display_grid = not enviroment.display_grid
            if event.key == K_PLUS:
                m = min(enviroment.SCREEN_HEIGHT,enviroment.SCREEN_WIDTH) / enviroment.PPM
                enviroment.x_spacing = min(m, enviroment.x_spacing + 0.5)
                enviroment.y_spacing = min(m, enviroment.y_spacing + 0.5)
            if event.key == K_MINUS:
                enviroment.x_spacing = max(0.5, enviroment.x_spacing - 0.5)
                enviroment.y_spacing = max(0.5, enviroment.y_spacing - 0.5)
            if event.key == K_p:
                enviroment.pause = not enviroment.pause
            if event.key == K_h:
                enviroment.display_hud = not enviroment.display_hud
            if event.key == K_r:
                enviroment.resetGame()
            if event.key == K_o:
                enviroment.display_highscores = not enviroment.display_highscores
            if event.key == K_b:
                enviroment.boat_vision_on = not enviroment.boat_vision_on
            if event.key == K_m:
                enviroment.manual_control = not enviroment.manual_control
            if event.key == K_f:
                frame_by_frame_mode = not frame_by_frame_mode
            if not next_frame and event.key == K_n:
                next_frame = True

# Also want to move this to the learner class
def handleKeyboardInput(keys):

    if keys[K_w]:
        action = enviroment.INCREASE_POWER
    elif keys[K_s]:
        action = enviroment.DECREASE_POWER
    elif keys[K_a]:
        action = enviroment.STEER_LEFT
    elif keys[K_d]:
        action = enviroment.STEER_RIGHT
    else:
        action = enviroment.DO_NOTHING

    return action

def display_values(values, labels=None):

    #fig, ax = plt.subplots()

    #values = values / np.linalg.norm(values)

    index = np.arange(len(values))

    #bar_width = ?

    rect = plt.bar(index, values, color='b', label='values')

    if labels:
        plt.xticks(index, labels)

    plt.pause(0.00001)

def main(argv):
    global frame_by_frame_mode
    global next_frame

    # Create the enviroment
    env = Enviroment(FLAGS)

    # Append the data based on the enviroment used
    FLAGS.input_data_size = env.state_size
    FLAGS.num_actions = 5

    # Create the agent used to navigate in the world
    agent = Agent(FLAGS)

    # The replay memory
    memory = ReplayMemory(FLAGS)

    ########################
    # Setup some variables #
    ########################
    accumulated_rewards = []
    actions = np.zeros(FLAGS.num_actions)
    update_network_every = 100
    total_steps = 0
    goals_reached = 0
    ########################

    # Used for live plot updating in matplotlib
    plt.ion()

    learning_start_time = time.clock()

    for game in range(FLAGS.num_games):

        #####################################
        # Initialize all the run variables  #
        #####################################

        # The total reward for this game, we want to maximize this in principle
        total_reward = 0
        time_step = 0
        # Reset the game before the start of every new instance
        env.resetGame()
        current_state = env.currentState()
        actions_distr = np.zeros(5)

        while not env.gameHasEnded():

            checkEventQueue(env)

            if frame_by_frame_mode and not next_frame:
                continue
            elif frame_by_frame_mode and next_frame:
                print('Next frame')
                next_frame = False

            total_steps += 1
            time_step += 1

            if env.manual_control:
                keys = pygame.key.get_pressed()
                action = handleKeyboardInput(keys)
            else:
                # Only start predicting actions after a certain amount of time, before that select random actions and only fill the memory
                if total_steps < FLAGS.replay_start_size:
                    action = np.zeros(FLAGS.num_actions)
                    action[np.random.randint(FLAGS.num_actions)] = 1
                else:
                    action = agent.select_epsilon_greedy(current_state)

            # Advance the enviroment by one step based on the selected action
            env.nextState(action)

            if FLAGS.test:
                continue

            # If test flag in turned on this is where the loop ends
            ##############################################################################
            ##############################################################################

            # Update the action distribution statistics
            actions_distr = actions_distr + action

            if FLAGS.display_action_distribution and total_steps % 100 == 0:
                display_values(actions_distr)

            # Update the accumulated state vector (n last states concatenated into a single vector, we feed this into the neural network)
            next_state = env.currentState()

            # Get the reward for this state and acumulate the total reward for this run
            reward = env.getReward()
            total_reward += reward

            # Save the transition into memory
            memory.save_transition(current_state, action, reward, next_state)

            # If we are only collecting experience atm we don't want to perform learning just yet
            if total_steps <= FLAGS.replay_start_size:
                current_state = next_state
                continue
            elif total_steps == FLAGS.replay_start_size:
                print("Ending random collection of data")

            #################################
            """
            if total_steps % 1000 == 0:
                activations = agent.sess.run([layer for layer in agent.hidden_layers], feed_dict={agent.current_state:current_state})
                plt.clf()
                for i in range(1, len(activations)+1):
                    plt.subplot(1, len(activations), i)
                    plt.hist(activations[i-1].flatten(), bins=10)
                    plt.pause(0.000001)
            """
            ###################################

            # Sample of minibatch of experiences
            minibatch = memory.sample(FLAGS.minibatch_size)

            # Unpack the minibatch
            current_states, actions, rewards, next_states, batch_size = minibatch

            # Learn from the sampled transitions
            agent.learn_from_transition(current_states, actions, rewards, next_states, batch_size)

            # Decay the epsilon (for epsilon-greedy strategy selection)
            if total_steps % FLAGS.decay_epsilon_every == 0:
                agent.decay_epsilon()

            # Update the frozen network every n steps
            if total_steps % update_network_every == 0:
                agent.update_frozen_network()

            # Record the data every n time steps if recording activated
            if FLAGS.record_learning_data and (total_steps % FLAGS.log_data_every == 0):
                agent.record_running_summaries(current_state, action, reward, next_state)

            current_state = next_state

        if env.level.boat.goal_reached:
            goals_reached += 1

        agent.record_run_summaries(game,  goals_reached,  total_reward, time_step)

        print('Game number ', game, ' has ended in ', time_step, ' steps.')
        print('Boat has come to goal: ', env.level.boat.goal_reached)
        print('Total reward was: ', total_reward)
        print('Average game time: ', (time.clock() - learning_start_time) / (game+1))

        if FLAGS.save_agent:
            agent.save_network(game)

if __name__ == "__main__" :
    parser = argparse.ArgumentParser()
    parser.add_argument('--player_name', type=str, default='Jure', help='Name of the human (or non-human) player')
    parser.add_argument('--level', type=str, default='1', help='The name of the level to load')
    parser.add_argument('--manual_control', dest='manual_control', action='store_true', help='Use this option if you want to steer the boat yourself')
    parser.add_argument('--logdir', type=str, default='/home/ros/Student_project/train_data', help='Directory for saving data')
    parser.add_argument('--num_games', type=int, default=1, help="Number of games for the simulation to run")
    parser.add_argument('--debug_mode', dest='debug_mode', action='store_true', help='Toggle debug mode on/off')
    parser.add_argument('--no-debug_mode', dest='debug_mode', action='store_false', help='Toggle debug mode on/off')
    parser.add_argument('--log_data_every', type=int, default=1000, help='Log a data point every n steps')
    parser.add_argument('--num_hidden_layers', type=int, default=1, help='The number of hidden layers for the neural network')
    parser.add_argument('--hidden_layer_size', type=int, default=10, help='The size of the network hidden layers (all layers same size atm)')
    parser.add_argument('--epsilon', type=float, default=1, help='The probability of selecting a random action')
    parser.add_argument('--end_epsilon', type=float, default=0.05, help='The minimum epsilon decayed to, with the selected epsilon decay')
    parser.add_argument('--epsilon_decay', type=float, default=0.05, help='How much to decrease the epsilon by with linear decay')
    parser.add_argument('--epsilon_decay_strategy', type=str, default='linear', help='The was to decay the epsilon during learning, options: [linear]')
    parser.add_argument('--decay_epsilon_every', type=int, default=50000, help='On how many steps to decay the epsilon')
    parser.add_argument('--trainer', type=str, default='Adam', help='Select the optimization algorithm, options: [Gradient_descent, RMS_prop, Adam, Momentum]')
    parser.add_argument('--learning_rate', type=float, default=0.1, help='The learnig rate')
    parser.add_argument('--display_world', dest='display_world', action='store_true', help='Use this feature if you want the simulation to be displayed')
    parser.add_argument('--no-display_world', dest='display_world', action='store_false', help='Use this feature if you want to run the simulation without displaying it')
    parser.add_argument('--minibatch_size', type=int, default=1, help='The size of the minibatch to sample from memory to learn from')
    parser.add_argument('--replay_start_size', type=int, default=100, help='A uniform policy is run for this number of steps before learning starts and the resulting experience is used to populate the replay memory')
    parser.add_argument('--max_time_ticks', type=int, default=5000, help='The maximum number of ticks to run the simulation before reseting. Prevents getting stuck too much')
    parser.add_argument('--memory_max_size', type=int, default=100000, help='The number of last frames seen to be kept in memory')
    parser.add_argument('--display_action_distribution', dest='display_action_distribution', action='store_true', help='Use this if you want a live histogram of selected actions')
    parser.add_argument('--ticks_per_second', type=int, default=10, help='Sets the number of time the agent gets to make a decision every second')
    parser.add_argument('--record_learning_data', dest='record_learning_data', action='store_true', help='Toggle to save/not-save the data created while learning')
    parser.add_argument('--save_agent', dest='save_agent', action='store_true', help='Toggle to save the agent (network parameters)')
    parser.add_argument('--no-record_learning_data', dest='record_learning_data', action='store_false', help='Toggle to save/not-save the data created while learning')
    parser.add_argument('--no-save_agent', dest='save_agent', action='store_false', help='Toggle to save the agent (network parameters)')
    parser.add_argument('--load_saved_agent', type=str, help='Select the folder to load the saved agent from')
    parser.add_argument('--test', action='store_true', help='Use this flag if you want to test an agent without learning')
    parser.set_defaults(display_world=True)
    parser.set_defaults(debug_mode=False)
    parser.set_defaults(manual_control=False)
    parser.set_defaults(display_action_distribution=False)
    parser.set_defaults(record_learning_data=True)
    parser.set_defaults(save_agent=True)
    FLAGS, unparsed = parser.parse_known_args()

    # Create the name of the run directory based on the current datetime
    FLAGS.run_dir = FLAGS.logdir + '/run_' + str(datetime.now())

    if FLAGS.load_saved_agent:
        with open(FLAGS.logdir + '/' + FLAGS.load_saved_agent + '/.params.pickle') as f:
            LOADED_FLAGS = pickle.load(f)

            FLAGS.num_hidden_layers = LOADED_FLAGS.num_hidden_layers
            FLAGS.hidden_layer_size = LOADED_FLAGS.hidden_layer_size
            FLAGS.minibatch_size = LOADED_FLAGS.minibatch_size
            FLAGS.action_repeat = LOADED_FLAGS.action_repeat

    # Save the run parameters to a file
    # First create the folder is not created already
    if not os.path.exists(FLAGS.run_dir):
        os.makedirs(FLAGS.run_dir)
    if FLAGS.record_learning_data:
        with open(FLAGS.run_dir + '/.params.pickle', 'w') as f:
            pickle.dump(FLAGS, f)

    pygame.init()

    tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)

    pygame.quit()

    print("Done!")


import json
from decimal import Decimal
from ask_sdk_model.ui import Card, StandardCard, Image
from ask_sdk_model.interfaces.alexa.presentation.apl.render_document_directive import RenderDocumentDirective

from intents.base_intent import BaseIntent

class StatusIntent(BaseIntent):

    SPEECH_TEMPLATE = "Pi Hole is currently {}. In the past 24 hours it had {} queries from {} clients, and blocked {} of them ({}%)."

    CARD_TEXT_TEMPLATE = "Status: {}\n" \
                         "Total Queries: {} ({} clients)\n" \
                         "Blocked Queries: {}\n" \
                         "Blocked Percent: {}%\n"

    def get_speech_message(self) -> str:
        status_message = StatusIntent.SPEECH_TEMPLATE.format(self.get_client().status, self.get_client().total_queries,
                                                             self.get_client().unique_clients, self.get_client().blocked,
                                                             Decimal(self.get_client().ads_percentage).normalize())

        return status_message

    def get_card(self) -> Card:

        card = StandardCard(
            title="Pi-hole Alexa",
            image=Image(
                small_image_url=self.get_images_url() + "/logo.png",
                large_image_url=self.get_images_url() + "/logo_large.png"
            ),
            text=StatusIntent.CARD_TEXT_TEMPLATE.format(self.get_client().status, self.get_client().total_queries,
                                                        self.get_client().unique_clients, self.get_client().blocked,
                                                        Decimal(self.get_client().ads_percentage).normalize())
        )

        return card

    def get_apl_directive(self) -> RenderDocumentDirective:

        with open('apl_templates/status.json') as status_json_file:
            status_json_doc = status_json_file.read() \
                .replace("${logo_url}", self.get_images_url() + "/logo_" + self.get_client().status + ".png") \
                .replace("${total_queries}", self.get_client().total_queries) \
                .replace("${total_clients}", self.get_client().unique_clients) \
                .replace("${total_queries_image}", self.get_images_url() + "/globe.jpg") \
                .replace("${total_queries_image_large}", self.get_images_url() + "/globe.jpg") \
                .replace("${blocked_queries}", self.get_client().blocked) \
                .replace("${blocked_queries_image}", self.get_images_url() + "/hand.jpg") \
                .replace("${blocked_queries_image_large}", self.get_images_url() + "/hand.jpg") \
                .replace("${blocked_percent}", str(Decimal(self.get_client().ads_percentage).normalize())) \
                .replace("${blocked_percent_image}", self.get_images_url() + "/pie.jpg") \
                .replace("${blocked_percent_image_large}", self.get_images_url() + "/pie.jpg") \
                .replace("${blocked_domains}", self.get_client().domain_count) \
                .replace("${blocked_domains_image}", self.get_images_url() + "/file.jpg") \
                .replace("${blocked_domains_image_large}", self.get_images_url() + "/file.jpg")

            status_document = json.loads(status_json_doc)

            return RenderDocumentDirective(
                document=status_document['document'],
                datasources=status_document['datasources'],
                token='status_document'
            )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 31 23:07:47 2018

@author: sitaram
"""

import pandas as pd
import numpy as np
from collections import Counter
from itertools import combinations
import sys

def combinations_function(x):
#     print(x[0])
    if len(x)==1:
        return [(x[0],x[0])]
    else:
        return list(combinations(x,2))

def generate_table(data_set,number):
    data_set.reset_index(inplace=True,drop=True)
    reference_name=data_set['source_id'][0]
    ll=[]
    cc=[]
    combo=[]
    ui_combo=[]
    source_values=[]

    journal_list=data_set.groupby(['source_id'])['reference_issn'].apply(list).values
    reference_list=data_set.groupby(['source_id'])['cited_source_uid'].apply(list).values

    #Calculating the journal pairs for each publication
    pairs=list(map(combinations_function, journal_list))
    ref_pairs=list(map(combinations_function,reference_list))
    #Calculating the pairs for journal and cited references
    #print('looping through and creating pairs')
    for index,row in data_set.iterrows():
        if row['source_id']==reference_name:
            ll.append(row['reference_issn'])
            # cc.append(row['cited_source_uid'])
        else:
            ll.sort()
            # cc.sort()
            if len(ll)==1:
                no_of_pairs=1
                # combo.extend([(ll[0],ll[0])])
                # ui_combo.extend([(cc[0],cc[0])])
            else:
                no_of_pairs=((len(ll)*(len(ll)-1))/2)
                # combo.extend(list(combinations(ll,2)))
                # ui_combo.extend(list(combinations(cc,2)))
            source_values.extend([reference_name]*int(no_of_pairs))
            ll.clear()
            # cc.clear()
            reference_name=row['source_id']
            ll.append(row['reference_issn'])
            # cc.append(row['cited_source_uid'])
    no_of_pairs=((len(ll)*(len(ll)-1))/2)
    source_values.extend([reference_name]*int(no_of_pairs))
    #print('Calculating pairs done')

    del(data_set)

    df_c=pd.DataFrame([z for x in pairs for z in x],columns=['A','B'])
    df_c['source_id']=pd.Series(source_values)
    df_=pd.DataFrame([z for x in ref_pairs for z in x],columns=['C','D'])
    df_c['wos_id_pairs']=df_['C'].astype(str)+','+df_['D'].astype(str)
    del(df_)
    df_c['journal_pairs']=df_c['A']+','+df_c['B']

    # del(df_)
    #print('len of file',len(df_c))

    #print('Join with z scores file to create final lookup table')
    data_set=pd.read_csv(z_scores)

    #Joining with z scores to develop final table
    full_list=pd.merge(df_c.iloc[:,2:],data_set,on='journal_pairs',how='inner')
    #print('Writing to file')
    if number==0:
        full_list.to_csv(destination_file,index=False)
    else:
        full_list.to_csv(destination_file,mode='a',index=False,header=False)

filename=sys.argv[1]
z_scores=sys.argv[2]
destination_file=sys.argv[3]

column_names=['source_id','source_year','source_issn','source_document_id_type','cited_source_uid',
              'reference_year','reference_issn','reference_document_id_type']

fields=['source_id','cited_source_uid','reference_issn']

data_set=pd.read_csv(filename,usecols=fields)
#print('Generating Lookup Table')

#Sorting the input file by source_id and reference_issn
data_set.sort_values(by=['source_id','reference_issn','cited_source_uid'],inplace=True)
data_set.reset_index(inplace=True,drop=True)
data_set['reference_issn']=data_set['reference_issn'].astype(str)

end_point=3000000
start_point=0
total_len=len(data_set)

if total_len < 3000000:
    #print('Entered if block')
    generate_table(data_set,0)
else:
    #print('Entered else block')
    number=0
    source_id=data_set['source_id'][end_point]
    while end_point <= total_len:
        end_point+=1
        if source_id != data_set['source_id'][end_point]:
            #print('end_point',data_set['source_id'][end_point])
            #print('end_point-1',data_set['source_id'][end_point-1])
            generate_table(data_set.iloc[start_point:end_point,:],number)
            number+=1
            start_point=end_point
            end_point+=3000000
            if end_point < total_len:
                source_id=data_set['source_id'][end_point]
            if end_point > total_len:
                end_point=total_len
                generate_table(data_set.iloc[start_point:,:],number)
                end_point=total_len+1

print('Done generating z scores file')


import pygame as pg
import sys
import random

def check_bound(obj_rct,scr_rct): # 画面外の判別
    """
    obj_rect:こうかとんrct または 爆弾rct
    scr_rct:スクリーンrct
    領域内:+1/領域外:-1
    """
    yoko,tate = +1,+1
    if obj_rct.left < scr_rct.left or obj_rct.right > scr_rct.right:
        yoko = -1
    if obj_rct.top < scr_rct.top or obj_rct.bottom > scr_rct.bottom:
        tate = -1

    return yoko, tate

def check_bomb(rct,lst): # 爆弾増幅判定（ランダム）
    if random.randint(0,1000) == 0: # 1/1000の確率で爆弾が増える
        now_x, now_y = rct.center
        create_bomb(now_x, now_y,lst)

def create_bomb(x,y,l): # 新しい爆弾の設定
    """
    x:爆弾の位置のx軸
    y:爆弾の位置のy軸
    l:爆弾のリスト
    """
    new_bomb_sfc = pg.Surface((20,20)) # Surface
    new_bomb_sfc.set_colorkey((0, 0, 0))
    pg.draw.circle(new_bomb_sfc, (255, 0, 0), (10, 10), 10)
    new_bomb_rct = new_bomb_sfc.get_rect() # Rect
    new_bomb_rct.centerx = x
    new_bomb_rct.centery = y
    # 爆弾の動きの設定
    move_x = random.randint(-5,5)
    move_y = random.randint(-5,5)
    l.append([new_bomb_sfc,new_bomb_rct,move_x,move_y]) # 爆弾のリストに追加

def main():
    pg.display.set_caption("逃げろ！こうかとん") # 練習１
    scrn_sfc = pg.display.set_mode((1600,900))
    scrn_rct = scrn_sfc.get_rect()

    bg_sfc = pg.image.load("fig/pg_bg.jpg")
    bg_rct = bg_sfc.get_rect()

    clock = pg.time.Clock()

    # 練習３
    x, y = 900, 400
    tori_sfc = pg.image.load("fig/6.png")
    tori_sfc = pg.transform.rotozoom(tori_sfc, 0, 2.0)
    tori_rct = tori_sfc.get_rect()
    tori_rct.center = x,y

    # 練習５
    bomb_sfc = pg.Surface((20,20))
    bomb_sfc.set_colorkey((0, 0, 0))
    pg.draw.circle(bomb_sfc, (255, 0, 0), (10, 10), 10)
    bomb_rct = bomb_sfc.get_rect()
    bomb_rct.centerx = random.randint(0, scrn_rct.width)
    bomb_rct.centery = random.randint(0, scrn_rct.height)

    # 練習６
    vx, vy = +5, +5

    bombs = [[bomb_sfc,bomb_rct,vx,vy]] # 爆弾のリスト作成

    while True:
        scrn_sfc.blit(bg_sfc, bg_rct) # 練習２

        for event in pg.event.get():
            if event.type == pg.QUIT: return
        # # 練習４
        # key_states = pg.key.get_pressed()
        # if key_states[pg.K_UP]: tori_rct.centery -= 1
        # if key_states[pg.K_DOWN]: tori_rct.centery += 1
        # if key_states[pg.K_LEFT]: tori_rct.centerx -= 1
        # if key_states[pg.K_RIGHT]: tori_rct.centerx += 1

        # if yoko == -1:
        #     if key_states[pg.K_LEFT]:
        #         tori_rct.centerx += 1
        #     if key_states[pg.K_RIGHT]:
        #         tori_rct.centerx -= 1

        # if tate == -1:
        #     if key_states[pg.K_UP]:
        #         tori_rct.centery += 1
        #     if key_states[pg.K_DOWN]:
        #         tori_rct.centery -= 1

        # マウスの場所にこうかとんを移動させる
        yoko, tate = check_bound(tori_rct, scrn_rct)
        # もし画面外なら中に入るように動く
        if yoko == -1:
            if x < scrn_rct.width/2:
                x += 1
            if x > scrn_rct.width/2:
                x -= 1
        elif tate == -1:
            if y < scrn_rct.height/2:
                y += 1
            if y > scrn_rct.height/2:
                y -= 1
        # それ以外はマウスの場所に移動
        else:
            if event.type == pg.MOUSEMOTION:
                x, y = event.pos # マウスの座標

        tori_rct.center = (x,y) # こうかとんの座標更新

        # こうかとんの描画
        scrn_sfc.blit(tori_sfc, tori_rct)

        for bomb in bombs: # すべての爆弾の動き
            # 爆弾を増幅させるかの処理
            check_bomb(bomb[1],bombs)
            yoko, tate = check_bound(bomb[1], scrn_rct) # 画面外判定

            # 画面外なら動きの符号を反転させる（反射する）
            if yoko == -1:
                bomb[2] *= yoko
            if tate == -1:
                bomb[3] *= tate

            # 爆弾を動かす
            bomb[1].move_ip(bomb[2], bomb[3])
            scrn_sfc.blit(bomb[0], bomb[1])

            #爆弾に当たったら終了
            if bomb[1].colliderect(tori_rct):
                return

        pg.display.update()
        clock.tick(1000)

if __name__ == "__main__":
    pg.init() # 初期化
    main() # ゲームの本体
    pg.quit() # 初期化の解除
    sys.exit()


# -*- coding: utf-8 -*-
"""
Created on Sun Jun 20 20:22:02 2021

@author: sushantpatil

Link to the thread : https://www.kaggle.com/startupsci/titanic-data-science-solutions
"""

import numpy as np
import pandas as pd
import random as rnd

import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron
from sklearn.linear_model import SGDClassifier
from sklearn.tree import DecisionTreeClassifier

# loading the data
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')

# looking at the data shape and its features
print(train.head(5))
print(train.shape)
print(test.shape)
print(train.columns)

# take a look at each column and split features as categorical and numerical
# Some of the categorical variables are : survived, embarked, sex  Ordinal : pclass

# some of the numerical variable are continuous: fare,age  discrete : parch, sibsp,

# Now, take a look at the values we are missing and data types of our features
# print('_'*30+ '\ntrain null vals')
# print(train.isnull().sum().sort_values(ascending=False))
# print('_'*30 + '\ntest null vals')
# print(test.isnull().sum().sort_values(ascending=False))
print(train.info())
#print(test.info())

# From the actual stats, around 40% of the people on board survived
# Now, we check the % survived on our given dataset
print(train.Survived.value_counts())
# Represents the actual survival stat data approx.
# Now we look at some categorical data and try to check if some categories are more inclined
print(train.Pclass.value_counts())
print(train.Sex.value_counts())
print(train.SibSp.value_counts())
print(train.Parch.value_counts())

# takeaways :
# there more lower (3rd) class people on the cruise
# more males
# most of the people had no sibling or spouse
# most of the had no parent or children

# Lets look at some distinct feature correlations
# Upper class have higher survival rates
print(train[['Pclass','Survived']].groupby(['Pclass']).mean())

# Females have higher survival rates
print(train[['Sex','Survived']].groupby(['Sex']).mean())

print(train[['SibSp','Survived']].groupby(['SibSp']).mean())
print(train[['Parch','Survived']].groupby(['Parch']).mean())

# Analysing by visualizing data

# # Correlating numerical features
# p1 = sns.FacetGrid( train, col='Survived' )
# p1.map( plt.hist, 'Age', bins = 20, edgecolor='black', color= '#DC143C', alpha= 0.7 )

# # Correlating numerical and ordinal features(Pclass)

# # We can combine multiple features for identifying correlations using a single plot.
# # This can be done with numerical and categorical features which have numeric values.
# p2 = sns.FacetGrid( train, col='Survived', row='Pclass', height=3, aspect= 1 )
# p2.map(plt.hist, 'Age', bins = 20, edgecolor='black', color= '#ADFF2F', alpha = 0.7 )

# # Correlating categorical features
# p3 = sns.FacetGrid( train , row='Embarked', height=3, aspect=2 )
# p3.map(sns.pointplot, 'Pclass', 'Survived', 'Sex', palette='pastel')
# p3.add_legend()

# # Something for me to ponder upon, the above line code just helped compare
# # survival rates of male and female of each Pclass according to their embarkation
# # THATS 4 CATEGORICAL FEATS ONTO A 2D PLOT

# # Correlating categorical and numerical features
# p4 = sns.FacetGrid( train, row='Embarked', col='Survived', height=3, aspect=2 )
# p4.map( sns.barplot, 'Sex', 'Fare', alpha=0.6, ci= None)

# Correcting by dropping features

# we will be dropping Ticket and Cabin columns from the dataset

train.drop(['Ticket','Cabin'], axis=1, inplace=True)
test.drop(['Ticket', 'Cabin'], axis=1, inplace=True)

print(train.shape)
print(test.shape)

# Feature Extraction/ Feature Engineering

# there's a brief passage extracting useful information from Name and Passenger Id
# for simplicity purpose, I am omitting the particular part of study and directly drop both those columns

train.drop(['Name', 'PassengerId'], axis=1, inplace=True)
test.drop(['Name'], axis=1, inplace=True)

print(train.shape)
print(test.shape)

#print(train.head(5))
from sklearn.preprocessing import LabelEncoder

lbl = LabelEncoder()
lbl.fit(train['Sex'])
# 1 is male
# 0 is female
train['Sex'] = lbl.fit_transform(train['Sex'])
test['Sex'] = lbl.fit_transform(test['Sex'])
lbl2= LabelEncoder()
lbl2.fit(train['Pclass'])
# 0 is upper 1 is middle 2 is lower class
train['Pclass'] = lbl2.fit_transform(train['Pclass'])
test['Pclass'] = lbl2.fit_transform(test['Pclass'])

#print(train.head(5))
#print(test.head(5))

# Now, lets deal with missing values for each feature one by one

#print(train.isnull().sum())
#print(test.isnull().sum())
# Age
print(train['Age'].isnull().sum())
# We can fill NaN values with Median values
ageMed = np.nanmedian(train['Age'])
ageMed2 = np.nanmedian(test['Age'])
print(np.nanmedian(train['Age']))
print(np.nanmedian(test['Age']))
train['Age'].fillna(ageMed, inplace=True)
test['Age'].fillna(ageMed2, inplace=True)
print(train['Age'].isnull().sum())
print(test['Age'].isnull().sum())

train.loc[train['Age']<=16,'Age']=0
train.loc[(train['Age']>16) & (train['Age']<=32),'Age']=1
train.loc[(train['Age']>32) & (train['Age']<=64),'Age']=2
train.loc[(train['Age']>64),'Age']=3
test.loc[test['Age']<=16,'Age']=0
test.loc[(test['Age']>16) & (test['Age']<=32),'Age']=1
test.loc[(test['Age']>32) & (test['Age']<=64),'Age']=2
test.loc[(test['Age']>64),'Age']=3

#print(train.head(5))

# Creating new features combining existing features

train['FamilySize'] = train['Parch']+train['SibSp']+1
train['isAlone']=0
train.loc[train['FamilySize']==1, 'isAlone']=1

test['FamilySize'] = test['Parch']+test['SibSp']+1
test['isAlone']=0
test.loc[test['FamilySize']==1, 'isAlone']=1
#print(train.head(5))

# Now we can drop SibSp, Parch, FamilySize
train.drop(['SibSp','Parch','FamilySize'], axis=1, inplace=True)
test.drop(['SibSp','Parch','FamilySize'], axis=1, inplace=True)
#print(train.head(5))

train['AgeClass'] = train['Pclass']*train['Age']
test['AgeClass'] = test['Pclass']*test['Age']
print(train.head(5))
print(test.head(5))
print('-'*60)
#print(train.isnull().sum())
#print(test.isnull().sum())
embarkedMed = train['Embarked'].mode()
embarkedMed2 = test['Embarked'].mode()
#print(embarkedMed[0])
train['Embarked'].fillna(value=embarkedMed[0], inplace=True)
test['Embarked'].fillna(value=embarkedMed2[0], inplace=True)
#print(train.head(5))
fareMed = test['Fare'].median()
test['Fare'].fillna(fareMed, inplace=True)
train['Fare'] = train['Fare'].round(2)
test['Fare'] = test['Fare'].round(2)

#print(pd.qcut(train['Fare'], 4))
train.loc[train['Fare']<=7.91,'Fare']=0
train.loc[(train['Fare']>7.91) & (train['Fare']<=14.45),'Fare']=1
train.loc[(train['Fare']>14.45) & (train['Fare']<=31),'Fare']=2
train.loc[(train['Fare']>31),'Fare']=3
test.loc[test['Fare']<=7.91,'Fare']=0
test.loc[(test['Fare']>7.91) & (test['Fare']<=14.45),'Fare']=1
test.loc[(test['Fare']>14.45) & (test['Fare']<=31),'Fare']=2
test.loc[(test['Fare']>31),'Fare']=3
print(train.isnull().sum())
print(test.isnull().sum())

lbl3 = LabelEncoder()
lbl3.fit(train['Embarked'])
# C is 0 Q is 1 S is 2
train['Embarked'] = lbl3.fit_transform(train['Embarked'])
test['Embarked'] = lbl3.fit_transform(test['Embarked'])
print(train.head(5))
print(test.head(5))
#print(train.isnull().sum())
train['Fare'] = train['Fare'].astype(int)
train['Age'] = train['Age'].astype(int)
train['AgeClass'] = train['AgeClass'].astype(int)
test['Fare'] = test['Fare'].astype(int)
test['Age'] = test['Age'].astype(int)
test['AgeClass'] = test['AgeClass'].astype(int)
print('-'*60)
print(train.head(5))
print(test.head(5))
# Model Selection
y = train['Survived']
X = train.drop(['Survived'], axis=1)
X_test = test.drop(['PassengerId'], axis=1)

print(y.shape)
print(X.shape)
print(X_test.shape)
# print(y.head(5))
# print(X.head(5))
# print(X_test.head(5))

# LogisticRegression

logR = LogisticRegression()
logR.fit(X, y)
preds_log = logR.predict(X_test)
acc_log = logR.score(X, y)
#print(acc_log)
coeff_df = pd.DataFrame(train.columns.delete(0))
coeff_df.columns = ['Feature']
coeff_df["Correlation"] = pd.Series(logR.coef_[0])

coeff_df.sort_values(by='Correlation', ascending=False)
#print(coeff_df.head(10))

# Support Vector Machines

svc = SVC()
svc.fit(X,y)
preds_svc = svc.predict(X_test)
acc_svc = svc.score(X,y)
#print(acc_svc)

# K Nearest Neighbors

knn = KNeighborsClassifier( n_neighbors=6 )
knn.fit(X,y)
preds_knn = knn.predict(X_test)
acc_knn = knn.score(X,y)
#print(acc_knn)

# Gaussian Naive Bayes Classifier

gaussiannb = GaussianNB()
gaussiannb.fit(X,y)
preds_gnb = gaussiannb.predict(X_test)
acc_gnb = gaussiannb.score(X,y)
#print(acc_gnb)

# Perceptron

perc = Perceptron()
perc.fit(X,y)
preds_perc = perc.predict(X_test)
acc_perc = perc.score(X,y)
#print(acc_perc)

# Linear SVC

linsvc = LinearSVC()
linsvc.fit(X,y)
preds_linsvc = linsvc.predict(X_test)
acc_linsvc = linsvc.score(X,y)
#print(acc_linsvc)

# Stochastic Gradient Descent

sgd = SGDClassifier()
sgd.fit(X,y)
preds_sgd = sgd.predict(X_test)
acc_sgd = sgd.score(X,y)
#print(acc_sgd)

# Decision Tree Classifier

dtree = DecisionTreeClassifier()
dtree.fit(X,y)
preds_dtree = dtree.predict(X_test)
acc_dtree = dtree.score(X,y)
#print(acc_dtree)

# Random Forest Classifier

rforest = RandomForestClassifier()
rforest.fit(X,y)
preds_rforest = rforest.predict(X_test)
acc_rforest = rforest.score(X,y)
#print(acc_rforest)

models = pd.DataFrame( {
    'Model' : [ 'Logistic Regression', 'SVM', 'KNN', 'GaussianNB', 'Perceptron', 'LinearSVC', 'SGD', 'DecisionTree', 'RandomForest' ],
    'Scores' : [ acc_log, acc_svc, acc_knn, acc_gnb, acc_perc, acc_linsvc, acc_sgd, acc_dtree, acc_rforest ]
    } )
print(models.head(10))
print('Run Complete!')


import json
import os

# 加载JSON文件到Python对象
with open('/home/rongch05/openfaas/training/results_by_appID/TFT.json', 'r') as f:
    data = json.load(f)

# 切割数据并写入多个小文件
print(type(data))
chunk_size = 1000 # 每个小文件的数据量
output_dir = 'split_results_by_appID/TFT'  # 小文件的输出目录
if not os.path.exists(output_dir):
    os.makedirs(output_dir)

chunks = []
tmp = list(data.items())
print(len(data), data.keys())

# 'agg_metrics', 'item_metrics'
for key in data.keys():
    print(type(data[key]))
    for i in range(0, len(data[key]), chunk_size):
        chunk = data[key][i:i+chunk_size]
        print(len(chunk))
        chunks.append(chunk)

    for i, chunk in enumerate(chunks):
        filename = f'{output_dir}/{key}_chunk_{i}.json'
        print(filename)
        with open(filename, 'w') as f:
            f.write(json.dumps(chunk, indent=4, sort_keys=True))


'''
This script take the 'data.json' source file from `dirData` and parse it.
It will create geojson file as output.
'''
import geopandas as gpd
import json
import os

########################
# Parameters
########################
dirData = '2023-01-29'

########################
# Constantes
########################
dirAsset = 'assets'
dirRegion = 'regions'
baseUrl = 'https://delivrez.fr/public/posts/'

gpxNs = 'osmand'
gpxNsUrl = 'https://osmand.net'

edibleFile = 'edibles'
giveboxFile = 'giveboxes'
bookcasesFile = 'bookcases'

def newGeojsonStruct():
    return {
        'type': 'FeatureCollection',
        'features': []
    }

def convertJsonToGeoJson():
    print('Read source file data.json in folder: ' + dirData)
    with open(dirData + '/data.json') as file:
        data = json.load(file)

    attrs = []
    types = []

    bookcases = newGeojsonStruct()
    giveboxes = newGeojsonStruct()
    edibles = newGeojsonStruct()

    for d in data:
        attrs.extend(list(d.keys()))
        type = d['type']
        types.append(type)
        feature = {
            'type': 'Feature',
            'properties': {
                'name': d['title'],
                'codetype': d['type'],
                'type': 'Delivrez',
                'desc': d['observation'],
                'color': '#00842b',
                'background': 'square'
            },
            'geometry': {
                'type': 'Point',
                'coordinates': [
                    float(d['lon']),
                    float(d['lat'])
                ]
            }
        }
        if d['picture'] is not None:
            feature['properties']['website'] = baseUrl + d['picture']
        if 'INE' in type:
            feature['properties']['type'] += ' edible'
            edibles['features'].append(feature)
        if 'DSB' in type:
            feature['properties']['type'] += ' givebox'
            giveboxes['features'].append(feature)
        if 'BSB' in type:
            feature['properties']['type'] += ' bookcase'
            feature['properties']['amenity_subtype'] = 'public_bookcase'
            feature['properties']['icon'] = 'public_bookcase'
            bookcases['features'].append(feature)

    attrs = set(attrs)
    types = set(types)
    print('Dataset attributes: ' + ', '.join(attrs))
    print('Dataset types: ' + ', '.join(types))
    print('Objects found :')
    print('- ' + str(len(edibles['features'])) + ' INE: Incredible Edible')
    print('- ' + str(len(giveboxes['features'])) + ' DSB: give box (not only books)')
    print('- ' + str(len(bookcases['features'])) + ' BSB: public bookcase')

    dumpGeojsonToFile(edibles, os.path.join(dirData, edibleFile + '.geojson'))
    dumpGeojsonToFile(giveboxes, os.path.join(dirData, giveboxFile + '.geojson'))
    dumpGeojsonToFile(bookcases, os.path.join(dirData, bookcasesFile + '.geojson'))

    print('Geojson files created in folder: ' + dirData)

def dumpGeojsonToFile(data, path):
    with open(path, 'w') as out:
        json.dump(data, out, indent=2, ensure_ascii=True)

def dumpToGpx(data, path):
    data.to_file(path, driver='GPX', GPX_USE_EXTENSIONS=True, GPX_EXTENSIONS_NS=gpxNs, GPX_EXTENSIONS_NS_URL=gpxNsUrl)

def convertGeojsonToGpx(file):
    print('Convert ' + file + ' from geojson to GPX')
    path = os.path.join(dirData, file + '.geojson')
    bookcases = gpd.read_file(path)
    path = os.path.join(dirData, file + '.gpx')
    dumpToGpx(bookcases, path)

def filterBookcasesByArea():
    print('Start of filtering bookcases by region')
    pathRegion = os.path.join(dirAsset, dirRegion)
    pathBookcase = os.path.join(dirData, bookcasesFile + '.geojson')
    areaFilename = os.listdir(pathRegion)

    print('Filter bookcases by region:')
    for areaName in areaFilename:
        areaFile = os.path.join(dirAsset, dirRegion, areaName)
        if os.path.isfile(areaFile):
            region = gpd.read_file(areaFile)
            regionName = os.path.basename(areaName)
            regionName = os.path.splitext(regionName)[0]
            bookcases_filtered = gpd.read_file(pathBookcase, mask=region)
            bookcases_filtered['type'] += ' ' + regionName
            print(' - ' + str(bookcases_filtered.size) + ' bookcases in ' + regionName)
            bookcases_filtered.to_file(os.path.join(dirData, dirRegion, areaName), driver='GeoJSON')
            filepath = os.path.join(dirData, dirRegion, regionName + '.gpx')
            dumpToGpx(bookcases_filtered, filepath)

    print('End of filtering bookcases by region')

print('Start parsing data.json from folder: ' + dirData)

pathRegion = os.path.join(dirData, dirRegion)
if not os.path.exists(pathRegion):
    print('Create folder: ' + pathRegion)
    os.makedirs(pathRegion)

convertJsonToGeoJson()
convertGeojsonToGpx(bookcasesFile)
convertGeojsonToGpx(edibleFile)
convertGeojsonToGpx(giveboxFile)
filterBookcasesByArea()

print('End of parsing')


#G. Проверка даты
#ограничение по времени на тест 2 секунды
#ограничение по памяти на тест 256 мегабайт
#вводстандартный ввод
#стандартный вывод
#Задана дата в формате день месяц год в виде трёх целых чисел. Гарантируется, что:

#день — это целое число от 1 до 31,
#месяц — это целое число от 1 до 12,
#год — это целое число от 1950 до 2300.
#Проверьте, что заданные три числа соответствуют корректной дате (в современном григорианском календаре).
#Напоминаем, что в соответствии с современным календарём год считает високосным, если для этого года верно хотя бы одно из утверждений:
#делится на 4, но при этом не делится на 100;
#делится на 400.
#Например, годы 2012 и 2000 являются високосными, но годы 1999, 2022 и 2100 — нет.
#Входные данные
#В первой строке записано целое число t (1≤t≤1000) — количество наборов входных данных в тесте.

#Наборы входных данных в тесте являются независимыми. Друг на друга они никак не влияют.

#Каждые набор входных данных задаётся одной строкой, в которой записаны три целых числа d, m, y (1≤d≤31,1≤m≤12,1950≤y≤2300) — день, месяц и год даты для проверки.

#Выходные данные
#Для каждого набора входных данных выведите YES, если соответствующая дата является корректной (т.е. существует такая дата в современном календаре). Выведите NO в противном случае.
#Вы можете выводить ответ в любом регистре (например, вывод yEs, yes, Yes и YES всё ещё будет считаться корректным).

import datetime

def check(date):
    try:
        time = datetime.datetime.strptime(date, '%d %m %Y')
        return True
    except ValueError:
        return False

t = int(input())
for i in range(t):
    date =input()
    if check(date)==True:
        print('YES')
    else:
        print('NO')


"""Clean up on unique constraints

Revision ID: 881dc818423c
Revises: 47ef746d0398
Create Date: 2023-06-27 10:15:37.832703

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '881dc818423c'
down_revision = '47ef746d0398'
branch_labels = None
depends_on = None

def upgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    op.alter_column('authors', 'name',
               existing_type=sa.VARCHAR(),
               nullable=False)
    op.alter_column('books', 'name',
               existing_type=sa.VARCHAR(),
               nullable=False)
    op.alter_column('books', 'number_of_pages',
               existing_type=sa.INTEGER(),
               nullable=False)
    op.drop_index('ix_books_number_of_pages', table_name='books')
    op.drop_index('ix_books_name', table_name='books')
    op.create_index(op.f('ix_books_name'), 'books', ['name'], unique=False)
    op.create_unique_constraint('author_book_name_uc', 'books', ['author_id', 'name'])
    # ### end Alembic commands ###

def downgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint('author_book_name_uc', 'books', type_='unique')
    op.drop_index(op.f('ix_books_name'), table_name='books')
    op.create_index('ix_books_name', 'books', ['name'], unique=False)
    op.create_index('ix_books_number_of_pages', 'books', ['number_of_pages'], unique=False)
    op.alter_column('books', 'number_of_pages',
               existing_type=sa.INTEGER(),
               nullable=True)
    op.alter_column('books', 'name',
               existing_type=sa.VARCHAR(),
               nullable=True)
    op.alter_column('authors', 'name',
               existing_type=sa.VARCHAR(),
               nullable=True)
    # ### end Alembic commands ###


"""Truncates bgl3_full fixtures for faster runtime.

The runtime of this script is slow (use web services) because it is only run
once.
"""

import os

from Bio import SeqIO
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord

from schemarecomb import ParentSequences
from schemarecomb import PDBStructure

trunc_dir = os.path.dirname(os.path.realpath(__file__))
bgl3_full_dir = os.path.join(trunc_dir, '..', 'bgl3_full')

seq_len = 50  # final size of alignment between pdb and p0_aligned
num_parents = 6

pdb_fn = os.path.join(bgl3_full_dir, '1GNX.pdb')
pdb = PDBStructure.from_pdb_file(pdb_fn)
parents_fn = os.path.join(bgl3_full_dir, 'bgl3_sequences.fasta')
parents = ParentSequences.from_fasta(parents_fn, pdb_structure=pdb)

# Reduce number of parents.
parents.records = parents.records[:num_parents]

# Get parent alignment and renumber pdb.
print(f'aligning {num_parents} parents')
parents.align()

# Find the first index that doesn't have any gaps in the alignment.
for pdb_index, aa in enumerate(parents.pdb_structure.amino_acids):
    aln_index = aa.index
    if '-' not in parents.alignment[aln_index]:
        break
first_aa_index = aln_index
last_aa_index = parents.pdb_structure.amino_acids[pdb_index+seq_len-1].index

print(first_aa_index, last_aa_index)

new_records = []
aligned_sequences = [''.join(seq) for seq in zip(*parents.alignment)]
for sr, aln_seq in zip(parents.records, aligned_sequences):
    new_seq = aln_seq[first_aa_index:last_aa_index+1]
    new_sr = SeqRecord(Seq(new_seq), id=sr.id, name=sr.name,
                       description=sr.description)
    new_records.append(new_sr)

reduced_parents = ParentSequences(new_records, pdb, prealigned=True)

# Write parents
new_parents_fn = os.path.join(trunc_dir, 'bgl3_trunc_aln.fasta')
SeqIO.write(reduced_parents.records, new_parents_fn, 'fasta')

# Write pdb
with open('1GNX_trunc.pdb', 'w') as f:
    pdb = reduced_parents.pdb_structure
    aa_strs = ['\n'.join(aa.to_lines()) for aa in pdb.amino_acids]
    out_str = '\n'.join(aa_strs)
    f.write(out_str)


# pylint: disable=no-self-use
from unittest import mock

import pytest

from pynes import cpu
from pynes.instructions import cmp

@pytest.fixture
def test_cpu():
    test_cpu = cpu.Cpu()
    test_cpu.memory = bytearray(b'\x00\x00\x05\x00')

    yield test_cpu

@pytest.mark.parametrize(('test_value', 'expected'), [(1, True), (10, False)])
def test_negative(test_cpu, test_value, expected):
    memory_address = 2
    cmp._compare(test_cpu, test_value, memory_address)

    assert test_cpu.status.negative == expected

def test_zero(test_cpu):
    cmp._compare(test_cpu, 5, 2)

    assert not test_cpu.status.carry
    assert test_cpu.status.zero
    assert not test_cpu.status.negative

def test_carry(test_cpu):
    cmp._compare(test_cpu, 10, 2)

    assert test_cpu.status.carry
    assert not test_cpu.status.zero
    assert not test_cpu.status.negative

class TestCompareInstructions:
    def test_cmp(self, test_cpu):
        test_cpu.accumulator = mock.sentinel.accumulator

        with mock.patch.object(cmp, '_compare') as compare:
            cmp.cmp(test_cpu, 2)
        compare.assert_called_with(test_cpu, mock.sentinel.accumulator, 2)

    def test_cpx(self, test_cpu):
        test_cpu.register_x = mock.sentinel.register_x

        with mock.patch.object(cmp, '_compare') as compare:
            cmp.cpx(test_cpu, 2)
        compare.assert_called_with(test_cpu, mock.sentinel.register_x, 2)

    def test_cpy(self, test_cpu):
        test_cpu.register_y = mock.sentinel.register_y

        with mock.patch.object(cmp, '_compare') as compare:
            cmp.cpy(test_cpu, 2)
        compare.assert_called_with(test_cpu, mock.sentinel.register_y, 2)


import os
import argparse
import copy

from otk.data_utils import MatDataset
from otk.models_deepsea import SeqAttention
from torch.utils.data import DataLoader, Subset
import torch
from torch import nn
from torch.optim.lr_scheduler import StepLR, MultiStepLR, ReduceLROnPlateau
import torch.optim as optim
import numpy as np
from sklearn.metrics import (
    roc_auc_score, log_loss, accuracy_score,
    precision_recall_curve, average_precision_score
)

from timeit import default_timer as timer

name = 'DeepSEA'
datadir = '../data/{}'.format(name)

def load_args():
    parser = argparse.ArgumentParser(
        description="OT attention for Encode DeepSEA experiments",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        '--seed', type=int, default=1, metavar='S',
        help='random seed')
    parser.add_argument(
        '--batch-size', type=int, default=128, metavar='M',
        help='input batch size for training')
    parser.add_argument(
        "--hidden-size", default=1536, type=int,
        help="hidden size for conv1d and attn layers")
    parser.add_argument(
        "--filter-size", default=16, type=int,
        help="filter size for embedding conv1d")
    parser.add_argument(
        "--weight-decay", type=float, default=1e-06,
        help="regularization parameter for sup CKN")
    parser.add_argument(
        "--eps", default=1.0, type=float, help="eps for OT kernel")
    parser.add_argument(
        "--attn-layers", default=1, type=int, help="number of OT attn layers")
    parser.add_argument(
        '--heads', type=int, default=1, help='number of heads for attention layer')
    parser.add_argument(
        '--out-size', type=int, default=64, help='number of supports for attention layer')
    parser.add_argument(
        '--max-iter', type=int, default=30, help='max iteration for ot kernel')
    parser.add_argument(
        '--hidden-layer', action='store_true', help='use one hidden-layer in classfier')
    parser.add_argument(
        '--position-encoding', default=None, choices=['gaussian', 'hard'],
        help='position encoding type')
    parser.add_argument(
        '--position-sigma', default=0.1, type=float, help='sigma for position encoding')
    parser.add_argument("--lr", type=float, default=0.001, help='initial learning rate')
    parser.add_argument("--report-step", type=int, default=1000,
        help="report stat step during training")
    parser.add_argument(
        "--outdir", default='', type=str, help="output path")
    parser.add_argument(
        "--subset", type=int, default=None, help='train on a subset')
    args = parser.parse_args()
    args.use_cuda = torch.cuda.is_available()

    args.save_logs = False
    if args.outdir != "":
        args.save_logs = True

    return args

def compute_scores(y_true, y_pred, report_gt_feature_n_positives=0):
    n_features = y_true.shape[1]
    metric_fn = {'auROC': roc_auc_score, 'auPRC': average_precision_score}
    scores = {}
    for key in metric_fn:
        scores[key] = np.empty(n_features)
        scores[key].fill(np.nan)
    for i in range(n_features):
        feat_true = y_true[:, i]
        if np.count_nonzero(feat_true) > report_gt_feature_n_positives:
            for key in metric_fn:
                scores[key][i] = metric_fn[key](feat_true, y_pred[:, i])
    avg_scores = {}
    for key in metric_fn:
        avg_scores[key] = np.nanmean(scores[key])
    return avg_scores, scores

def eval_epoch(model, data_loader, criterion, return_pred=False, use_cuda=False):
    model.eval()
    running_loss = 0.0
    y_pred = torch.Tensor(len(data_loader.dataset), model.nclass)
    y_true = torch.Tensor(len(data_loader.dataset), model.nclass)
    index = 0
    tic = timer()
    for data, label in data_loader:
        size = data.shape[0]
        y_true[index: index + size] = label
        if use_cuda:
            data = data.cuda()
            label = label.cuda()

        with torch.no_grad():
            output = model(data)
            loss = criterion(output, label) / size
            y_pred[index: index + size] = output.data.sigmoid().cpu()

        running_loss += loss.item() * size
        index += size
    toc = timer()

    epoch_loss = running_loss / len(data_loader.dataset)
    scores, all_scores = compute_scores(y_true.numpy(), y_pred.numpy())
    print('Val Loss: {:.4f} auROC: {:.4f} auPRC: {:.4f} Time: {:.2f}s'.format(
           epoch_loss, scores['auROC'], scores['auPRC'], toc - tic))
    if return_pred:
        return epoch_loss, scores, all_scores, y_true, y_pred
    return epoch_loss, scores, all_scores

def load_checkpoint(args):
    if args.save_logs:
        checkpoint_file = args.outdir + "/checkpoint_last.pt"
        if os.path.isfile(checkpoint_file):
            return torch.load(checkpoint_file)
    return None

def main():
    args = load_args()
    print(args)
    torch.manual_seed(args.seed)
    if args.use_cuda:
        torch.cuda.manual_seed(args.seed)
    np.random.seed(args.seed)

    train_dset = MatDataset(datadir, 'train')
    val_dset = MatDataset(datadir, 'valid')
    print("training size {}, validation size {}".format(len(train_dset), len(val_dset)))

    loader_args = {}
    if args.use_cuda:
        loader_args = {'num_workers': 1, 'pin_memory': True}

    val_loader = DataLoader(
        val_dset, batch_size=args.batch_size, shuffle=False, **loader_args)

    model = SeqAttention(
        919, args.hidden_size, args.filter_size, args.attn_layers,
        args.eps, args.heads, args.out_size, max_iter=args.max_iter,
        hidden_layer=args.hidden_layer, position_encoding=args.position_encoding,
        position_sigma=args.position_sigma)
    print(model)
    if args.use_cuda:
        model.cuda()

    criterion = nn.BCEWithLogitsLoss(reduction='sum')

    if args.save_logs:
        print("Loading model...")
        checkpoint = torch.load(args.outdir + "/checkpoint_best.pt")
        model.load_state_dict(checkpoint['weight'])

    print("Validating")
    eval_epoch(model, val_loader, criterion, use_cuda=args.use_cuda)

    print("Testing...")
    test_dset = MatDataset(datadir, 'test')
    test_loader = DataLoader(
        test_dset, batch_size=args.batch_size, shuffle=False, **loader_args)
    test_loss, test_avg_score, test_scores, y_true, y_pred = eval_epoch(
        model, test_loader, criterion, return_pred=True, use_cuda=args.use_cuda)

    if args.save_logs:
        import pandas as pd
        test_avg_score = pd.DataFrame.from_dict(test_avg_score, orient='index')
        test_avg_score.to_csv(args.outdir + '/metric.csv',
                  header=['value'], index_label='metric')
        np.savez_compressed(args.outdir + '/ypreds', ytrue=y_true.numpy(), ypred=y_pred.numpy())
        for key in test_scores:
            np.savetxt(args.outdir + '/{}.txt'.format(key), test_scores[key])
    return

if __name__ == "__main__":
    main()