# -*- coding: utf-8 -*-
"""
Created on Fri Dec 11 14:34:04 2015

@author: 89965
fonctions de structurelles diverses
"""
import os
import re
import logging
import subprocess
from collections import defaultdict
import psutil

import pyetl.formats.formats as F
import pyetl.formats.mdbaccess as DB
from .outils import charge_mapping, remap, prepare_elmap, renseigne_attributs_batch

LOGGER = logging.getLogger('pyetl')

def map_struct(regle):
    """mappe la structure clef etrangeres et fonctions"""
    charge_mapping(regle, mapping=regle.schema.mapping)

def _map_schemas(regle, obj):
    '''essaye de trouver un mapping pour une classe'''
    if obj is None:
        if regle.getvar("schema_entree"):
            schema_origine = regle.stock_param.schemas[regle.getvar("schema_entree")]
            print('-------------------------mapping', schema_origine)
#        else:
#            return
#        if regle.params.val_entree.val:
#            schema2 = regle.stock_param.init_schema(regle.params.val_entree.val,
#                                                    modele=schema_origine, origine='B')
#        else:
        return
    else:
        schema_origine = obj.schema.schema
        if regle.params.val_entree.val:
            schema2 = regle.stock_param.init_schema(regle.params.val_entree.val,
                                                    modele=schema_origine, origine='B')
        else:
            schema2 = obj.schema.schema
    regle.schema = schema2
    if schema2.elements_specifiques:
        for i in schema2.elements_specifiques:
#            print('mapping specifique', i)
            spec = schema2.elements_specifiques[i]
            mapped = remap(spec, regle.elmap)
#            print('mapping specifique', i, len(spec), '->', len(mapped))
            schema2.elements_specifiques[i] = mapped
    else:
        LOGGER.info("pas d'elements specifiques")

#        print("-----------------------------pas d'elements specifiques")

    for i in schema_origine.classes:
        schema2.get_classe(i, modele=schema_origine.classes[i], cree=True)
    for i in list(schema_origine.classes.keys()):
#        print ('map_schemas ',schema_origine.nom,i,regle.mapping.get(i))
        if i in regle.mapping:
            schema2.renomme_classe(i, regle.mapping[i])

        # mapping foreign keys :

#    print("mapping effectue", len(schema2.classes))
    for clef in schema2.classes:
        if clef in regle.mapping_attributs:
            for orig, dest in regle.mapping_attributs[clef].items():
                schema2.classes[clef].rename_attribut(orig, dest)

def applique_mapping(regle):
    """gere les clefs etrangeres et les elements speciaux dans les mappings"""
    mapping = regle.schema.mapping
    regle.elmap = prepare_elmap(mapping)
    _map_schemas(regle, None)
    regle.nbstock = 0
    for i in mapping:
        for scl in regle.schema.classes.values():
            scl.renomme_cible_classe(i, mapping[i])

def h_map2(regle):
    """ prepare le mapping des structures"""
    regle.store = True
    regle.blocksize = 1
    regle.nbstock = 0
    regle.traite_stock = applique_mapping

def f_map2(regle, obj):
    '''#aide||mapping en fonction d'une creation dynamique de schema
  #aide_spec||parametres: mappe les structures particulieres
   #pattern2||;;;map;=#struct;;
    '''
    regle.schema = obj.schema.schema
    regle.nbstock = 1

def h_map(regle):
    ''' precharge le fichier de mapping et prepare les dictionnaires'''
    regle.dynlevel = 0 # les noms de mapping dependent ils des donnees d entree
    regle.mapping = None
    regle.schema = None
#    if regle.params.att_sortie.val == '#schema': # mapping d un schema existant
#        schema2 =
    regle.changeschema = True
    fich = regle.params.cmp1.val
    if "[F]" in fich:
        regle.dynlevel = 2
    elif "[C]" in fich:
        regle.dynlevel = 1
    if regle.dynlevel:
        regle.clefdyn = ""
    else:
        charge_mapping(regle)
        _map_schemas(regle, None)

def f_map(regle, obj):
    '''#aide||mapping en fonction d'un fichier
  #aide_spec||parametres: map; nom du fichier de mapping
 #aide_spec2||si #schema est indique les objets changent de schema
    #pattern||?=#schema;?C;;map;C;;
  #test||obj||^#schema;test;;map;%testrep%/refdata/map.csv;;||atv;toto;A
 #test2||obj||^#schema;test;;map+-;%testrep%/refdata/map.csv;;||cnt;2
    '''
#    print ("dans map ===============",obj)
    if regle.dynlevel: # attention la regle est dynamique
        clef_dyn = regle.stock_param.chemin_courant if regle.dynlevel == 1\
                   else regle.stock_param.fichier_courant
        if clef_dyn != regle.clef_dyn:
            charge_mapping(regle)
    if not regle.schema:
        _map_schemas(regle, obj)
    clef = obj.ident
    schema2 = regle.schema
    if clef in regle.mapping:
        nouv = regle.mapping.get(clef)
        obj.setident(nouv, schema2=schema2)
        if clef in regle.mapping_attributs:
            for orig, dest in regle.mapping_attributs[clef].items():
                try:
                    obj.attributs[dest] = obj.attributs[orig]
                    del obj.attributs[orig]
                except KeyError:
                    obj.attributs[dest] = ''

        return True
#    print ('====================== mapping non trouve', clef)
#    print ('definition mapping', '\n'.join([str(i)+':\t\t'+str(regle.mapping[i])
#                                            for i in sorted(regle.mapping)]))
    return False

def store_traite_stock(regle):
    ''' relache les objets '''
    store = regle.tmpstore

    reverse = regle.params.cmp2.val == 'rsort'
#    print ("tri inverse ",reverse)
    if isinstance(store, list):
        if regle.params.cmp2.val:
            keyval = lambda obj: "|".join(obj.attributs.get(i, '')
                                          for i in regle.params.att_entree.liste)
            store.sort(key=keyval, reverse=reverse)
        for obj in store:
#            print ('store: relecture objet ', obj, obj.schema.identclasse,obj.schema.info)
            regle.stock_param.moteur.traite_objet(obj, regle.branchements.brch["end:"])
    else:
        for clef in sorted(store.keys(), reverse=reverse) if regle.params.cmp2.val else store:
            obj = store[clef]
            regle.stock_param.moteur.traite_objet(obj, regle.branchements.brch["end:"])
    h_stocke(regle) # on reinitialise

def h_stocke(regle):
    '''marque la regle comme stockante'''
#    print ('stockage tmpstore ', regle.params.att_entree.liste)
    regle.store = True
    regle.stocke_obj = True # on stocke les objets et pas que la clef
    regle.nbstock = 0
    regle.traite_stock = store_traite_stock
    regle.tmpstore = dict() if regle.params.cmp1.val else list()
    # mode comparaison : le stock est reutilise ailleurs (direct_reuse)=False
    regle.direct_reuse = not 'cmp' in regle.params.cmp1.val
    regle.fold = regle.params.cmp1.val == 'cmpf'
    if regle.params.cmp2.val == 'clef':
        regle.stocke_obj = False
        regle.tmpstore = set()

def f_stocke(regle, obj):
    '''#aide||stockage temporaire d'objets pour assurer l'ordre dans les fichiers de sortie
  #aide_spec||liste de clefs,tmpstore;uniq;sort|rsort : stockage avec option de tri
  #aide_spec2||liste de clefs,tmpstore;cmp;nom : prechargement pour comparaisons
   #pattern1||;;?L;tmpstore;?=uniq;?=sort;||
   #pattern2||;;?L;tmpstore;?=uniq;?=rsort;||
   #pattern3||;;?L;tmpstore;=cmp;A;?=clef||
   #pattern4||;;?L;tmpstore;=cmpf;A;?=clef||
       #test||obj;point;4||^;;V0;tmpstore;uniq;rsort||^;;C1;unique||atv;V0;3;
      #test2||obj;point;4||^V2;;;cnt;-1;4;||^;;V2;tmpstore;uniq;sort||^;;C1;unique;||atv;V2;1;
    '''
#    regle.stock.append(obj)
    if obj.virtuel:
        return True
    if regle.direct_reuse:
        regle.nbstock += 1

    if regle.params.cmp1.val:
        if len(regle.params.att_entree.liste) > 1:
            clef = "|".join(obj.attributs.get(i, '') for i in regle.params.att_entree.liste)
        else:
            clef = obj.attributs.get(regle.params.att_entree.val, '')
        if regle.stocke_obj:
            regle.tmpstore[clef] = obj
        else:
            regle.tmpstore.add(obj)
        return True
#    print ('store: stockage objet ', obj, obj.schema.identclasse,obj.schema.info)
    regle.tmpstore.append(obj)
    return True

def h_uniq(regle):
    ''' stocke les clefs pour l'unicite '''
    regle.tmpstore = set()

def f_uniq(regle, obj):
    '''#aide||unicite de la sortie laisse passer le premier objet et filtre le reste
    #aide_spec||liste des attibuts devant etre uniques si #geom : test geometrique
    #pattern||;?=#geom;?L;unique;;;
    #test||obj;point;2||^;;C1;unique||+fail:;;;;;;;pass>;;||cnt;1
    #test2||obj;point;2||^;;C1;unique-||cnt;1
    #test3||obj;point;2||^;#geom;;unique-||cnt;1
    #test4||obj;point;2||^;#geom;C1;unique-||cnt;1
    '''
#    regle.stock.append(obj)
    clef = str(tuple(tuple(i) for i in obj.geom_v.coords))\
               if regle.params.val_entree.val == '#geom' else ''
    clef = clef + "|".join(obj.attributs.get(i, '') for i in regle.params.att_entree.liste)

#    print ('uniq ',clef, regle.params.att_entree.val )
    if clef in regle.tmpstore:
        return False
    regle.tmpstore.add(clef)
    return True

def h_uniqcnt(regle):
    ''' stocke les clefs pour l'unicite '''
    regle.maxobj = regle.params.cmp1.num if regle.params.cmp1.num else 1
    regle.cnt = regle.maxobj > 1
    regle.tmpstore = defaultdict(int)

def f_uniqcnt(regle, obj):
    '''#aide||unicite de la sortie laisse passer les N premiers objet et filtre le reste
    #pattern||A;?=#geom;?L;unique;?N;||sortie
     #schema||ajout_attribut
       #test||obj;point;4||^X;;C1;unique;2;||+fail:;;;;;;;pass>;;||cnt;2
      #test2||obj;point;4||^X;;C1;unique-;2;||cnt;2
      #test3||obj;point;4||^X;#geom;;unique-;2;||cnt;2
      #test4||obj;point;4||^X;#geom;C1;unique-;2;||cnt;2
      #test4||obj;point;4||V0;1;;;V0;2;;set;;;||^X;#geom;V0;unique>;1;;||cnt;1
    '''
#    regle.stock.append(obj)
    clef = str(tuple(tuple(i) for i in obj.geom_v.coords))\
           if regle.params.val_entree.val == '#geom' else ''
    clef = clef + "|".join(obj.attributs.get(i, '') for i in regle.params.att_entree.liste)
    regle.tmpstore[clef] += 1
    obj.attributs[regle.params.att_sortie.val] = str(regle.tmpstore[clef])
    if regle.tmpstore[clef] > regle.maxobj:
        return False
    return True

def sortir_traite_stock(regle):
    '''ecriture finale'''
    print('traite stock sortir', regle.final)
    if regle.final:
        regle.f_sortie.ecrire_objets(regle, True)
        regle.nbstock = 0
        return
    for groupe in list(regle.stockage.keys()):
        for obj in regle.recup_objets(groupe):
            regle.f_sortie.ecrire_objets_stream(obj, regle, False)
            regle.stock_param.moteur.traite_objet(obj, regle.branchements.brch["end:"])
    regle.nbstock = 0

def h_sortir(regle):
    '''preparation sortie'''

    if regle.params.att_sortie.val == "#schema": # on force les noms de schema pour l'ecriture
        regle.nom_fich_schema = regle.params.val_entree.val
    else:
        regle.nom_fich_schema = regle.params.cmp2.val
    regle.nom_base = os.path.basename(regle.params.cmp2.val
                                      if regle.params.cmp2.val else regle.nom_fich_schema)

    if regle.debug:
        print("nom de schema ", regle.nom_fich_schema)

    if '[' in regle.params.cmp1.val: # on a defini un fanout
        tmplist = regle.params.cmp1.val.find('[')
        #print("valeur ii ", regle.params.cmp1,ii)

        regle.setvar("fanout", regle.params.cmp1.val[tmplist+1:-1])
        regle.params.cmp1.val = regle.params.cmp1.val[:tmplist]
    regle.f_sortie = F.Writer(regle.params.cmp1.val) # tout le reste
#    print ('positionnement writer ',regle, regle.params.cmp1.val)
    if regle.f_sortie.nom_format == 'sql': # gestion des dialectes sql et du mode connecté
        destination = regle.f_sortie.writerparms.get('destination')
        dialecte = regle.f_sortie.writerparms.get('dialecte')
        regle.f_sortie.writerparms['reinit'] = regle.getvar('reinit')
        regle.f_sortie.writerparms['nodata'] = regle.getvar('nodata')
        if destination: # on va essayer de se connecter
            connection = DB.dbaccess(regle.stock_param, destination)
            if connection.valide:
                regle.f_sortie.gensql = connection.gensql # la on a une instance connectee
        elif dialecte:
            regle.f_sortie.gensql = dialecte.gensql()
#        print ('sortie',regle.ligne,regle.f_sortie.writerparms)
    elif regle.f_sortie.nom_format == 'file': #gestion de fichiers de texte generiques
        dialecte = regle.f_sortie.writerparms.get('dialecte')
        regle.ext = dialecte

    if regle.params.cmp2.val and regle.params.cmp2.val != "#print":
        rep_base = regle.getvar('_sortie', loc=0)
#   print('positionnement sortie', rep_base, os.path.join(rep_base, regle.params.cmp2.val))

        regle.setvar('_sortie', os.path.join(rep_base, regle.params.cmp2.val), loc=1)

    regle.fanout = regle.getvar("fanout", 'groupe')\
                   if regle.f_sortie.multiclasse else 'classe'
#    print("fanout de sortie",regle.fanout)
    regle.calcule_schema = regle.f_sortie.calcule_schema
    regle.memlimit = int(regle.getvar('memlimit', 0))
    regle.store = None
    regle.nbstock = 0
    regle.traite_stock = sortir_traite_stock
#    regle.liste_attributs = regle.params.att_entree.liste
    if regle.stock_param.debug:
        print('sortir :', regle.params.att_entree.liste)
    regle.final = True
    regle.menage = True
    #print ('icsv: sortir copy:',regle.copy,'stream:',regle.stock_param.stream)
    if regle.copy and regle.getvar("mode_sortie", "A") == "D":
    # cette regle consomme les objets sauf si on est en mode copie et streaming
        regle.final = False
        regle.copy = False
    regle.valide = True
#    print ('fin preparation sortie ',regle.f_sortie.writerparms)

def setschemasortie(regle, obj):
    '''positionne le schema de sortie pour l objet '''
    if regle.nom_fich_schema:# on copie le schema pour ne plus le modifier apres ecriture
        regle.change_schema_nom(obj, regle.nom_fich_schema)
    if obj.schema and obj.schema.amodifier(regle):
        obj.schema.setsortie(regle.f_sortie, os.path.join(regle.getvar('_sortie'),
                                                          os.path.dirname(regle.params.cmp1.val)))
        obj.schema.setminmaj(regle.f_sortie.minmaj)
    if regle.params.att_entree.liste:
        obj.liste_atttributs = regle.params.att_entree.liste

def f_sortir(regle, obj):
    '''#aide||sortir dans differents formats
  #aide_spec||parametres:?(#schema;nom_schema);?liste_attributs;sortir;format[fanout]?;?nom
    #pattern||?=#schema;?C;?L;sortir;?C;?C||sortie
       #test||redirect||obj||^Z;ok;;set||^;;;sortir;csv;#print||end
    '''
    if obj.virtuel: # on ne traite pas les virtuels
        return True
    listeref = obj.liste_attributs
    schemaclasse_ref = obj.schema

    setschemasortie(regle, obj)
    if regle.store is None: # on decide si la regle est stockante ou pas
        regle.store = regle.f_sortie.calcule_schema and\
            (not obj.schema or not obj.schema.stable)
        if regle.store: # on ajuste les branchements
            regle.setstore()

    if regle.store:
        regle.nbstock += 1
        groupe = obj.attributs["#groupe"]
#        print("stockage", obj.ido, groupe, regle)
        if groupe != "#poubelle":
            nom_base = regle.nom_base
            #regle.stock_param.nb_obj+=1
            if regle.stock_param.stream: #sortie classe par classe
                if groupe  not in regle.stockage:
                    regle.f_sortie.ecrire_objets(regle, False) # on sort le groupe precedent
                    regle.compt_stock = 0
            regle.endstore(nom_base, groupe, obj, regle.final,
                           geomwriter=regle.f_sortie.tmp_geom, nomgeom=regle.f_sortie.nom_fgeo)
            return True

    regle.f_sortie.ecrire_objets_stream(obj, regle, False)
    obj.schema = None

    if regle.final:
        return True
    # la on regenere l'objet et on l'envoie dans le circuit poutr la suite
    obj.setschema(schemaclasse_ref)
    obj.liste_attributs = listeref
        # on reattribue le schema pour la sortie en simulant une copie
    return True

def valreplace(chaine, obj):
    '''remplace les elements provenant de l objet '''
    vdef = r'\[(#?[a-zA-Z_][a-zA-Z0-9_]*)\]'
    repl = lambda x: obj.attributs.get(x.group(1), '')
    return re.sub(vdef, repl, chaine)

def preload(regle, obj):
    '''prechargement'''
    vrep = lambda x: regle.resub.sub(regle.repl, x)
    chaine_comm = vrep(regle.params.cmp1.val)
    regle.setvar('nocomp', False)
    process = psutil.Process(os.getpid())

    mem1 = process.memory_info()[0]
    if obj and regle.params.att_entree.val:
        entree = obj.attributs.get(regle.params.att_entree.val, regle.fich)
    else:
        entree = regle.entree if regle.entree else valreplace(regle.fich, obj)

    print('------- preload commandes:(', chaine_comm, ') f:', entree,
          'clef', regle.params.att_sortie.val)
    if chaine_comm: # on precharge via une macro
        nomdest = regle.params.cmp2.val if regle.params.cmp2.val.startswith('#') \
                                        else '#'+ regle.params.cmp2.val
        processor = regle.stock_param.getpyetl(chaine_comm, entree=entree, rep_sortie=nomdest)
        processor.process()
        renseigne_attributs_batch(regle, obj, processor.retour)

        print('------- preload ', processor.store)
        regle.stock_param.store.update(processor.store) # on rappatrie les dictionnaires de stockage
        regle.setvar('storekey', processor.retour)  # on stocke la clef

    else:
#        racine = regle.stock_param.racine
        chemin = os.path.dirname(entree)
        fichier = os.path.basename(entree)
        ext = os.path.splitext(fichier)[1]
        lecteur = regle.stock_param.reader(ext)
        regle.reglestore.tmpstore = dict()
        nb_total = 0
        try:
            nb_total = lecteur.lire_objets('', chemin, fichier, regle.stock_param,
                                           regle.reglestore)
            regle.stock_param.store[regle.params.cmp2.val] = regle.reglestore.tmpstore
        except FileNotFoundError:
            regle.stock_param.store[regle.params.cmp2.val] = None
            print('fichier inconnu', os.path.join(chemin, fichier))

    mem2 = process.memory_info()[0]
    mem = mem2-mem1
    print('------- preload ', nb_total, mem, '--------', int(mem/(nb_total+1)))

def h_preload(regle):
    '''prechargement'''
    obj = None
    mapper = regle.stock_param
    reglestore = mapper.interpreteur(";;;;;;"+regle.params.att_sortie.val+
                                     ";tmpstore;cmp;"+regle.params.cmp2.val, "", 99999)
    regle.reglestore = reglestore
    regle.repl = lambda x: obj.attributs.get(x.group(1), '')
    regle.resub = re.compile(r'\[(#?[a-zA-Z_][a-zA-Z0-9_]*)\]')
    fich = regle.params.val_entree.val
#    fich = fich.replace('[R]', regle.stock_param.racine)
    regle.fich = fich
    regle.dynlevel = 0
    if '[R]' in fich:
        regle.dynlevel = 1
    if "[F]" in fich:
        regle.dynlevel = 2
    elif "[G]" in fich:
        regle.dynlevel = 1
    elif "[" in fich:
        regle.dynlevel = 3
    regle.entree = None

    if regle.dynlevel == 0: # pas de selecteur on precharge avant de lire
        regle.entree = regle.params.val_entree.val
        regle.fich = regle.entree
        preload(regle, None)
        regle.valide = "done"

    print('==================h_preload===', regle.dynlevel, regle.valide)

def f_preload(regle, obj):
    '''#aide||precharge un fichier en appliquant une macro
  #aide_spec||parametres clef;fichier;attribut;preload;macro;nom
 #aide_spec1||les elements entre [] sont pris dans l objet courant
 #aide_spec2||sont reconnus[G] pour #groupe et [F] pour #classe pour le nom de fichier
    #pattern||A;?C;?A;preload;?C;C
    #!test||
    '''
    fich = regle.fich

    if regle.dynlevel > 0:
        fich = fich.replace('[G]', obj.attributs['#groupe'])
        fich = fich.replace('[R]', regle.stock_param.racine)
        fich = fich.replace('[F]', obj.attributs['#classe'])
        if fich != regle.entree:
            regle.entree = fich
            print('==================f_preload===', regle.stock_param.racine, regle.entree)

            preload(regle, obj)
#            print ('chargement ',regle.params.cmp2.val,
#                   regle.stock_param.store[regle.params.cmp2.val])
    return True

def compare_traite_stock(regle):
    """ sort les objets detruits"""
    for obj in regle.comp.values():
        obj.attributs[regle.params.att_sortie.val] = 'supp'
        obj.setident(regle.precedent)
        regle.stock_param.moteur.traite_objet(obj, regle.branchements.brch["supp:"])
    regle.comp = None
    regle.nbstock = 0

#def compare_traite_stock(regle):
#    """ sort les objets detruits"""
#    for clef, obj in regle.comp.items():
#        if obj.redirect is None:
#            obj.attributs[regle.params.att_sortie.val]='supp'
#            regle.stock_param.moteur.traite_objet(obj, regle.branchements.brch["supp:"])
#        regle.comp[clef] = None
#    regle.comp = None
#    regle.nbstock = 0

def h_compare(regle):
    """comparaison a une reference"""
    regle.branchements.addsortie('new:')
    regle.branchements.addsortie('supp:')
    regle.branchements.addsortie('diff:')
    regle.branchements.addsortie('orig:')
#    regle.taites = set()
    regle.store = True
    regle.nbstock = 0
    regle.comp = None
    regle.precedent = None
    regle.traite_stock = compare_traite_stock

def f_compare2(regle, obj):
    '''#aide||compare a un element precharge
  #aide_spec||parametres clef;fichier;attribut;preload;macro;nom
 #aide_spec2||sort en si si egal en sinon si different
 #aide_spec3||si les elements entre [] sont pris dans l objet courant
    #pattern||A;;?L;compare2;A;C
     #helper||compare
     #schema||ajout_attribut
    #!test||
    '''
    if regle.precedent != obj.ident:
        comp = regle.stock_param.store[regle.params.cmp2.val]
        if regle.comp and comp is not regle.comp:
            compare_traite_stock(regle)
            regle.nbstock = 1
        regle.comp = comp
        if regle.comp:
            if regle.params.att_entree.liste:
                regle.comp2 = {i:([i.attributs[j] for j in
                                   regle.params.att_entree.liste]) for i in regle.comp}
            else:
                regle.comp2 = {i:([i.attributs[j] for j in
                                   sorted([k for k in i.attributs if k[0] != "#"])])
                               for i in regle.comp}
#    print ('comparaison ', len(regle.comp), regle.comp)
    try:
        if len(regle.params.cmp1.liste) > 1:
            clef = "|".join(obj.attributs.get(i, '') for i in regle.params.att_entree.liste)
        else:
            clef = obj.attributs[regle.params.cmp1.val]
        ref = regle.comp2[clef]
        regle.ref.add(clef)
    except KeyError:
        obj.redirect = "new:"
        obj.attributs[regle.params.att_sortie.val] = 'new'
        return False
    if regle.params.att_entree.liste:
        compare = all([obj.attributs[i] == ref.attributs[i]
                       for i in regle.params.att_entree.liste])
    else:
        atts = {i for i in obj.attributs if i[0] != "#"}
        kref = {i for i in ref.attributs if i[0] != "#"}
    #    id_att = atts == kref
        compare = atts == kref and all([obj.attributs[i] == ref.attributs[i]
                                        for i in atts]) and obj.geom == ref.geom
    if compare:
        return True
    obj.redirect = "diff:"
    obj.attributs[regle.params.att_sortie.val] = 'diff'
    ref.attributs[regle.params.att_sortie.val] = 'orig'
    regle.stock_param.moteur.traite_objet(ref, regle.branchements.brch["orig:"])
    # on remet l'original dans le circuit
    return False

def f_compare(regle, obj):
    '''#aide||compare a un element precharge
  #aide_spec||parametres clef;fichier;attribut;preload;macro;nom
 #aide_spec2||sort en si si egal en sinon si different
 #aide_spec3||si les elements entre [] sont pris dans l objet courant
    #pattern||A;;?L;compare;A;C
     #schema||ajout_attribut
    #!test||
    '''
    if regle.precedent != obj.ident: # on vient de changer de classe
        if regle.comp:
            compare_traite_stock(regle)
            regle.nbstock = 1
        regle.comp = regle.stock_param.store[regle.params.cmp2.val]
        regle.precedent = obj.ident
#    print ('comparaison ', len(regle.comp), regle.comp)
    if regle.comp is None:
        return False

    try:
        if len(regle.params.cmp1.liste) > 1:
            clef = "|".join(obj.attributs.get(i, '')
                            for i in regle.params.att_entree.liste)
        else:
            clef = obj.attributs[regle.params.cmp1.val]
        ref = regle.comp.pop(clef)
    except KeyError:

        obj.redirect = "new:"
        obj.attributs[regle.params.att_sortie.val] = 'new'
        return False
    if regle.params.att_entree.liste:
        compare = all([obj.attributs[i] == ref.attributs[i]
                       for i in regle.params.att_entree.liste])
    else:
        atts = {i for i in obj.attributs if i[0] != "#"}
        kref = {i for i in ref.attributs if i[0] != "#"}
    #    id_att = atts == kref
        compare = atts == kref and all([obj.attributs[i] == ref.attributs[i]
                                        for i in atts]) and obj.geom == ref.geom
    if compare:
        return True
    obj.redirect = "diff:"
    obj.attributs[regle.params.att_sortie.val] = 'diff'
    ref.attributs[regle.params.att_sortie.val] = 'orig'
    ref.setident(obj.ident) # on force l'identite de l'original
    regle.stock_param.moteur.traite_objet(ref, regle.branchements.brch["orig:"])
    # on remet l'original dans le circuit
    return False

def f_run(regle, obj):
    '''#aide||execute un programme exterieur
  #aide_spec||attribut qui recupere le resultat, parametres , run , nom, parametres
    #pattern||?A;?C;?A;run;C;?C
     #schema||ajout_attribut
    '''
    chaine = ' '.join((regle.params.cmp1.val, regle.params.cmp2.val,
                       obj.attributs.get(regle.params.att_entree.val, regle.params.val_entree.val)))
    fini = subprocess.run(chaine, stderr=subprocess.STDOUT)
    if regle.params.att_sortie.val:
        obj.attributs[regle.params.att_sortie.val] = str(fini)


from lib import setter, getter, io_tools
import argparse

parser = argparse.ArgumentParser()
parser.add_argument("--config", type = str, help = "path to campaigns config json file")
parser.add_argument("--PU", type = str, help = "name of the pileup sample to set sitewhitelist for")
parser.add_argument("--sites",  type = str, nargs = "*", help = "site whitelist for the pileup")
args = parser.parse_args()

config_dict = io_tools.import_jsonfile_as_OrderedDict(args.config)
campaigns = getter.get_campaigns_given_PU(config_dict, args.PU)

for campaign in campaigns:
    config_dict[campaign]['secondaries'][args.PU]['SiteWhitelist'] = args.sites

io_tools.export_dict_to_jsonfile(config_dict, 'campaigns.json')


#programmers_단어 변환

#=== import module ===#
from collections import deque
#=== variable declare ===#

#=== Function define ===#
def solution(begin, target, words):
  if target not in words: return 0; #불가능한 경우

  queue = deque();
  queue.append([begin,0]); #current, visited
  level = 0;
  succeed = False;
  while queue and not succeed:
    level += 1;
    for i in range(len(queue)):
      current,visited = queue.popleft();
      for idx in range(len(words)):
        if visited & (1 << idx) != 0: continue; #이미 방문한 단어
        nextWord = words[idx];
        diff = 0;
        for i in range(len(current)):
          if current[i] != nextWord[i]: diff += 1;
        if diff != 1: continue; #다른 것이 2개 이상이라서 한번에 변환 불가능
        if nextWord == target: #성공 조건
          succeed = True; break;
        queue.append([nextWord,visited | (1 << idx)]);

  if succeed: return level;
  else: return 0;

#=== main function ===#
print(solution("hit","cog",["hot", "dot", "dog", "lot", "log"]));


import re
import os
import string
import shutil
import tempfile
import fontforge
import argparse
from string import Template
from pathlib import Path
from bs4 import BeautifulSoup
from bs4.formatter import XMLFormatter

class Colors:
    OK = '\033[92m'
    INFO = '\033[94m'
    WARN = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'

class SVGProcessor:
    _path = None
    _spool = None
    _font_name = 'IconFont'
    _qml_namespace = 'IconFont'
    _qml_element_name = 'Icon'
    _copyright = '(C) 2020 GONICUS GmbH'
    _out_path = '.'
    _strip_rect = False
    _qt = False

    def __init__(self, source_path, spool):
        self._path = source_path
        self._spool = spool

    def run(self):
        objects = {}
        objects_lt = {}
        index = 61000

        for path in Path(self._path).rglob('*.svg'):
            try:
                svg = BeautifulSoup(open(path).read().encode('utf-8'), 'xml')
            except FileNotFoundError:
                print(f"{Colors.FAIL}âœ—{Colors.ENDC} file not found {Colors.BOLD}{path}{Colors.ENDC}")
                return

            if self._process(svg, path):
                spool_name = os.path.join(self._spool, f'{index}.svg')
                with open(spool_name, 'w') as f:
                    f.write(svg.prettify(formatter=XMLFormatter()))

                objects[index] = spool_name
                objects_lt[index] = os.path.splitext(str(path)[len(self._path) + 1:])[0]
                index += 1

        # Do font processing
        if self._make_font(objects):
            self._write_header()
            self._write_cpp(objects_lt)
            if self._qt:
                self._write_qml()

    def _write_header(self):
        font_name = self._font_name.upper()
        file_name = self._font_name + '.h'
        if self._qt:
            header = Template("""#ifndef ${FONT_NAME}_H
#define ${FONT_NAME}_H
#include <QObject>
#include <QtQml>
class ${NAME}Resolver : public QObject {
    Q_OBJECT
    QML_ELEMENT
public:
    explicit ${NAME}Resolver(QObject* parent = nullptr);
    virtual ~${NAME}Resolver() {}
    Q_INVOKABLE quint16 indexOfPath(const QString& iconPath);
};
#endif
""")
        else:
            header = Template("""#ifndef ${FONT_NAME}_H
#define ${FONT_NAME}_H
#include <cstdint>
#include <string>
namespace $NAME {
uint16_t index(const std::string& path);
}
#endif
""")

        with open(os.path.join(self._out_path, file_name), 'w') as f:
            f.write(header.substitute(FONT_NAME=font_name, NAME=self._font_name))
            print(f'{Colors.OK}âœ“{Colors.ENDC} {f.name} has been generated')

    def _write_cpp(self, objects):
        font_name = self._font_name.upper()
        file_name = self._font_name + '.cpp'
        data = '\n'.join(f'               {{ "{name}", {index} }},' for index, name in objects.items())

        if self._qt:
            code = Template("""#include <QFontDatabase>
#include <QHash>
#include "${NAME}.h"
${NAME}Resolver::${NAME}Resolver(QObject* parent) : QObject(parent) {
    static bool initialized = false;
    if (!initialized) {
        initialized = true;
        QFontDatabase::addApplicationFont(":/${NAME}.ttf");
    }
}
quint16 ${NAME}Resolver::indexOfPath(const QString& iconPath) {
    static QHash<const QString, quint16> lookup_table {
$DATA
    };
    return lookup_table.value(iconPath, 0);
}
""")
        else:
            code = Template("""#include <iostream>
#include <map>
#include "${NAME}.h"
namespace $FONT_NAME {
    uint16_t index(const std::string& path) {
        static std::map<std::string, uint16_t> lookup_table {
$DATA
        };
        auto idx = lookup_table.find(path);
        return idx == lookup_table.end() ? 0 : idx->second;
    }
}
""")

        with open(os.path.join(self._out_path, file_name), 'w') as f:
            f.write(code.substitute(NAME=self._font_name, FONT_NAME=font_name, DATA=data))
            print(f'{Colors.OK}âœ“{Colors.ENDC} {f.name} has been generated')

    def _write_qml(self):
        font_name = self._font_name.upper()
        file_name = self._font_name + '.qml'
        code = Template("""import QtQuick 2.15
import ${COMPONENT} 1.0 as IconFont
/// Loads and displays an icon of the icon font by giving the path to the icon svg file
Item {
    id: control
    width: icon.implicitWidth
    height: control.size
    /// Path to the icon svg file that should be loaded; empty string (default) unloads the icon
    property string iconPath
    /// Size of the icon in pixels (default: 32)
    property int size: 32
    /// Color of the icon (default: black)
    property alias color: icon.color
    IconFont.${NAME}Resolver {
        id: resolver
    }
    Text {
        id: icon
        text: String.fromCharCode(resolver.indexOfPath(control.iconPath))
        verticalAlignment: Text.AlignVCenter
        horizontalAlignment: Text.AlignHCenter
        anchors.centerIn: parent
        font.family: "${NAME}"
        font.pixelSize: control.size
    }
}
""")

        with open(os.path.join(self._out_path, self._qml_element_name + ".qml"), 'w') as f:
            f.write(code.substitute(FONT_NAME=font_name, NAME=self._font_name, COMPONENT=self._qml_namespace))
            print(f'{Colors.OK}âœ“{Colors.ENDC} {f.name} has been generated')

    def _process(self, svg, path):
        # Skip icons that have no square dimensions
        main = svg.find('svg')
        if 'width' in main and 'height' in main:
            if main['width'] != main['height']:
                print(f"{Colors.WARN}âš {Colors.ENDC} {Colors.BOLD}{path}{Colors.ENDC} aspect ratio is not 1:1 - skipping")
                return False

            # Remove unit from size
            width = int(re.findall(r'\d+', main['width'])[0])
            height = int(re.findall(r'\d+', main['height'])[0])

        # Remove bounding rectangles if any
        if self._strip_rect:
            for rect in svg.find_all('rect'):
                if int(re.findall(r'\d+', rect['height'])[0]) == height and int(re.findall(r'\d+', rect['width'])[0]) == width:
                    rect.extract()

        # Find element
        element = self._findElement(svg)

        # Check if there's no element
        if len(svg.find_all(element)) == 0:
            print(f"{Colors.WARN}âš {Colors.ENDC} file {Colors.BOLD}{path}{Colors.ENDC} has no relevant elements - skipping")
            return False

        # Check if there's more than one element
        if len(svg.find_all(element)) != 1:
            print(f"{Colors.INFO}ðŸ›ˆ{Colors.ENDC} file {Colors.BOLD}{path}{Colors.ENDC} has no too many elements")

        # Skip icons that use a 'rotate'
        if svg.find(element, transform=re.compile('^rotate\(')):
            print(f"{Colors.WARN}âš {Colors.ENDC} file {Colors.BOLD}{path}{Colors.ENDC} contains rotation - skipping")
            return False

        return True

    def _findElement(self, svg):
        for el in ['path', 'polygon', 'rect', 'circle']:
            if len(svg.find_all(el)) != 0:
                return el

        return None

    def _make_font(self, objects):
        first = True
        font = fontforge.font()
        font.encoding = 'UnicodeFull'
        font.fontname = self._font_name
        font.familyname = self._font_name
        font.fullname = self._font_name
        font.copyright = self._copyright

        for index, path in objects.items():
            if first:
                char = font.createChar(87)
                char.importOutlines(str(path))
                first = False

            char = font.createChar(index)

            try:
                char.importOutlines(str(path))
            except FileNotFoundError:
                print(f"{Colors.FAIL}âœ—{Colors.ENDC} file not found {Colors.BOLD}{path}{Colors.ENDC}")
                return False

        font.selection.all()

        path = os.path.join(self._out_path, self._font_name + ".ttf")
        font.generate(path)
        print(f'{Colors.OK}âœ“{Colors.ENDC} {path} has been generated')

        return True

    def __set_font_name(self, name):
        allowed = set(string.ascii_lowercase + string.ascii_uppercase + string.digits + '_')
        if set(name) <= allowed:
            self._font_name = name
        else:
            print(f"{Colors.FAIL}âœ—{Colors.ENDC} only uppercase/lowercase characters, digits and _ are allowed for the font name")
            exit()

    def __get_font_name(self):
        return self._font_name

    def __set_out_path(self, path):
        self._out_path = path

    def __get_out_path(self):
        return self._out_path

    def __set_copyright(self, data):
        self._copyright = data

    def __get_copyright(self):
        return self._copyright

    def __set_strip_rect(self, data):
        self._strip_rect = data

    def __get_strip_rect(self):
        return self._strip_rect

    def __set_qt(self, data):
        self._qt = data

    def __get_qt(self):
        return self._strip_rect

    def __set_qml_element(self, data):
        self._qml_element_name = data

    def __get_qml_element(self):
        return self._qml_element_name

    def __set_qml_namespace(self, data):
        self._qml_namespace = data

    def __get_qml_namespace(self):
        return self._qml_namespace

    font_name = property(__get_font_name, __set_font_name)
    out = property(__get_out_path, __set_out_path)
    copyright = property(__get_copyright, __set_copyright)
    strip_rect = property(__get_strip_rect, __set_strip_rect)
    qt = property(__get_qt, __set_qt)
    qml_namespace = property(__get_qml_namespace, __set_qml_namespace)
    qml_element = property(__get_qml_element, __set_qml_element)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('source')
    parser.add_argument('--font-name', help='name of the generated font', default='IconFont')
    parser.add_argument('--copyright', help='copyright notice placed inside the generated TTF file', default='(C) 2020 GONICUS GmbH')
    parser.add_argument('--output', help='path where generated files are placed', default='.')
    parser.add_argument('--strip-bounding-rect', action="store_true", help='path where generated files are placed')
    parser.add_argument('--qt', action="store_true", help='whether to build Qt/QML style output files')
    parser.add_argument('--qml-namespace', help='name of the QML namespace used in your .pro file', default='IconApp')
    parser.add_argument('--qml-element', help='name of the QML icon element for this font', default='Icon')
    args = parser.parse_args()

    with tempfile.TemporaryDirectory() as spool:
        processor = SVGProcessor(args.source, spool)
        processor.font_name = args.font_name
        processor.out = args.output
        processor.copyright = args.copyright
        processor.strip_rect = args.strip_bounding_rect
        processor.qt = args.qt
        processor.qml_element = args.qml_element
        processor.qml_namespace = args.qml_namespace
        processor.run()

    del processo


import gluonbook as gb
from mxnet.gluon import data as gdata
import sys
import time
import matplotlib.pyplot as plt

mnist_train = gdata.vision.FashionMNIST(train=True)
mnist_test = gdata.vision.FashionMNIST(train=False)

# 训练集和测试集中每个类别的图像分别为6000, 1000, 因此len(mnist_train)=60000, len(mnist_test) = 10000
print(len(mnist_train), len(mnist_test))

# feature 对应高和宽均为28像素的图像, 每个像素的数值为0-255之间的8位无符号整数(unit8). 使用三维NDArray存储
feature, label = mnist_train[0]

print(feature.shape, feature.dtype)

print(label, type(label), label.dtype)

# 将数值标签转成相应的文本标签
def get_fashion_mnist_labels(labels):
    text_labels = ['t-shirt', 'trouser', 'pullover', 'dress', 'coat',
                   'sandal', 'shirt', 'sneaker', 'bag', 'ankle boot']
    return [text_labels[int(i)] for i in labels]

# 定义可以在一行里画出多个图像和对应标签的函数
def show_fashion_mnist(images, labels):
    #gb.use_svg_display()
    # 这里的 _ 表示我们忽略（不使用）的变量。
    _, figs = plt.subplots(1, len(images), figsize=(12, 12))
    # zip() 函数用于将可迭代对象作为参数，将对象中对应的元素打包成一个个元组，然后返回由这些元组组成的对象。
    # 如果各个可迭代对象的元素个数不一致，则返回的对象长度与最短的可迭代对象相同。
    for f, img, lbl in zip(figs, images, labels):
        f.imshow(img.reshape((28, 28)).asnumpy())
        f.set_title(lbl)
        f.axes.get_xaxis().set_visible(False)
        f.axes.get_yaxis().set_visible(False)
    plt.show()

# 显示训练集中0-11号图像
X, y = mnist_train[0:12]
show_fashion_mnist(X, get_fashion_mnist_labels(y))

batch_size = 256
# Vision Transforms: Transforms can be used to augment input data during training. You can compose multiple transforms sequentially
# ToTensor: Converts an image NDArray to a tensor NDArray.
# 通过ToTensor类将图像数据从 uint8 格式变换成 32 位浮点数格式，并除以 255 使得所有像素的数值均在 0 到 1 之间。
# ToTensor类还将图像通道从最后一维移到最前一维来方便之后介绍的卷积神经网络计算。
transformer = gdata.vision.transforms.ToTensor()

# Gluon的DataLoader允许使用多进程来加速数据读取(暂不支持 Windows 操作系统)
# 通过参数num_workers来设置4个进程读取数据。
if sys.platform.startswith('win'):
    num_workers = 0
else:
    num_workers = 4

# transform_first(fn, lazy=True): Returns a new dataset with the first element of each sample transformed by the transformer function fn.
# 通过数据集的transform_first函数，我们将ToTensor的变换应用在每个数据样本（图像和标签）的第一个元素，即图像之上。
# class mxnet.gluon.data.DataLoader(dataset, batch_size=None, shuffle=False, sampler=None, last_batch=None, batch_sampler=None,
#                                   batchify_fn=None, num_workers=0, pin_memory=False, prefetch=None)
train_iter = gdata.DataLoader(mnist_train.transform_first(transformer),
                              batch_size, shuffle=True,
                              num_workers=num_workers)
# print(train_iter)

test_iter = gdata.DataLoader(mnist_test.transform_first(transformer),
                            batch_size, shuffle=False,
                            num_workers=num_workers)
# print(test_iter)

start = time.time()
for X, y in train_iter:
    continue
print('%.2f sec' % (time.time() - start))


# -*- coding: utf-8 -*-
import matplotlib.pyplot as plt
import numpy as np
import pickle
import configparser
import copy
import subprocess
from distutils.util import strtobool

import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
from torch.cuda.amp import autocast, GradScaler

# from AutoEncoder import AE,DataIO,FlowDataset,SlidingSampler,FSI
from AutoEncoder import AE
from AutoEncoder import DataIO as dio
from AutoEncoder import FlowDataset as fds
from AutoEncoder import SlidingSampler as ss

from ForceAutoEncoder import FAE
from ForceAutoEncoder import DataIO as dio_force
from ForceAutoEncoder import ForceDataset as forcds
from ForceAutoEncoder import SlidingSampler as ss_force

from ConvxOpt import ConvxOpt, FSI

"""Set our seed and other configurations for reproducibility."""
seed = 10
torch.manual_seed(seed)
np.random.seed(seed)
torch.cuda.manual_seed(seed)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True

""" Define GradScaler """
scaler = GradScaler()  # point1: Scaling the gradient information

""" read config file """
setup = configparser.ConfigParser()
setup.read('input.ini')
epochs = int(setup['DeepLearning']['epochs'])
learning_rate = float(setup['DeepLearning']['learning_rate'])
optthresh = float(setup['DeepLearning']['optthresh'])
target_loss  = float(setup['DeepLearning']['target_loss'])
batch_size = int(setup['DeepLearning']['batchsize'])
window_size = int(setup['DeepLearning']['batchsize'])
sliding = int(setup['DeepLearning']['sliding'])
fc_features = int(setup['DeepLearning']['full_connected'])

control = strtobool(setup['Control']['control'])
inptype = int(setup['Control']['inptype'])

ured = float(setup['MPC']['ured'])
R    = float(setup['MPC']['R'])

"""We set the preference about the CFD"""
dt  = float(setup['CFD']['dt'])
mach= float(setup['CFD']['mach'])
re  = float(setup['CFD']['re'])
iz  = int(setup['CFD']['iz'])

"""We set the start step, the last step, the intervals"""
nst = int(setup['MPC']['nst'])
nls = int(setup['MPC']['nls'])
nin = int(setup['CFD']['nin'])

""" Dataset """
gpaths = setup['CFD']['gpaths']
fpaths = setup['CFD']['fpaths']
fmpaths= setup['Control']['fmpaths']

""" Set Dynamics """
print('Set Dynamics...\n')
dataio = dio(nst,nls,nin,gpaths,fpaths,iz,fmpaths)

grids,ibottom = dataio.readgrid()
js,je,ks,ke,ls,le,ite1,ite2,jd,imove = ibottom

# cropped indices
jcuts = [0,je+1  ,1]
kcuts = [0,ke+1-2,1]
lcuts = [0,le+1-100,1]

# output cropped grid
dataio.tweak_writegrid(['grid_z0003'],grids,jcuts,kcuts,lcuts)

flows  = dataio.readflow()
control_inp = None
if control: control_inp = dataio.readformom(inptype)
# Set Tensor form
transform = torchvision.transforms.Compose([
    torchvision.transforms.ToTensor()
    ])

test_dataset = fds(2,jcuts,kcuts,lcuts,flows,control_inp,control,transform)

sampler = ss(test_dataset,batch_size,sliding)

test_loader = torch.utils.data.DataLoader(
    test_dataset,
    sampler = sampler
)

orgdatas = []

for batch,label,u in test_loader:
    test = batch[0][0]
    tmp = label
    orgdatas.append(test)

maxstep = int( torch.max(tmp).item() )

print('Set Forces...')
dioforce = dio_force(nst,nls,nin,gpaths,fpaths,iz,fmpaths)
forces = dioforce.readformom(0) # 0 : Only CL
transform_force = torchvision.transforms.Compose([
    torchvision.transforms.ToTensor()
     ])
test_dataset_force = forcds(2,jcuts,kcuts,lcuts,forces,window_size,sliding,control_inp,control,transform_force)

sampler_force = ss_force(test_dataset_force,window_size,sliding)

test_loader_force = torch.utils.data.DataLoader(
    test_dataset_force,
    sampler = sampler_force
)

print('Start MPC')
#  use gpu if available
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

""" Load models """
model = torch.load("learned_model")
model_force = torch.load("learned_model_force")

reconstruction = []
step = nst

""" set instances """
convxopt = ConvxOpt(batch_size,inptype)
horizon  = convxopt.T # horizontal window

fsi = FSI(jcuts,kcuts,lcuts,iz,dataio,mach,re,dt,inptype,ured,horizon)

with torch.no_grad():
    # Initial state variables D_0 (X_0, Y_0)
    features = next( iter(test_loader) ) # D_0

    for icount in range(maxstep):
        print('step = ', step)
        step = step + nin*sliding

        # # Set Fluid Force
        # batch = features[0]
        # batch = torch.squeeze(batch)
        # batch = batch.to(torch.float32).to('cuda')
        if control: u = torch.squeeze(features[2]).to(torch.float32).to('cuda')

        # ## standalized input batches
        # shift = torch.mean(batch,(0,2,3)).to(torch.float32)
        # scale = torch.std(batch,(0,2,3)).to(torch.float32)
        # for i in range(5):
        #     batch[:,i,:,:] = (batch[:,i,:,:] - shift[i])/(scale[i]+1.0e-11)

        # ## compute reconstructions using autocast
        # with autocast(False): # point 2 :automatic selection for precision of the model
        #     if control:
        #         inp = [batch,u]
        #     else:
        #         print('MPC needs control')
        #         exit()

        #     ### Extract gx in latent space and A, B matrices
        #     gx,A,B = model.encoder_forMPC(inp)
        #     cvec = gx[:,:horizon]
        # ## prepare the objective function

        # exit()
        # ## unstandalized
        # for i in range(5):
        #     X_tilde[:,i,:,:] = X_tilde[:,i,:,:] * (scale[i]+1.0e-11) + shift[i]

        # Deep FSI
        # forces = fsi.calc_force(X_tilde[:ind_half],u[:ind_half])
        ''' test '''
        fluid_forces = next(iter(test_loader_force))[0].to(torch.float32).to('cuda')
        struct_forces = fsi.structure_force(u,inptype,ured,mach)
        struct_forces = torch.from_numpy(struct_forces)[None].to(torch.float32).to('cuda')
        ''''''''''''

        ## map forces into the latent space
        ### map fluid forces
        batch = fluid_forces

        with autocast(False): # point 2 :automatic selection for precision of the model
            if control:
                inp = [batch,u[0]]
            else:
                print('MPC needs control')
                exit()

            ### Extract gx in latent space and A, B matrices
            gx,Af,Bf = model_force.encoder_forMPC(inp)
            cvec_fluid = gx[:,:horizon]

        ### map structure forces
        batch = struct_forces

        with autocast(False): # point 2 :automatic selection for precision of the model
            if control:
                inp = [batch,u[0]]
            else:
                print('MPC needs control')
                exit()

            ### Extract gx in latent space and A, B matrices
            gx,_,_ = model_force.encoder_forMPC(inp)
            cvec_struct = gx[:,:horizon]

        # MPC
        cforces = [fluid_forces,struct_forces]
        u_optim = convxopt.solve_cvx(cforces,R,Af,Bf)
        exit()
        reconstruction.append(X_tilde[0].cpu())

    # """ Calc recreated error """
    recerrors = []
    for i,X_tilde in enumerate(reconstruction):
        recdata = X_tilde.cpu().numpy()
        orgdata = orgdatas[i].cpu().numpy()

        # data shape = (batch * channels * height * width)
        # error_norm = np.linalg.norm(recdata-orgdata,axis=1,ord=1)
        # org_norm = np.linalg.norm(orgdata,axis=1,ord=1)

        error_norm = np.linalg.norm(recdata-orgdata,axis=0,ord=1)
        org_norm = np.linalg.norm(orgdata,axis=0,ord=1)

        recerror = error_norm/(org_norm)
        recerrors.append(recerror)

    f = open('recerrors.pickle', 'wb')
    pickle.dump(recerrors, f)

"""## Visualize Results
Let's try to reconstruct some test images using our trained autoencoder.
"""
print('Post')
with torch.no_grad():
    nstepall = np.arange(nst,nls+nin,nin*sliding)

    # write grid
    out_gfiles = [
        './grid_z0003'
    ]
    dataio.writegrid(out_gfiles,grids,jcuts,kcuts,lcuts)

    # write flow
    statedic = []

    for i,rec in enumerate(reconstruction):
        batch = rec.cpu().numpy()

        nstep = nstepall[i]
        fname = 'recflows/u3.0/recflow_z{:0=2}_{:0=8}'.format(iz,nstep)

        q = copy.deepcopy( batch )

        dataio.writeflow(fname,q,jcuts,kcuts,lcuts)


from django.contrib.auth import get_user_model
from django.core.validators import MaxValueValidator, MinValueValidator
from django.db import models
from django.utils.translation import gettext_lazy as _

from library_test_project.users.models import ScoreAbs

User = get_user_model()

class Author(models.Model):
    name = models.CharField(_("Name of author"), max_length=255)

class Genre(models.Model):
    name = models.CharField(_("Name of genre"), max_length=255)

class Book(ScoreAbs, models.Model):
    author = models.ForeignKey(Author, on_delete=models.CASCADE, related_name="books", verbose_name=_("author"))
    genre = models.ForeignKey(Genre, on_delete=models.CASCADE, related_name="genre", verbose_name=_("genre"))
    name = models.CharField(_("Name of book"), max_length=255)
    description = models.TextField(_("Description"))
    published_date = models.DateTimeField(_("Published date"), auto_now_add=True)
    scored_users = models.ManyToManyField(User, through="BookScoredUsers")

class Comment(models.Model):
    owner = models.ForeignKey(User, on_delete=models.CASCADE, related_name="comments", verbose_name=_("Owner"))
    book = models.ForeignKey(Book, on_delete=models.CASCADE, related_name="comments", verbose_name=_("Book"))
    text = models.TextField(_("Text"))
    created_at = models.DateTimeField(_("Date of creation"), auto_now_add=True)

class UserFavoriteBooks(models.Model):
    book = models.ForeignKey(Book, on_delete=models.CASCADE, related_name="favorited_users")
    user = models.ForeignKey(User, on_delete=models.CASCADE, related_name="favorites")

    class Meta:
        unique_together = ["book", "user"]

class BookScoredUsers(models.Model):
    book = models.ForeignKey(Book, on_delete=models.CASCADE)
    user = models.ForeignKey(User, on_delete=models.CASCADE)
    score = models.FloatField(validators=[MinValueValidator(1), MaxValueValidator(10)])

    class Meta:
        unique_together = ["book", "user"]


from collections import Counter
from contextlib import contextmanager, asynccontextmanager
import logging
import time

logger = logging.getLogger(__name__)

class TimingStats(Counter):

    def __init__(self, verbose: bool = False):
        super().__init__()
        self.verbose = verbose

    @contextmanager
    def scope(self, key, *, verbose=False):
        t1 = time.monotonic()
        yield
        sec = time.monotonic() - t1
        self[key] += sec
        if self.verbose:
            logger.debug(f"{key} took {sec:.3f} seconds")

    @asynccontextmanager
    async def async_scope(self, key, *, verbose=False):
        t1 = time.monotonic()
        yield
        sec = time.monotonic() - t1
        self[key] += sec
        if self.verbose:
            logger.debug(f"{key} took {sec:.3f} seconds")

    def report_strings(self):
        return [f"{key}: {sec:.1f} sec" for key, sec in self.items()]


# TODO: prevent utf-8 encoding errors in CSVs
# TODO: add a progress bar for all timed processes
# TODO: Maintain History of organizations analyzed
# TODO: Show time taken to scrape and analyze (tock - tick)

#Importing Libraries
import contextlib
import csv
import json
import os
import re
import time
import warnings
from platform import platform, system

import matplotlib.pyplot as plt
import requests
import spacy
import torch
import trafilatura
from bs4 import BeautifulSoup
from newsapi import NewsApiClient
from rich import box, print
from rich.align import Align
from rich.console import Console
from rich.layout import Layout
from rich.panel import Panel
from rich.progress import track
from rich.syntax import Syntax
from rich.text import Text
from spacy import displacy
from spacy.lang.en.stop_words import STOP_WORDS
from spacytextblob.spacytextblob import SpacyTextBlob
from transformers import AutoModelForSequenceClassification, AutoTokenizer

warnings.simplefilter(action='ignore', category=FutureWarning)

import pandas as pd

# =========================#
# UTIL FUNCTIONS           #
# =========================#
def parse_text_from_web(webURL: str) -> str:
    """Extracts the text from the main content of the web page. Removes the ads, comments, navigation bar, footer, html tags, etc
    Args:
        webURL (str): URL of the web page
    Returns:
        str: clean text from the web page
    Raises:
        trafilatura.errors.FetchingError: If the URL is invalid or the server is down
    """

    with contextlib.suppress(Exception):
        downloaded = trafilatura.fetch_url(webURL)
        return trafilatura.extract(
            downloaded,
            include_comments=False,
            include_tables=False,
            with_metadata=False,
            include_formatting=True,
            target_language='en',
            include_images=False,
        )

# =========================#
# cleanup FUNCTIONS        #
# =========================#
def cleanup_text(text: str) -> str:
    """Clean up the text by removing special characters, numbers, whitespaces, etc for further processing and to improve the accuracy of the model.
    Args:
        text (str): text to be cleaned up
    Returns:
        str: cleaned up text
    """

    # text = re.sub(r'\d+', '', text)  # remove numbers

    # text = re.sub(r'\s+', ' ', text)  # remove whitespaces
    with contextlib.suppress(Exception):
        # remove special characters except full stop and apostrophe
        text = re.sub(r'[^a-zA-Z0-9\s.]', '', text)

        # text = text.lower()  # convert text to lowercase
        text = text.strip()  # remove leading and trailing whitespaces
        text = text.encode('ascii', 'ignore').decode('ascii')  # remove non-ascii characters

        # split text into words without messing up the punctuation
        text = re.findall(r"[\w']+|[.,!?;]", text)

        text= ' '.join(text)
        return text.replace(' .', '.')

# ========================#
# SCRAPING                #
# ========================#
def scrape_news(organization: str) -> list:
    # sourcery skip: inline-immediately-returned-variable, use-contextlib-suppress

    try:
        # newsAPI
        api_key=os.getenv('NEWSAPI')

        newsapi = NewsApiClient(api_key=api_key)

        # get TOP articles, 1st page, grab 3 articles
        all_articles = newsapi.get_everything(q=organization, from_param='2022-12-20', to='2023-01-12', language='en', sort_by='relevancy', page=1, page_size=10)

        return all_articles

    except Exception as e:
        pass

# ========================#
# WRITE TO CSV            #
# ========================#
def write_to_csv(organization: str, all_articles: dict) -> None:
    with open('CSVs/COMMON.csv', 'w', encoding='utf-8', newline='') as file:
        writer = csv.writer(file)
        writer.writerow(["Article", "Title", "Description",  "URL", "Content", "Published"])
        for idx, article in enumerate(all_articles['articles'], start=1):
            title= article['title'].strip()
            description= article['description'].strip()
            publishedAt= article['publishedAt']
            newsURL= article['url']

            content= parse_text_from_web(newsURL)
            content= cleanup_text(content)

            # download the content from the url
            writer.writerow([idx, article['title'], article['description'], article['url'], content, publishedAt])

            print(f"âœ… [bold green]SUCCESS! Wrote {idx} - [bold blue]{title}[/bold blue] to [gold1]{organization}[/gold1].csv")

        # Adding the parsed content to the CSV
        print(f"[bold green]DONE! WROTE {len(all_articles['articles'])} ARTICLES TO [r]COMMON.csv[/r][/bold green]")

# ========================#
# SENTIMENT scoring       #
# ========================#

#egt the headlines
def get_headline(content, organization):
    r = requests.get(content)
    #parse the text
    soup = BeautifulSoup(r.content, "html.parser")
    if soup.find('h1'):
        headline=soup.find('h1').get_text()
        if len(headline.split())<=2:
            headline="No Headline"

    else:
        headline="No Headline"

    # TODO: HANDLE IMPROVISATION OF HEADERS LATER

    return headline

def sentiment_score_to_summary(sentiment_score: int) -> str:
        """
        Converts the sentiment score to a summary
        Args:
            sentiment_score (int): sentiment score
        Returns:
            str: summary of the sentiment score
        """
        if sentiment_score == 1:
            return "Extremely Negative"
        elif sentiment_score == 2:
            return "Somewhat Negative"
        elif sentiment_score == 3:
            return "Generally Neutral"
        elif sentiment_score == 4:
            return "Somewhat Positive"
        elif sentiment_score == 5:
            return "Extremely Positive"

#calculate the sentiment score
def sentiment_analysis(content: str) -> None:
    """
    Performs sentiment analysis on the text and prints the sentiment score and the summary of the score
    Args:
        content (str): text/url to be analyzed
    """
    tokenizer = AutoTokenizer.from_pretrained(
        "nlptown/bert-base-multilingual-uncased-sentiment")
    model = AutoModelForSequenceClassification.from_pretrained(
        "nlptown/bert-base-multilingual-uncased-sentiment")
    tokens = tokenizer.encode(
        content, return_tensors='pt', truncation=True, padding=True)
    result = model(tokens)
    result.logits
    sentiment_score = int(torch.argmax(result.logits))+1
    return sentiment_score_to_summary(sentiment_score)

# sourcery skip: identity-comprehension
def process_csv(organization):

    with open ('word-store/negative_words.txt', 'r', encoding='utf-8') as file:
        negative_words_list = file.read().splitlines()

    with open ('word-store/bad_words.txt', 'r', encoding='utf-8') as file:
        bad_words = file.read().splitlines()

    with open ('word-store/countries.txt', 'r', encoding='utf-8') as file:
        countries = file.read().splitlines()

    with open('word-store/lawsuits.txt', 'r', encoding='utf-8') as file:
        lawsuits = file.read().splitlines()

    with open('word-store/harassment.txt', 'r', encoding='utf-8') as file:
        harassment = file.read().splitlines()

# ========================#
# Creating Final csv      #
# ========================#
    #definig charset
    with open('CSVs/COMMON-PROCESSED.csv', 'w', encoding='utf-8', newline='') as summary:

        # read first row from Uber.csv
        with open('CSVs/COMMON.csv', 'r', encoding='utf-8') as file:
            try:
                reader = csv.reader(file)
                next(reader)

                # write to csv
                writer = csv.writer(summary)

                # do for every news article
                writer.writerows([["Article", "Headline", "Headline Sentiment", "Offense Rating", "Negative Words", "Offensive Words", "Tags"]])

                print("[bold gold1]===============================[/bold gold1]\n\n")
                for idx, row in enumerate(reader, start=1):
                    url= row[3]
                    raw_text = row[4]

                    # parse_text_from_web(webURL)

                    headline=get_headline(url, organization)
                    headline_sentiment=sentiment_analysis(headline)

                    negative_words=[]
                    offensive_words=[]
                    tags=[]

                    # init ofense rating
                    offense_rating=0

                    # tag as negative
                    if headline_sentiment == "Extremely Negative":
                        offense_rating+=200

                    elif headline_sentiment == "Somewhat Negative":
                        offense_rating+=100

                    nlp_text= nlp(raw_text)

                    # add custom entities
                    for word in nlp_text:
                        # if it is a negative word
                        if word.text.lower() in negative_words_list:
                            offense_rating+=10
                            negative_words.append(word.text)

                        # if it is a highly offensive word
                        elif word.text.lower() in bad_words:
                            offense_rating+=50
                            offensive_words.append(word.text)

                        # if the article is talks about lawsuits
                        if word.text.lower() in lawsuits:
                            offense_rating+=30
                            tags.append("lawsuit")

                        # if the article is about harassment
                        if word.text.lower() in harassment:
                            offense_rating+=50
                            tags.append("harassment")

                        # does article mention a country?
                        if word.text.lower() in countries:
                            tags.append("country")

                        # does article mention a person
                        if word.ent_type_ == "PERSON":
                            tags.append(word)

                    if offense_rating>20:
                        offense_rating-=10

                    # Write each row
                    writer.writerow(
                        [
                            idx,
                            headline,
                            headline_sentiment,
                            offense_rating,
                            list(negative_words),
                            list(offensive_words),
                            list(tags),
                        ]
                    )
                    print(f"Article {idx} written to csv")

                print(f"âœ” [bold u r]\nSUCCESS! Finished processing COMMON-PROCESSED.csv[/bold u r]")

            except Exception as e:
                print(e)
                print(e.__class__)
                print(e.__doc__)
                print(e.__traceback__)

# ========================#
# Display temp output     #
# ========================#

#visualize the text in html
def visualize(organization):
    raw_text = ''
    with open('CSVs/COMMON.csv', 'r', encoding='utf-8') as file:
        reader = csv.reader(file)
        next(reader)

        # do for every news article
        for idx, row in enumerate(reader, start=1):
            raw_text += row[4]

        nlp_text = nlp(raw_text)

        print("\nðŸš€ [bold magenta r]NER COMPLETE, all words tagged...[/bold magenta r]")
        # serve the displacy visualizer
        displacy.serve(nlp_text, style="ent")

# ========================#
# Merging Raw data        #
# ========================#

def merge_csv(csv1, csv2, organization):

    df1 = pd.read_csv(csv1, encoding='unicode_escape')
    df2 = pd.read_csv(csv2, encoding='unicode_escape')
    df = pd.merge(df1, df2, on='Article')
    import random

    num=random.randint(1, 100)

    # # check if COMMON-ANALYSIS exists then copy and rename it to COMMON-ANALYSIS-1
    # if os.path.exists('CSVs/COMMON-ANALYSIS.csv'):
    #     os.rename('CSVs/COMMON-ANALYSIS.csv', f'CSVs/COMMON-ANALYSIS-{num}.csv')

    df.to_csv('CSVs/COMMON-ANALYSIS.csv', index=False)
    print("CSVs merged to COMMON-ANALYSIS.csv")

# ========================#
# cleaing up -2           #
# ========================#
# RUN SAME FUNCTION TWICE
def final_cleanup(organization):

    df = pd.read_csv('CSVs/COMMON-ANALYSIS.csv', encoding='unicode_escape')

    # write - to empty cells in offensive words
    df['Offensive Words'] = df['Offensive Words'].fillna('-')

    # write - to empty cells in negative words
    df['Negative Words'] = df['Negative Words'].fillna('-')

    # write - to empty cells in tags
    df['Tags'] = df['Tags'].fillna('-')

    # clean up tags
    df['Tags'] = df['Tags'].str.replace('[', '').str.replace(']', '').str.replace("'", '')

    # clean up offensive words
    df['Offensive Words'] = df['Offensive Words'].str.replace('[', '').str.replace(']', '').str.replace("'", '')

    # clean up negative words
    df['Negative Words'] = df['Negative Words'].str.replace('[', '').str.replace(']', '').str.replace("'", '')

    df.to_csv('CSVs/COMMON-ANALYSIS.csv', index=False)
#get orgainizations url
def get_sub_url(organization):

    with open ('CSVs/COMMON-ANALYSIS.csv', 'r', encoding='utf-8') as f:
        with open ('CSVs/COMMON-ANALYSIS.csv', 'w', encoding='utf-8') as f2:
            publisher=[]
            reader = csv.reader(f)
            url = [row[4] for row in reader]
            # remove www. and https:// from url
            url = [re.sub(r'www.', '', i) for i in url]
            url = [re.sub(r'https://', '', i) for i in url]
            for x in url:
                name= x.split('.com/')[0]
                publisher.append(name)

            # replace items from publisher where character length is more than 40 with '-'
            publisher = [re.sub(r'.{40,}', '-', i) for i in publisher]
            print(publisher)
    print("CSVs cleaned up to COMMON-ANALYSIS.csv")
# sourcery skip: identity-comprehension
nlp = spacy.load("en_core_web_trf")

# ========================#
# Console Output          #
# ========================#

# no tests for this function as it is not called anywhere in the command directly
def get_terminal_width() -> int:
    """
    Gets the width of the terminal.
    Returns:
        int: width of the terminal.
    """
    try:
        width, _ = os.get_terminal_size()
    except OSError:
        width = 80

    if system().lower() == "windows":
        width -= 1

    return width

def print_banner(console) -> None:
    """
    Prints the banner of the application.
    Args:
        console (Console): Rich console object.
    """

    banner = """
::::    ::::  :::::::::: ::::::::: :::::::::::     :::              :::     ::::    :::     :::     :::     :::   :::  :::::::: ::::::::::: ::::::::
+:+:+: :+:+:+ :+:        :+:    :+:    :+:       :+: :+:          :+: :+:   :+:+:   :+:   :+: :+:   :+:     :+:   :+: :+:    :+:    :+:    :+:    :+:
+:+ +:+:+ +:+ +:+        +:+    +:+    +:+      +:+   +:+        +:+   +:+  :+:+:+  +:+  +:+   +:+  +:+      +:+ +:+  +:+           +:+    +:+
+#+  +:+  +#+ +#++:++#   +#+    +:+    +#+     +#++:++#++:      +#++:++#++: +#+ +:+ +#+ +#++:++#++: +#+       +#++:   +#++:++#++    +#+    +#++:++#++
+#+       +#+ +#+        +#+    +#+    +#+     +#+     +#+      +#+     +#+ +#+  +#+#+# +#+     +#+ +#+        +#+           +#+    +#+           +#+
#+#       #+# #+#        #+#    #+#    #+#     #+#     #+#      #+#     #+# #+#   #+#+# #+#     #+# #+#        #+#    #+#    #+#    #+#    #+#    #+#
###       ### ########## ######### ########### ###     ###      ###     ### ###    #### ###     ### ########## ###     ######## ########### ########
            """
    width = get_terminal_width()
    height = 10
    # defining the panel
    panel = Panel(
        Align(
            Text(banner, style="green"),
            vertical="middle",
            align="center",
        ),
        width=width,
        height=height,
        subtitle="[bold blue]Built for CRIF Hackathon 2023![/bold blue]",
    )
    console.print(panel)

# ========================#
# Call of funtions        #
# ========================#

#start cli
console = Console(record=False, color_system="truecolor")
print_banner(console)

# sourcery skip: inline-immediately-returned-variable

# ========================#
print(Panel.fit("[bold green reverse]ENTER AN ORGANIZATION NAME TO PERFORM MEDIA ANALYSIS ON[/bold green reverse]"))
organization=input()

articles=scrape_news(organization)
write_to_csv(organization, articles)
process_csv(organization)

file1='CSVs/COMMON.csv'
file2='CSVs/COMMON-processed.csv'
merge_csv(file1, file2, organization)

final_cleanup(organization)
final_cleanup(organization)

# get_sub_url(organization)
print(Panel.fit("[bold green reverse]ANALYSIS COMPLETE.[/bold green reverse]\nNow performing Named Entity Recognition on the articles and preparing a visualization."))
visualize(organization)


from ex2_utils import *
import matplotlib.pyplot as plt
from random import randrange
import numpy as np
import cv2

def presentation(plots, titles):
    n = len(plots)

    if n == 1:
        plt.imshow(plots[0], cmap='gray')
        plt.title(titles[0])
        plt.show()
        return

    if n == 2:
        fig, ax = plt.subplots(1, 2, figsize=(12, 8))
    elif n % 2 == 0:
        fig = plt.figure(figsize=(12, 8))
        plt.gray()
        for i in range(n):
            ax = fig.add_subplot(2, 2, i + 1)
            ax.imshow(plots[i])
            ax.title.set_text(titles[i])
        plt.show()
        return
    else:
        fig, ax = plt.subplots(1, n, figsize=(4 * n, 4))

    for i in range(n):
        ax[i].set_title(titles[i])
        ax[i].imshow(plots[i], cmap='gray')
        plt.tight_layout()
    plt.show()

def conv1Demo():
    n = randrange(10)
    Signals, Kernels = list(), list()

    for i in range(n):
        Signals.append(np.random.randint(5, size=10))
        Kernels.append(np.random.randint(5, size=10))

    good_ans = 0
    for i in range(n):
        for j in range(n):
            np_convolution = np.convolve(Signals[i], Kernels[j])
            my_convolution = conv1D(Signals[i], Kernels[j])
            if np_convolution.all() == my_convolution.all():
                good_ans += 1

    if good_ans == len(Signals) * len(Kernels):
        print("conv1Demo: All test are passed!\nGood Job!\n")
    else:
        print("conv1Demo: Some of test aren't passed!\nTry Again!\n")

def conv2Demo():
    img = cv2.imread('pool_balls.jpeg', 0)
    Kernels = [np.array([[-1, 1], [1, 1]], dtype=np.float64),
               np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float64),
               np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float64),
               np.array([[0., 0.25, 0.5, 0.75, 1], [0.2, 0.4, 0.6, 0.8, 1],
                         [1., 1.25, 1.5, 1.75, 2], [1.2, 1.4, 1.6, 1.8, 2]], dtype=np.float64)]

    for i in range(4):
        if Kernels[i].sum() != 0:
            Kernels[i] /= (Kernels[i].sum())

    good_ans = 0
    for kernel in Kernels:
        cv2_convolution = cv2.filter2D(img, -1, kernel, borderType=cv2.BORDER_REPLICATE)
        my_convolution = conv2D(img, kernel)
        if cv2_convolution.all() == my_convolution.all():
            good_ans += 1

    if good_ans == len(Kernels):
        print("conv2Demo: All test are passed!\nGood Job!\n")
    else:
        print("conv1Demo: Some of test aren't passed!\nTry Again!\n")

def derivDemo():
    img = cv2.imread('pool_balls.jpeg', 0)
    direction, magnitude, x_der, y_der = convDerivative(img)

    plots = [direction, magnitude, x_der, y_der]
    titles = ["Direction", "Magnitude", "X Derivative", "Y Derivative"]
    presentation(plots=plots, titles=titles)
    print("derivDemo: Good Job!\n")

def blurDemo():
    img = cv2.imread("coins.jpg", 0)
    kernel_size = 5
    plots = [img, blurImage2(img, kernel_size)]
    titles = ['Image - non blurring', 'CV2 Blur']
    presentation(plots=plots, titles=titles)
    print("blurDemo: Good Job!\n")

def edgeDetectionSobelDemo():
    img = cv2.imread("boxman.jpg", 0)
    opencv_solution, my_solution = edgeDetectionSobel(img, thresh=0.1)
    plots = [img, opencv_solution, my_solution]
    titles = ['Original Image', 'CV2 Sobel', 'My Sobel']
    presentation(plots=plots, titles=titles)
    print("edgeDetectionSobelDemo: Good Job!\n")

def edgeDetectionZeroCrossingLOGDemo():
    img = cv2.imread("boxman.jpg", 0)
    edge_matrix = edgeDetectionZeroCrossingLOG(img)
    presentation(plots=[edge_matrix], titles=["Laplacian of Gaussian\nZero Crossing Edge Detection"])
    print("edgeDetectionZeroCrossingLOGDemo: Good Job!\n")

def edgeDetectionCannyDemo():
    img = cv2.imread("pool_balls.jpeg", 0)
    cv2_canny, my_canny = edgeDetectionCanny(img, 50, 100)
    plots = [img, cv2_canny, my_canny]
    titles = ['Original Image', 'CV2 Canny Edge Detection', 'My Canny Edge Detection']
    presentation(plots=plots, titles=titles)
    print("edgeDetectionCannyDemo: Good Job!\n")

def edgeDemo():
    edgeDetectionSobelDemo()
    edgeDetectionZeroCrossingLOGDemo()
    edgeDetectionCannyDemo()

def houghDemo():
    img = cv2.imread('coins.jpg', 0)
    min_radius, max_radius = 10, 20

    circles = houghCircle(img, min_radius, max_radius)

    fig, ax = plt.subplots()
    ax.imshow(img, cmap='gray')
    for x, y, radius in circles:
        circles_plots = plt.Circle((x, y), radius, color='r', fill=False)
        ax.add_artist(circles_plots)
    plt.title("Circle\nMy houghCircle Implementation")
    plt.show()
    print("houghDemo: Good Job!\n")

def main():
    print("ID: 316451749\nHave Fun! :)\n")
    conv1Demo()
    conv2Demo()
    derivDemo()
    blurDemo()
    edgeDemo()
    houghDemo()

if __name__ == '__main__':
    main()


from flask import request

from werkzeug.exceptions import NotFound, BadRequest, Conflict

from db import db
from managers.brand import BrandManager
from managers.category import CategoryManager
from models import BrandModel, CategoryModel
from models.enums import GenderType
from models.products import ProductsModel, ProductImages, ProductPair

from sqlalchemy.sql.expression import text

from utils.operations import db_add_items, db_delete_items

def check_pair_or_image_product(item, product, item_id, product_id, item_name="item"):

    if not item:
        raise NotFound(f"There is not {item_name} with id: {item_id}")

    if not product:
        raise NotFound(f"There is not product with id: {product_id}")

    if item not in product.pairs and item not in product.images:
        raise BadRequest(
            f"{item_name} with id: {item_id} is not attached to product with id: {product_id}"
        )

class ProductManager:
    @staticmethod
    def create_product(product_data):
        images = []
        for image in product_data["images"]:
            img = ProductImages(img_url=image)
            images.append(img)

        product_pair = []
        for obj in product_data["pairs"]:
            pair = ProductPair(**obj)
            product_pair.append(pair)

        print(product_data["pairs"])
        brand_q = BrandManager.get_by_name_query(product_data["brand_name"])
        category_q = CategoryManager.get_by_title_query(product_data["category_title"])
        brand = brand_q.first()
        category = category_q.first()
        if not brand:
            raise NotFound("There is no brand with that name")
        if not category:
            raise NotFound("There is no category with that name")

        with db.session.no_autoflush:

            product = ProductsModel(
                title=product_data["title"],
                description=product_data["description"],
                price=product_data["price"],
                discount=product_data["discount"],
                gender=GenderType[product_data["gender"]],
            )
            brand.products.append(product)
            category.products.append(product)

            for img in images:
                product.images.append(img)

            for pair in product_pair:
                product.pairs.append(pair)

        db_add_items(product, category, brand)

        return product

    @staticmethod
    def add_image(id, image_data):
        image = ProductImages(img_url=image_data["img_url"], product_id=id)

        db_add_items(image)
        return image

    @staticmethod
    def delete_image(id, image_id):

        image = ProductImages.query.filter_by(id=image_id["id"]).first()
        product = ProductsModel.query.filter(
            ProductsModel.id == id, text("is_deleted is FALSE")
        ).first()

        check_pair_or_image_product(image, product, image_id["id"], id, "images")
        db_delete_items(*image)

        return f"You delete image with id: {image_id['id']} successfully", 202

    @staticmethod
    def edit_image(product_id, images_data):
        images_ids = [id for id in images_data["ids"]]
        new_urls = [url for url in images_data["urls"]]
        product = ProductsModel.query.filter_by(id=product_id).first()

        new_images = [
            ProductImages(product_id=product_id, img_url=url) for url in new_urls
        ]
        old_images = [ProductImages.query.filter_by(id=id).first() for id in images_ids]

        if len(images_ids) != len(new_urls):
            raise BadRequest(
                "You should add same number of new images such as number of deleted one"
            )

        if not product:
            raise NotFound(f"There is not product with id: {product_id}")

        for image in old_images:
            if image not in product.images:
                raise NotFound(
                    f"The id:{id} is not attached to product with id:{product_id}"
                )

        try:
            db_add_items(*new_images)
            db_delete_items(old_images)
        except:
            raise BadRequest("You cannot do that operation")

        return {"message": "You successful edit images"}

    @staticmethod
    def add_pair(id, pair_data):
        product = ProductsModel.query.filter(
            ProductsModel.id == id, text("is_deleted is FALSE")
        ).first()
        is_pair = ProductPair.query.filter_by(
            size=pair_data["size"], color=pair_data["color"], product_id=id
        ).first()
        if is_pair:
            raise Conflict(
                f"Pair with color: {pair_data['color']} and {pair_data['size']} already attached to product with id: {id}"
            )
        if not product:
            raise NotFound("There is no product with that id")
        pair = ProductPair(**pair_data, product_id=id)
        db_add_items(pair)
        return pair

    @staticmethod
    def delete_pair(id, pair_id):
        product = ProductsModel.query.filter(
            ProductsModel.id == id, text("is_deleted is FALSE")
        ).first()
        pair = ProductPair.query.filter_by(id=pair_id["id"]).first()

        check_pair_or_image_product(pair, product, pair_id["id"], id, "pair")

        db_delete_items(pair)

        return f"You delete image with id: {pair_id['id']} successfully", 202

    @staticmethod
    def edit_pair(product_id, pair_id, pair_data):
        product = ProductsModel.query.filter_by(id=product_id).first()
        pair = ProductPair.query.filter_by(id=pair_id).first()

        check_pair_or_image_product(pair, product, pair_id, product_id, "pair")

        # pair.size = pair_data["size"]
        # pair.color = pair_data["color"]
        pair.quantity = pair_data["quantity"]

        db_add_items(pair)

        return pair

    @staticmethod
    def sell_pair(pairs):
        for pair in pairs:
            pair.quantity -= 1

        return pairs

    @staticmethod
    def edit_product_base_info(id_, product_data):
        # product_q = ProductsModel.query.filter(
        #     ProductsModel.id == id_, text("is_deleted is FALSE")
        # )

        product_q = ProductsModel.query.filter_by(id=id_)
        product = product_q.first()

        if not product:
            raise NotFound("This product does not exist.")
        product_q = ProductsModel.query.filter(ProductsModel.id == id_)

        old_brand = product.brand
        old_category = product.category
        new_brand = BrandManager.get_by_name(product_data["brand_name"])
        new_category = CategoryManager.get_by_name(product_data["category_title"])
        if not new_brand:
            raise NotFound("There is no brand with that name")
        if not new_category:
            raise NotFound("There is no category with that name")

        product_data.pop("brand_name")
        product_data.pop("category_title")
        with db.session.no_autoflush:
            print(product_data)
            product_q.update(product_data)
            if not old_brand.name == new_brand.name:
                old_brand.products.remove(product)
                new_brand.products.append(product)
            if not old_category.title == new_category.title:
                old_category.products.remove(product)
                new_category.products.append(product)

        db_add_items(product, new_category, old_category, new_brand, old_brand)

        return product

    @staticmethod
    def get_one(id_, for_admin=False):
        if for_admin:
            product = ProductsModel.query.filter_by(id=id_).first()
        else:
            product = ProductsModel.query.filter(
                ProductsModel.id == id_, text("is_deleted is FALSE")
            ).first()
        if not product:
            raise NotFound("This product does not exist.")
        return product

    @staticmethod
    def get_all(for_admin=False):

        category_title = request.args.get("category")
        brand_name = request.args.get("brand")
        gender = request.args.get("gender")

        category_f = CategoryModel.title == category_title
        brand_f = BrandModel.name == brand_name
        if gender not in GenderType.list() and gender:
            raise NotFound("There is not gender with that name")
        gender_f = ProductsModel.gender == gender

        if not category_title:
            category_f = True
        if not brand_name:
            brand_f = True
        if not gender:
            gender_f = True

        if for_admin:
            products = (
                ProductsModel.query.join(ProductsModel.category)
                .join(ProductsModel.brand)
                .filter(brand_f, category_f, gender_f)
            )
        else:
            products = (
                ProductsModel.query.join(ProductsModel.category)
                .join(ProductsModel.brand)
                .filter(brand_f, text("is_deleted is FALSE"), category_f, gender_f)
            )

        return products.all()

    @staticmethod
    def delete_product(id_):
        product = ProductsModel.query.filter(
            ProductsModel.id == id_, text("is_deleted is FALSE")
        ).first()

        if not product:
            raise NotFound("This product does not exist.")
        product.is_deleted = True
        db_add_items()

        return "Product is deleted", 202


from typing import Optional, Tuple

import numpy as np
import torch
from pytorch_lightning import LightningDataModule
from torch.utils.data import DataLoader, Dataset

from src.datamodules.components.diarization_dataset import (
    DiarizationDataset,
    DiarizationDatasetforInfer,
)

def collate_fn(batch):
    ys, ts, ilens = list(zip(*batch))
    ilens = np.array(ilens)
    ys = np.array(
        [
            np.pad(y, [(0, np.max(ilens) - len(y)), (0, 0)], "constant", constant_values=(-1,))
            for y in ys
        ]
    )
    ts = np.array(
        [
            np.pad(t, [(0, np.max(ilens) - len(t)), (0, 0)], "constant", constant_values=(+1,))
            for t in ts
        ]
    )
    ys = torch.from_numpy(np.array(ys)).to(torch.float32)
    ts = torch.from_numpy(np.array(ts)).to(torch.float32)
    ilens = torch.from_numpy(np.array(ilens)).to(torch.int32)
    return ys, ts, ilens

class DiarizationDataModule(LightningDataModule):
    def __init__(
        self,
        data_dirs: Tuple[str, str, str],
        chunk_size: int = 2000,
        context_size: int = 7,
        frame_size: int = 1024,
        frame_shift: int = 256,
        subsampling: int = 10,
        sample_rate: int = 8000,
        input_transform: str = "logmel23_mn",
        n_speakers: int = None,
        batch_sizes: Tuple[int, int, int] = (64, 64, 1),
        num_workers: int = 0,
    ):
        super().__init__()
        # this line allows to access init params with 'self.hparams' attribute
        self.save_hyperparameters(logger=False)

        self.data_train: Optional[Dataset] = None
        self.data_val: Optional[Dataset] = None
        self.data_test: Optional[Dataset] = None

    def prepare_data(self):
        pass

    def setup(self, stage: Optional[str] = None):
        """Load data. Set variables: `self.data_train`, `self.data_val`, `self.data_test`.

        This method is called by lightning when doing `trainer.fit()` and `trainer.test()`,
        so be careful not to execute the random split twice! The `stage` can be used to
        differentiate whether it's called before trainer.fit()` or `trainer.test()`.
        """
        if not self.data_train and not self.data_val and not self.data_test:
            train_dir, val_dir, test_dir = self.hparams.data_dirs
            self.data_train = DiarizationDataset(
                data_dir=train_dir,
                chunk_size=self.hparams.chunk_size,
                context_size=self.hparams.context_size,
                frame_size=self.hparams.frame_size,
                frame_shift=self.hparams.frame_shift,
                subsampling=self.hparams.subsampling,
                sample_rate=self.hparams.sample_rate,
                input_transform=self.hparams.input_transform,
                n_speakers=self.hparams.n_speakers,
            )
            self.data_val = DiarizationDataset(
                data_dir=val_dir,
                chunk_size=self.hparams.chunk_size,
                context_size=self.hparams.context_size,
                frame_size=self.hparams.frame_size,
                frame_shift=self.hparams.frame_shift,
                subsampling=self.hparams.subsampling,
                sample_rate=self.hparams.sample_rate,
                input_transform=self.hparams.input_transform,
                n_speakers=self.hparams.n_speakers,
            )
            self.data_test = DiarizationDatasetforInfer(
                data_dir=test_dir,
                chunk_size=self.hparams.chunk_size,
                context_size=self.hparams.context_size,
                frame_size=self.hparams.frame_size,
                frame_shift=self.hparams.frame_shift,
                subsampling=self.hparams.subsampling,
                sample_rate=self.hparams.sample_rate,
                input_transform=self.hparams.input_transform,
                n_speakers=self.hparams.n_speakers,
            )

    def train_dataloader(self):
        return DataLoader(
            dataset=self.data_train,
            batch_size=self.hparams.batch_sizes[0],
            num_workers=self.hparams.num_workers,
            shuffle=True,
            collate_fn=collate_fn,
        )

    def val_dataloader(self):
        return DataLoader(
            dataset=self.data_val,
            batch_size=self.hparams.batch_sizes[1],
            num_workers=self.hparams.num_workers,
            shuffle=False,
            collate_fn=collate_fn,
        )

    def test_dataloader(self):
        return DataLoader(
            dataset=self.data_test,
            batch_size=self.hparams.batch_sizes[2],
            num_workers=self.hparams.num_workers,
            shuffle=False,
        )


import math

from src.getTickers import *
from src.importData import *
from backtrader.indicators import ema
import datetime
GOINGDOWN_DAYS = 60

def hasNotIncreaseTooMuch(datahigh,datalow):
    heighest=0
    lowest=10000
    for i in range(-5, 0):
        heighest = max(heighest, datahigh[i])
        lowest = min(lowest, datalow[i])
    return datahigh < datalow*1.3

def todayIsLowest(dataclose):
    lowestClose = 10000
    for i in range(-GOINGDOWN_DAYS, -1):
        lowestClose = min(lowestClose, dataclose[i])
    return dataclose[0] <= lowestClose

def todayIsLowestClose(datalastclose,datalow):
    lowest = 10000
    for i in range(-GOINGDOWN_DAYS, -1):
        lowest = min(lowest, datalow[i])
    return datalastclose <= lowest

def findHighest(dataHighest):
    maxPrice = 0
    for i in range(-len(dataHighest)+1,0):
        maxPrice = max(maxPrice, dataHighest[i])
    return maxPrice

class zhaoMaoPiao(bt.Strategy):
    def log(self, txt, dt=None):
        dt = dt or self.datas[0].datetime.date(0)
        print('%s, %s' % (dt.isoformat(), txt))

    def __init__(self):
        self.ema18 = bt.ind.EMA(self.data, period=18)
        self.ema60 = bt.ind.EMA(self.data, period=60)
        self.dataClose = self.datas[0].close
        self.dataHigh = self.datas[0].high
        self.dataLow = self.datas[0].low

    def next(self):
        isGoingDownLongEnough = len(self) > GOINGDOWN_DAYS
        today = datetime.date(2021, 6, 11)
        curdate = self.datetime.date(ago=0)  # 0 is the default
        if(isGoingDownLongEnough and curdate==today):
            compareData = findHighest(self.dataHigh)
            print(curdate)
            if(self.dataClose[0] < compareData/1.5 and
               todayIsLowest(self.dataClose) and
              self.dataClose[0] < 20):
               if CURRENT_TICKER not in SELECTED_TICKERS:
                   print(CURRENT_TICKER)
                   print(curdate)
                   print(self.dataClose[0])
                   print(compareData)
                   SELECTED_TICKERS.append(CURRENT_TICKER)

        #print('date %s, current price %.2f, previous price %.2f' % (self.datas[0].datetime.datetime(), self.sampleData.close[0], self.sampleData.close[-1]))

tickers = getAllTickers()
for ticker in tickers:
    data0 = getDataFromYahooFinance(ticker)
    cerebro = bt.Cerebro()
    cerebro.addstrategy(zhaoMaoPiao)
    cerebro.adddata(data0)
    # print('----------------------------')
    print('Checking ticker: %s' % ticker)
   # print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue())
    CURRENT_TICKER = ticker
    SELECTED_FLAG = False

    cerebro.run()

print(SELECTED_TICKERS)


import os
import numpy as np
import matplotlib.pyplot as plt
import re
from io import StringIO
from skimage.external.tifffile import imsave
from scipy.interpolate import griddata
from scipy.signal import medfilt

def GetChunkFromTextFile(FileName, StartStr, StopStr, skip_header=0, skip_footer=0, LastHit=True, DataType='array'):
    # DataType means we can extract the chunk and then turn it into:
    # 1) Numpy table 'numpy'
    # 2) return the raw text 'raw'
    DataType = DataType.lower()

    # Read the file.
    try:
        with open(FileName, 'r') as myfile:
            data = myfile.read()
    except:
        print('Failed to open ' + FileName + '.  Skipping.')
        return

    # This regex looks for the data between the start and top strings.
    reout = re.compile('%s(.*?)%s' % (StartStr, StopStr), re.S)
    try:
        # Extract just the data we want.
        if LastHit == False:
            SectionStr = reout.search(data).group(1)
        else:
            SectionStr = reout.findall(data)[-1]
    except:
        # It is possible that the user asked for something that isn't in the file.  If so, just bail.
        return None

    if DataType == 'raw':
        # Now apply skip_header and skip_footer
        SectionData = SectionStr
        SectionData = ''.join(SectionData.splitlines(True)[skip_header:])
        if skip_footer > 0:
            SectionData = ''.join(SectionData.splitlines(True)[:-skip_footer])

    if DataType == 'float':
        SectionData = np.float(SectionStr)

    if DataType == 'array':
        # Convert it into a numpy array.
        SectionData = np.genfromtxt(StringIO(SectionStr), skip_header=skip_header, skip_footer=skip_footer, dtype=None)

    return SectionData

def ReadXSFVolume(FileName, verbose=True, WFOffset=(0,0,0), Cutoff=0.0):
    print(FileName)
    Datagrid = GetChunkFromTextFile(FileName,'BEGIN_DATAGRID_3D_UNKNOWN','END_DATAGRID_3D', DataType='raw')
    lines = Datagrid.splitlines()

    # Line 0 is the 'BEGIN_DATAGRID_3D_UNKNOWN' header.

    # Line 1 is the x, y, z dimensions of the cube in pixels.
    xPixels, yPixels, zPixels = map(int, lines[1].split())
    if verbose==True:
        print(f'Dimension of data cube is ({xPixels}, {yPixels}, {zPixels}) pixels.')

    # Line 2 is the origin.
    xOrigin, yOrigin, zOrigin = map(float, lines[2].split())
    if verbose==True:
        print(f'Origin of data cube is ({xOrigin}, {yOrigin}, {zOrigin}) angstroms.')

    # Lines 3-5 are the metric (or identify matrix if this is a cube with sides of length 1).
    Mstr = ' '.join(lines[3:6])
    M = np.array(list(map(float, Mstr.split()))).reshape(3,3).T
    if verbose==True:
        print('Metric is:')
        print(M)

    # All the rest of the lines are the volume values.
    vstr = ' '.join(lines[6:])
    v = np.array(list(map(float, vstr.split()))).reshape(xPixels, yPixels, zPixels)

    # Next we need a datacube which encompases the entire volume.
    # Make a cartesian grid of width 1 but same number of pixels as the xsf datacube.
    yp,xp,zp = np.meshgrid(np.linspace(0,1,xPixels), np.linspace(0,1,yPixels), np.linspace(0,1,zPixels))

    # Transform those coordinates to the same coordinate system as the xsf datacube.
    C = np.stack([xp,yp,zp], axis=0)
    x,y,z = np.einsum('ij,jklm->iklm', M,C)
    # Shift the origin to zero.
    x += xOrigin + WFOffset[0]
    y += yOrigin + WFOffset[1]
    z += zOrigin + WFOffset[2]

    # The cube x,y,z now represents the coordinates of the actual space that the orbital exists in.
    # we want to resample now using a new larger cube that includes the Wannier function.
    # Find the bounds of the cube.
    xmin = np.min(x); xmax = np.max(x);
    ymin = np.min(y); ymax = np.max(y);
    zmin = np.min(z); zmax = np.max(z);
    # Calculate the pixel sizes from the previous coordinate system.
    dx = np.linalg.norm(M.T[:,0])/xPixels
    dy = np.linalg.norm(M.T[:,1])/yPixels
    dz = np.linalg.norm(M.T[:,2])/zPixels
    # We want our new pixels to be square, so choose the smallest dx,dy,dz.
    dx = dy = dz = np.min([dx,dy,dz])
    # Calculate how many pixels that now is in our new cube.
    nx = np.ceil((xmax-xmin)/dx).astype(int)
    ny = np.ceil((ymax-ymin)/dy).astype(int)
    nz = np.ceil((zmax-zmin)/dz).astype(int)
    Y,X,Z = np.meshgrid(np.linspace(xmin,xmax,nx), np.linspace(ymin,ymax,ny), np.linspace(zmin,zmax,nz))

    # We are going to interpolate using griddata.
    # It expects an (n,D) array of points, whereas we have (x,y,z,D)
    # So collapse the first three dimensions (kind of, ravel all but the last dimension).
    xyz = np.stack([x,y,z],axis=3).reshape(-1,3)
    xyz.shape
    XYZ = np.stack([X,Y,Z],axis=3).reshape(-1,3)
    XYZ.shape
    # And interpolate/extrapolate v->V from xyz->XYZ.
    V = griddata(xyz, v.ravel(), XYZ, method='nearest')
    # Now that we are interpolated, reshape back to (x,y,z,D).
    V = V.reshape(X.shape)
    # Since we use nearest interpolation it comes out a bit noisy.  Fix it.
    V = medfilt(V)

    # # Now eliminate values close to zero.
    # # Vnew = np.zeros(V.shape)
    # # Vnew[V>Cutoff] = V
    # print(Cutoff)
    # Vind1 = V<Cutoff
    # Vind2 = V>(-Cutoff)
    # Vind = Vind1&Vind2
    # print(Vind)
    # V[Vind] = 1e-25

    # Our pixel sizes are different, and medfilt can also change the amplitudes a little.
    # Renormalize so that the total intensity in our new cube is the same as outside the cube.
    V /= np.sum(V)
    # V *= np.sum(v)
    # Note this will fail if the edge of the cube doesn't have zeros or close because the extrapolation
    # will extend that edge value out...

    # Now eliminate values close to zero.
    # Vnew = np.zeros(V.shape)
    # Vnew[V>Cutoff] = V
    print(Cutoff)
    Vind1 = V<Cutoff
    Vind2 = V>(-Cutoff)
    Vind = Vind1&Vind2
    V[Vind] = 1e-9

    return(X, Y, Z, V.astype('float32'))

if __name__ == '__main__':

    X,Y,Z,V = ReadXSFVolume('NiO_00001.xsf', verbose=False) #, Cutoff=0.001) #, WFOffset=(0,0,3.5945353))

    imsave('NiO_00001.tif', V)

    print('Done.')