import sys, os, datetime

import numpy as np
import torch
from torch.utils.tensorboard import SummaryWriter

# add the path to the files from this project
sys.path.append(os.path.dirname(__file__))

import world
import utils
import losses
import mesh_utils
from utils import *
import data_utils
import train_utils
from point_tri_net import PointTriNet_Mesher

### Experiment options

args = world.ArgsObject()
world.args = args

# System parameters
args.experiment_name = "fit_model"
args.run_name = datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%S")
experiments_dir = os.path.join(os.path.dirname(__file__), "..")
args.run_dir = os.path.join(os.path.dirname(__file__), "../training_runs", args.run_name)
args.log_dir = os.path.join(args.run_dir, "logs")
args.dataset_dir = os.path.join(experiments_dir, "..", "data")
args.debug_checks = False
args.disable_cuda = False

# set some defaults
set_args_defaults(args)
world.device = args.device
world.dtype = args.dtype

# Experiment parameters
args.load_weights = None

args.w_watertight = 1.0
args.w_dist_surf_tri = 1.0
args.w_dist_tri_surf = 1.0
args.w_overlap_kernel = 0.01

# Algorithm parameters
args.n_mesh_rounds = 5

# Training parameters
args.epochs = 3                         # Number of epochs to train for
args.lr = 1e-4                          # "Learning rate"
args.lr_decay = .5                      # How much to decrease the learning rate by, applied every decay_step samples (lr = lr * lr_decay)
args.decay_step = 10e99                 # Decay lr after processing this many samples (_not_ batches, samples)
args.batch_size = 1                     # Batch size (note accum parameter below)
args.batch_accum = 8                    # Accumulate over this many batches before stepping gradients
args.eval_every = 2048                  # Evaluate on the validation set after this many samples
args.eval_size = 512                    # Use this much of the validation set to evaluate (if less than full validation set size)

print("Beginning run " + str(args.run_name))

# Ensure the run directory exists
ensure_dir_exists(args.run_dir)

## Initialize the tensorboard writer
world.tb_writer = SummaryWriter(log_dir=args.log_dir)

# Load the dataset
T = []
train_dataset = data_utils.PointSurfaceDataset(dir_with_meshes="data/train/", transforms=T)
val_dataset = data_utils.PointSurfaceDataset(dir_with_meshes="data/val/", transforms=T)

train_loader = torch.utils.data.DataLoader(
    train_dataset,
    batch_size=world.args.batch_size,
    shuffle=True,
    num_workers=4,
    pin_memory=True,
    drop_last=True,
    # collate_fn=train_dataset.collate_fn
)
val_loader = torch.utils.data.DataLoader(
    val_dataset,
    batch_size=world.args.batch_size,
    shuffle=True,
    num_workers=4,
    pin_memory=True,
    drop_last=True,
    # collate_fn=val_dataset.collate_fn
)

# Construct a model
model = PointTriNet_Mesher()

if args.load_weights:
    weights_path = os.path.join(world.args.load_weights,"model_state_dict.pth")
    model.load_state_dict(torch.load(weights_path))

# Calls the model on a batch
def call_model_fn(model, batch, trainer=None):
    points = batch['vert_pos'].to(world.device)
    candidate_triangles, candidate_probs, proposal_triangles, proposal_probs = model.predict_mesh(points, n_rounds=args.n_mesh_rounds, sample_last=True)
    return {
            "candidates" : candidate_triangles, "probs": candidate_probs,
            "proposals" : proposal_triangles, "proposal_probs" : proposal_probs
           }

# Construct a loss
def loss_fn(batch, model_outputs, viz_extra=False, trainer=None):
    B = batch['vert_pos'].shape[0]

    # Data from the sample
    vert_pos_batch = batch['vert_pos'].to(world.device)
    surf_pos_batch = batch['surf_pos'].to(world.device)
    surf_normal_batch = batch['surf_normal'].to(world.device)

    # Outputs from model
    all_candidates = model_outputs['candidates']
    all_candidate_probs = model_outputs['probs']
    all_proposals = model_outputs['proposals']
    all_proposal_probs = model_outputs['proposal_probs']

    # Accumulate loss
    need_grad = all_candidate_probs.requires_grad
    total_loss = torch.tensor(0.0, dtype=vert_pos_batch.dtype, device=vert_pos_batch.device, requires_grad=need_grad)

    # Evaluate loss one batch entry at a time
    for b in range(B):

        vert_pos = vert_pos_batch[b, :]
        candidates = all_candidates[b,:,:]
        candidate_probs = all_candidate_probs[b, :]
        proposals = all_proposals[b,:,:]
        proposal_probs = all_proposal_probs[b, :]

        surf_pos = surf_pos_batch[b, :]
        surf_normal = surf_normal_batch[b, :]

        # Add all the terms
        loss_terms = losses.build_loss(args, vert_pos, candidates, candidate_probs, surf_pos=surf_pos, surf_normal=surf_normal, n_sample=1000)

        loss_terms["proposal_match"] = losses.match_predictions(
            candidates, candidate_probs.detach(),
            proposals, proposal_probs)

        this_loss = torch.tensor(0.0, dtype=vert_pos_batch.dtype, device=vert_pos_batch.device, requires_grad=need_grad)
        for t in loss_terms:
            this_loss = this_loss + loss_terms[t]

        # Log some stats
        if trainer is not None:
            if trainer.training:
                prefix = "train_"
                it = trainer.curr_iter + b
            else:
                prefix = "val_"
                it = trainer.eval_iter + b

            # log less
            if it % 10 == 0:

                for t in loss_terms:
                    world.tb_writer.add_scalar(prefix + t, loss_terms[t].item(), it)

                world.tb_writer.add_scalar(prefix + "sample loss", this_loss.item(), it)

                if it % 1000 == 0:
                    world.tb_writer.add_histogram(prefix + 'triangle_probs', candidate_probs.detach(), it)
                    world.tb_writer.add_histogram(prefix + 'triangle_proposal_probs', proposal_probs.detach(), it)

                world.tb_writer.add_scalar(prefix + "prob mean", torch.mean(candidate_probs).item(), it)
                world.tb_writer.add_scalar(prefix + "prob stddev", torch.std(candidate_probs).item(), it)

                if not trainer.training:
                    trainer.add_eval_stat_entry("prob mean", torch.mean(candidate_probs).item())
                    trainer.add_eval_stat_entry("prob std", torch.std(candidate_probs).item())
                    for t in loss_terms:
                        trainer.add_eval_stat_entry(t, loss_terms[t].item())

        total_loss = total_loss + this_loss

    return total_loss / B

# Train
with torch.autograd.set_detect_anomaly(world.debug_checks):

    trainer = train_utils.MyTrainer(
        args=world.args,
        model=model,
        call_model_fn=call_model_fn,
        loss_fn=loss_fn,
        train_loader=train_loader,
        val_loader=val_loader,
    )

    trainer.train()


#!/usr/bin/env python
# coding: utf-8

# In[1]:

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense,Dropout,BatchNormalization
from keras.callbacks import EarlyStopping,ModelCheckpoint
import keras
import pickle
import numpy as np
import tensorflow as tf
import glob
from tensorflow.keras import layers
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from pickle import dump
import pandas as pd
from collections import Counter

# ETL Standarization

def Scaler(df,direction):
    # Mediante esta funcion creamos y transfromamos los datos mediante un normalizador
# Standardizing the features
    scaler=StandardScaler()
    final_data = scaler.fit_transform(df)
# Download scaler
    dump(scaler, open(direction+'model_scaler.pkl', 'wb'))
    return final_data

# ETL to sequences

# Create model

def model_autoencoder(data,dim_1,dim_2,dim_3,dp):
    model = Sequential()
    model.add(Dense(dim_1, input_dim=data.shape[1], activation='relu'))
    model.add(BatchNormalization())
    model.add(Dense(dim_2, activation='relu'))
    model.add(BatchNormalization())
    model.add(Dense(dim_3, activation='relu'))
    model.add(BatchNormalization())
    model.add(Dense(dim_3, activation='relu'))
    model.add(BatchNormalization())
    model.add(Dense(dim_2, activation='relu'))
    model.add(BatchNormalization())
    model.add(Dense(dim_1, activation='relu'))
    model.add(Dense(data.shape[1]))
    model.compile(optimizer='adam', loss='mse')
    return model

    # Data, es data en forma de create_sequences
    # time_steps, es la cantidad de datos de sequencia que se quiere meter
    # dim out & dim in, dimensionalidad  de la red neuronalde lstm
    # dp is drop_out

# Training

def training(data,val,model,patience,direccion,bs,n_epochs,verbose):
    # data, model
    # patience, cantidad de epochs que no avanza para meteler checkpoints
    # bs es la cantidad de batch & n_epochs, es la cantidad maxima de epcohs que se pueden dar

    es = EarlyStopping(monitor='val_loss', mode='min', verbose=verbose, patience=patience)
    mc = ModelCheckpoint(direccion+'model_autoencoder.h5', monitor='val_loss', mode='min', verbose=verbose, save_best_only=True)
    model.fit(data,data,batch_size=bs,epochs=n_epochs,verbose=verbose,callbacks=[es,mc],validation_data=(val,val))

# Calculate P_Value
def P_value_calculate(model_dir,data_array,porcentaje_max,direction):
	model=keras.models.load_model(model_dir)
	data_array_pred=model.predict(data_array)
	resta=np.abs(data_array_pred - data_array)
	resta_global_total=resta.mean(axis=1)
	value_limited=sorted(resta_global_total,reverse=True)[int(len(resta_global_total)*porcentaje_max)]
	np.save(direction+'p_value.npy', np.array(value_limited))
	return resta_global_total

def Export_results_xlsx(resta_global_total,value_limited,data_select_dataframe,direccion):

# Calculate id (del array) out/in
	resta_global_binary=[]
	for r in resta_global_total:
    		if r>value_limited:
        		resta_global_binary.append(1)
    		else:
        		resta_global_binary.append(0)

	Final=pd.DataFrame(resta_global_binary)
	Final.columns=['Out/In']
	result = pd.merge(data_select_dataframe, Final, how="left",left_index=True, right_index=True).fillna(0)

	result.to_excel(direccion+"_resultados.xlsx")
	return result

def graphs(direccion,result):
	for prop in result.columns[1:-1]:
    		fig,ax=plt.subplots(figsize=(20, 16))
    		plt.plot(result['index'],result[prop],marker="o",label=prop)
    		plt.plot(result[result['Out/In']==1]['index'],result[result['Out/In']==1][prop],marker="o",color="red",linestyle='')
    		plt.title(prop, fontdict=None, loc='center', pad=None,fontsize=30)
    		plt.savefig(direccion+'Grap_'+prop+'.png', bbox_inches='tight', pad_inches=0.0)


#!/usr/bin/env python
# coding: utf-8

# # Creación de Dataset de Municipios de Colombia

# #### Tarea #1: Obtener la información para un municipio

# In[1]:

# Importar librerías principales

import requests
from bs4 import BeautifulSoup as bs
import pandas as pd
import numpy as np
import pickle

# In[3]:

# Cargar el contenido de la pagina web (Recurso: Wikipedia)
w = requests.get("https://es.wikipedia.org/wiki/Bogot%C3%A1")

# Convertir a un objeto "BeautifulSoup"
bsobj = bs(w.content)

# Imprimir (Revisar estructura del código html)
print(bsobj.prettify())

# In[4]:

# Revisar la información de la "tabla de datos" usando una clase específica

info_caja = bsobj.find(class_="infobox geography vcard")
info_filas = info_caja.find_all("tr")
for fila in info_filas:
    print(fila.prettify())

# In[5]:

# Función para obtener los valores dados diferentes casos, además, se reemplazan caracteres extraños (unicode).
def obtener_valores(fila_data):
    if fila_data.find("div") and fila_data.find("br"):
        return [texto for texto in fila_data.stripped_strings]
    else:
        return fila_data.get_text(" ").replace("\xa0", " ").replace("\u200b", "").strip()

# Crear función para eliminar etiquetas [1] (span) solo en el caso de que no tengan información relevante
def limpiar_etiquetas(bs_objeto):
    if bs_objeto.find_all("sup", {'class': 'reference separada'}):
        for etiqueta in bs_objeto.find_all("sup"):
            etiqueta.decompose()
    pass

limpiar_etiquetas(bsobj)

municipios_info = {}

# Crear una lista de títulos para tomar el indice desde el cuál empezar a obtener los datos
titulos = [fila.get('title') for fila in info_caja.find_all("a") if fila.get('title') is not None]
contador = 0
for indice, fila in enumerate(info_filas):
    if indice == 0 and fila.find("th"):
        municipios_info['municipio'] = fila.find("th").get_text(" ", strip=True)
    # En caso de que el tag "th" no contenga información o no exista, tomar el nombre del municipio del encabezado (tag "h1")
    elif indice == 0 and not fila.find("th"):
        municipios_info['municipio'] = bsobj.find("h1").get_text(" ", strip=True)
    elif indice < titulos.index('Coordenadas geográficas'): # Imágenes y otros datos irrelevantes no serán tenidos en cuenta
        continue
    else: # Las filas de tabla (tr) que no contengan elementos "th" & "td" serán evitadas
        try:
            # Se obtiene las llaves y se reemplazan caracteres extraños.
            # Se anexa un número a aquellas llaves que posean el mismo nombre "* Total" para evitar errores en los datos
            if fila.find("th").get_text(" ", strip=True) == "• Total":
                llaves = fila.find("th").get_text(" ", strip=True).replace("•", "").strip() + f"_{contador}"
                valores = obtener_valores(fila.find("td"))
                municipios_info[llaves] = valores
                contador += 1
            else:
                llaves = fila.find("th").get_text(" ", strip=True).replace("•", "").replace("\u200b", "").strip()
                valores = obtener_valores(fila.find("td"))
                municipios_info[llaves] = valores
        except AttributeError as e:
            pass

municipios_info

# #### Tarea #2: Obtener la información para todos los municipios de Colombia

# In[24]:

# Cargar el contenido de la página (Lista de municipios de Colombia por población)
url = requests.get("https://es.wikipedia.org/wiki/Anexo:Municipios_de_Colombia_por_poblaci%C3%B3n")

# Convertir a un objeto BS
bso = bs(url.content)

# Imprimir HTML
print(bso.prettify())

# In[25]:

# Extraer las etiquetas "href" para la clase especificada
municipios = bso.select(".wikitable.sortable td")
rutas = [m.a['href'] for m in municipios if m.a and m.get('align') is None]
print(rutas)

# In[26]:

# Extraer las rutas de cada municipio y formar junto al dominio, los enlaces completos.

dominio = "https://es.wikipedia.org"
enlaces = []
for ruta in rutas:
    enlaces.append(dominio + ruta)

# In[27]:

# En algunos casos, las rutas estaban conectadas a municipalidades de otros paises. Se cambian por los enlaces correctos
for n, i in enumerate(enlaces):
    if i == 'https://es.wikipedia.org/wiki/Aquitania':
        enlaces[n] = 'https://es.wikipedia.org/wiki/Aquitania_(Boyac%C3%A1)'
    elif i == 'https://es.wikipedia.org/wiki/Saman%C3%A1':
        enlaces[n] = 'https://es.wikipedia.org/wiki/Saman%C3%A1_(Caldas)'
    elif i == 'https://es.wikipedia.org/wiki/Roncesvalles':
        enlaces[n] = 'https://es.wikipedia.org/wiki/Roncesvalles_(Tolima)'

# In[28]:

# Crear función para obtener los valores dependiendo de la etiqueta, además, se eliminan caracteres extraños.
def obtener_valores(fila_data):
    if fila_data.find("div") and fila_data.find("br"):
        return [texto for texto in fila_data.stripped_strings]
    else:
        return fila_data.get_text(" ").replace("\xa0", " ").replace("\u200b", "").strip()

# Crear función para eliminar etiquetas [1] (span) solo en el caso de que no tengan información relevante
def limpiar_etiquetas(bs_objeto):
    if bs_objeto.find_all("sup", {'class': 'reference separada'}):
        for etiqueta in bs_objeto.find_all("sup"):
            etiqueta.decompose()
    pass

# Crear función para obtener la información de un municipio dada su dirección url.
def obtener_info(url):

    w = requests.get(url)

    bsobj = bs(w.content)
    info_caja = bsobj.find(class_="infobox geography vcard")
    info_filas = info_caja.find_all("tr")

    limpiar_etiquetas(bsobj)

    municipios_info = {}

    # Crear una lista de títulos para tomar el indice desde el cuál empezar a obtener los datos
    titulos = [fila.get('title') for fila in info_caja.find_all("a") if fila.get('title') is not None]
    contador = 0
    for indice, fila in enumerate(info_filas):
        if indice == 0 and fila.find("th"):
            municipios_info['municipio'] = fila.find("th").get_text(" ", strip=True)
        # En el caso de que no exista la etiqueta "th", será necesario tomar el nombre de la cabecera "h1" del código HTML.
        elif indice == 0 and not fila.find("th"):
            municipios_info['municipio'] = bsobj.find("h1").get_text(" ", strip=True)
        elif indice < titulos.index('Coordenadas geográficas'): # Imágenes y otros datos irrelevantes no serán tenidos en cuenta
            continue
        # Manejo de excepciones y reemplazo de strings en la obtención de los datos
        else:
            try:# Caso especial: En el caso de que una dos etiquetas "th" contengan el mismo texto, se le agrega un caracter
                # adicional al texto para diferenciarlos, y así evitar errores en la conexión con los "td" relacionados.
                if fila.find("th").get_text(" ", strip=True) == "• Total":
                    llaves = fila.find("th").get_text(" ", strip=True).replace("•", "").strip() + f"_{contador}"
                    valores = obtener_valores(fila.find("td"))
                    municipios_info[llaves] = valores
                    contador += 1
                # Evitar tomar los datos de IDH y Población Metropolitana, debido a que solo se encuentran contabilizados en pocos municipios (Ciudades Capitales)
                elif 'IDH' in fila.find("th").get_text(" ", strip=True).upper() or 'METROPOLITANA' in fila.find("th").get_text(" ", strip=True).upper():
                    continue
                else:
                    llaves = fila.find("th").get_text(" ", strip=True).replace("•", "").replace("\u200b", "").strip()
                    valores = obtener_valores(fila.find("td"))
                    municipios_info[llaves] = valores
            except AttributeError as e:
                pass

    return municipios_info

# In[11]:

# Probar la función "obtener_info()" para solo un municipio. Verificar que funcione correctamente.
obtener_info("https://es.wikipedia.org/wiki/Medell%C3%ADn")

# In[29]:

# Se inicia el proceso de extracción de datos para todos los municipios (Guardar todos los diccionarios en una lista)

municipios = []
for enlace in enlaces:
    try:
        municipios.append(obtener_info(enlace))
    except Exception as e:
        print(enlace)
        print(e)

# In[30]:

len(municipios)
# Se logra obtener los datos de 1118 municipios de 1122 existentes.
# Para el caso de los municipios de "Isla de San Andrés" e "Isla de Providencia" las tablas no soportan el método "find_all"
# En los otros dos casos, no existe una tabla para la obtención de los datos.

# In[31]:

# Definir una función para guardar el archivo .json y otra para cargar el archivo .json

import json

def guardar_datos(titulo, datos):
    with open(titulo, 'w', encoding='utf-8') as a:
        json.dump(datos, a, ensure_ascii=False, indent=2)

def cargar_datos(titulo):
    with open(titulo, encoding="utf-8") as a:
        return json.load(a)

# In[32]:

# Guardar resultados obtenidos hasta el momento.

guardar_datos("municipios_col.json", municipios)

# #### Tarea #3: Limpieza de Datos

# In[16]:

# Cargar datos

municipios_col = cargar_datos("municipios_col.json")

# In[17]:

# Imprimir todas las distintas llaves de la lista de dictionarios

all_keys = set().union(*(d.keys() for d in municipios_col))
all_keys

# Se identifican diversas llaves que no contienen información relevante, otras que contienen valores faltantes excesivos,
# algunas difieren en el nombre pero guardan valores similares.

# In[144]:

# Para este ejercicio, se trabaja con la información bajo un DataFrame (para facilitar algunas operaciones de limpieza)

municipios_df = pd.DataFrame(municipios_col)

# In[73]:

# Aquitania Boyacá (Cambiar link) https://es.wikipedia.org/wiki/Aquitania_(Boyac%C3%A1)
# Samaná Caldas (Cambiar link) https://es.wikipedia.org/wiki/Saman%C3%A1_(Caldas)
# Roncesvalles Tolima (Cambiar link) https://es.wikipedia.org/wiki/Roncesvalles_(Tolima)
# Eliminar key "Aeropuerto"
# Eliminar key "Autoridad tradicional Ne'jwesx 2019 - 2021"
# Eliminar "Flor", "Arbol", "Ave", "Cabecera", "Cabecera municipal", "Capital", "Barrios", "Veredas", "Caseríos"
# Eliminar "Centros Poblados", "Hermanada con", "Idioma oficial", "Moneda", "Máxima", "Mínima", "Matrícula"
# Unir Alcalde/Corregidor
# Unir localidades, comunas, barrios
# Eliminar "Otras S(s)uperficies" 'PIB (nominal)','PIB per cápita','Partidos gobernantes'
# Eliminar 'Presidente regional','Presupuesto anual', 'Temperatura', 'Temperatura promedio', 'Total_2'
# Ajustar verdadero nombre '[[Departamentos de \nColombia|Departamento]]','[xau[Subregiones de Antioquia'

# In[145]:

municipios_df.head()

# ##### ** Eliminar columnas con información irrelevante, incompleta o nula **

# In[146]:

columnas_erradas = ["Aeropuerto", "Flor", "Arbol", "Árbol", "Ave", "Cabecera", "Cabecera municipal",
        "Capital", "Barrios", "Veredas", "Caseríos", "Centros Poblados", "Nombre", "Fiestas mayores",
        "Hermanada con","Idioma oficial","Moneda","Máxima","Mínima","Matrícula","Patrono(a)","Localidades",
        "Otras Superficies:","PIB (nominal)","PIB per cápita","Partidos gobernantes","Corregimientos",
        "Otras superficies:", "Presidente regional","Presupuesto anual", "Temperatura", "Creación",
        "Temperatura promedio", "Total_2", "Eventos históricos","Superficie","Altitud", "Comunas",
        "Curso de agua","Código Dane (Divipola)","Código Dian","Correspondencia actual","Subdivisiones",
        "Autoridad tradicional Ne'jwesx 2019 - 2021", "Otros idiomas", "", "Prefijo telefónico",
        "Región", "Provincia", "Zona"]

for col in columnas_erradas:
    if col in municipios_df.columns:
        municipios_df = municipios_df.drop(col, 1)
    continue

#municipio['Superficie'] = municipio.pop('Total_0')
#municipio['Altitud Media'] = municipio.pop('Media')
# Unir Región, Subregión y Zona

# In[147]:

# Las columnas 'Total_0', Total_1', 'Urbana' y 'Media' contiene la información correspondiente
# a 'Superficie', 'Población', 'Pobl. Urbana' y 'Altitud Media' respectivamente, por lo tanto,
# cambiamos el nombre de las columnas.

municipios_df = municipios_df.rename(columns={'Total_0': 'Superficie', 'Total_1': 'Población Total',
                                              'Media': 'Altitud Media', 'Urbana': 'Pobl. Urbana'})

# ##### ** Unir columnas con igual información pero diferente nombre **

# In[149]:

# Reducir a una sola columna aquellas que contienen la misma información pero difieren en su encabezado

from re import search

columnas = {'lcald': 'Alcalde', 'epartamento': 'Departamento', 'unicipio': 'Municipio',
           'oblación': 'Población Total', 'Subregi': 'Subregión', 'Código': 'Código postal'}

for col in municipios_df.columns:
    for key, value in columnas.items():
        if search(key, col) and col != value:
            municipios_df[value].fillna(municipios_df[col], inplace=True)
            municipios_df = municipios_df.drop(col, 1)
        continue

municipios_df = municipios_df.drop("Subregión", 1)

# In[151]:

# Unir las columnas Alcalde/Corregidor

municipios_df['Alcalde'].fillna(municipios_df['Corregidor'], inplace=True)
municipios_df = municipios_df.drop('Corregidor', 1)

# ##### Extraer y añadir la información de códigos postales para todos los municipios

# In[153]:

# Extraer el contenido de la página y crear el objeto BS (Departamentos)

url = requests.get("https://codigo-postal.co/colombia/")
soup = bs(url.content)
print(soup.prettify())

# In[170]:

# Identificar todas las etiquetas "li" dentro de contenido de la clase especificada

info_box = soup.find(class_="column-list")
info_rows = info_box.find_all("li")
for row in info_rows:
    print(row.prettify())

# In[155]:

# Extraer y guardar los enlaces de cada departamento

dptos = []

for row in info_rows:
    dptos.append(row.a['href'])

dptos

# In[194]:

# Se define la función "get_urls" la cual accede al enlace del departamento, extrae y guarda los enlaces de todos sus municipios

def get_urls(dpto, mpios):
    url = requests.get(dpto)
    soup = bs(url.content)

    info_box = soup.find(class_="col-md-8")
    info_rows = info_box.find_all("ul")

    info_lists = []
    for row in info_rows:
        info_lists.append(row.find_all("li"))

    for info_list in info_lists:
        for renglon in info_list:
            mpios.append(renglon.a['href'])

    return mpios

# Extraer los enlaces de todos los municipios de cada departamento. Guardar en una misma lista todos los enlaces.
mpios = []
for dpto in dptos:
    get_urls(dpto, mpios)

# In[197]:

# En este caso, se procede a crear una nueva columna que contenga los nombres de cada municipio pero sin tildes (´) en caso...
# ... de que existan. Esto para facilitar el proceso de asignación del código postal para su correspondiente. Mas adelante...

import re
from unicodedata import normalize

municipios_df['New_municipios'] = ""

series = municipios_df['Municipio']
for index, value in series.items():
    municipios_df['New_municipios'][index] = normalize('NFC', re.sub(
        r"([^n\u0300-\u036f]|n(?!\u0303(?![\u0300-\u036f])))[\u0300-\u036f]+", r"\1",
        normalize( "NFD", value), 0, re.I))

# In[198]:

# Se asigna la nueva columna como indice del DataFrame
municipios_df = municipios_df.set_index('New_municipios')

# In[199]:

municipios_df.head()

# In[204]:

# La función "get_codes" accede al enlace del municipio, extrae y guarda el nombre del municipio y su código postal.

def get_codes(mpio, postales):
    try:
        url = requests.get(mpio)
        soup = bs(url.content)

        info_box_head = soup.find(class_="breadcrumb")
        info_rows_head = info_box_head.find_all("li")

        llave = info_rows_head[3].find("span").get_text(" ", strip=True)

        info_box_body = soup.find(class_="table-responsive")
        info_rows_body = info_box_body.find_all("tr")

        valor = info_rows_body[1].find("a").get_text(" ", strip=True)

        postales[llave] = valor

        return postales

    except AttributeError as e:
        print(f"¡{llave} no tiene código postal asignado!")
        pass

# Extraer el código postal para todos los municipio. Guardar la información en un diccionario.
postales = {}
for mpio in mpios:
    get_codes(mpio, postales)

# In[206]:

# La razón por la que se creó una nueva columna con los nombres de cada municipio sin tilde es porque es necesario asignarlos...
# ... usando el nombre de la llave, cuyos nombres no poseen tildes en los casos en los que debería tenerla. Por lo tanto, ...
# ... se utilizan los nuevos indices para

municipios_df = municipios_df.drop("Código postal", 1)

municipios_df['Código Postal'] = 0
for key, value in postales.items():
    for index, val in municipios_df['Clima'].items():
        if search(key, index):
            municipios_df['Código Postal'][index] = value

# In[210]:

# Se elimina la columna indice que fue tomada solo como referencia
municipios_df.reset_index(drop=True, inplace=True)

# In[217]:

municipios_df.head()

# ##### ** Convertir fechas en string a datetime **

# In[370]:

# Extraer la fecha de cada string de la columna "Fundación" ("5 de junio de 1935" --> "5/junio/1935")

for idx in municipios_df.index:
    try:
        if len(municipios_df['Fundación'][idx].split()) > 4:
            municipios_df['Fundación'][idx] = ' '.join(municipios_df['Fundación'][idx].split()[:5]).replace(' de', '').replace(' ', '/')
        continue
    except Exception as e:
        print(f"{idx}: {e}")

# In[371]:

# Extraer la fecha de cada string de la columna "Erección" ("5 de junio de 1935" --> "5/junio/1935")

for idx in municipios_df.index:
    try:
        if len(municipios_df['Erección'][idx].split()) > 4:
            municipios_df['Erección'][idx] = ' '.join(municipios_df['Erección'][idx].split()[:5]).replace(' de', '').replace(' ', '/')
        continue
    except Exception as e:
        print(f"{idx}: {e}")

# In[372]:

# Convertir las fechas de las columnas "Fundación" y "Erección" de formato string a datetime

from datetime import datetime
import locale # Usar locale para configurar la lectura de las fechas en idioma español.

locale.setlocale(locale.LC_TIME, 'es_ES.UTF-8')

columnas = ['Fundación', 'Erección']

for col in columnas:
    for idx in municipios_df.index:
        try:
            municipios_df[col][idx] = datetime.strptime(municipios_df[col][idx], '%d/%B/%Y')
        except Exception as e:
            print(f"{idx}: {e}")

# In[373]:

# Para aquellas fechas incompletas extraer, por lo menos, el año.

from re import search

columnas = ['Fundación', 'Erección']

for col in columnas:
    for idx in municipios_df.index:
        try:
            if not isinstance(municipios_df[col][idx], datetime):
                municipios_df[col][idx] = search('\d{4}', municipios_df[col][idx]).group(0)
        except Exception as e:
            print(f"{idx}: {e}")

# ##### ** Conversión de datos númericos, modificación de strings y  manejo de valores faltantes (NaN) **

# In[375]:

# Convertir valores de la columna superficie a flotantes, creando una nueva columna que muestre la unidad de medida en (km²)

municipios_df['Superficie (km²)'] = np.NaN

for idx in municipios_df.index:
    try:
        municipios_df['Superficie (km²)'][idx] = float(municipios_df['Superficie'][idx].split()[0])
    except ValueError:
        try:
            municipios_df['Superficie (km²)'][idx] = float(municipios_df['Superficie'][idx].split()[0].replace(',', '.'))
        except Exception as e:
            print(f"{idx}: {e}")

# Eliminar columna 'Superficie'
municipios_df = municipios_df.drop("Superficie", 1)

# In[376]:

# Convertir valores de la columna 'Altitud Media' a enteros, creando una nueva columna que muestre la unidad de medida en (msnm)

municipios_df['Altitud Media (m.s.n.m)'] = np.NaN

for idx in municipios_df.index:
    try:
        municipios_df['Altitud Media (m.s.n.m)'][idx] = float(municipios_df['Altitud Media'][idx].split()[0].replace(',', '.'))
    except Exception as e:
        print(f"{idx}: {e}")

# Eliminar columna 'Altitud Media'
municipios_df = municipios_df.drop("Altitud Media", 1)

# In[377]:

# Convertir valores de columna 'Densidad' a flotantes, creando una nueva columna que muestre la unidad de medida en (hab/km²)

municipios_df['Densidad (hab/km²)'] = np.NaN

for idx in municipios_df.index:
    try:
        municipios_df['Densidad (hab/km²)'][idx] = float(municipios_df['Densidad'][idx].split()[0])
    except (ValueError, AttributeError):
        try:
            municipios_df['Densidad (hab/km²)'][idx] = float(municipios_df['Densidad'][idx].split()[0].replace(',', '.'))
        except Exception as e:
            print(f"{idx}: {e}")

# Eliminar columna 'Densidad'
municipios_df = municipios_df.drop("Densidad", 1)

# In[397]:

# Convertir los valores de las columnas 'Población Total' y 'Pobl. Urbana' de string a enteros

columnas = {'Población Total': 'Población Total (hab)', 'Pobl. Urbana': 'Pobl. Urbana (hab)'}

municipios_df['Pobl. Urbana (hab)'] = np.NaN
municipios_df['Población Total (hab)'] = np.NaN

for key, value in columnas.items():
    for idx in municipios_df.index:
        try:
            municipios_df[value][idx] = int(municipios_df[key][idx].split(' hab.')[0].replace(' ', '').replace(',', ''))
        except Exception as e:
            print(f"{idx}: {e}")

    # Eliminar la columna correspondiente (anterior)
    municipios_df = municipios_df.drop(key, 1)

# In[432]:

# Acortar el tamaño de las coordenadas, mostrando solo la latitud y la longitud.

for idx in municipios_df.index:
    try:
        municipios_df['Coordenadas'][idx] = municipios_df['Coordenadas'][idx].split('/')[0].replace('\ufeff', '').strip()
    except Exception as e:
        print(f"{idx}: {e}")

# In[431]:

# Extraer solo el nombre del alcalde de cada municipio (originalmente aparece el año de posesión y el partido político)

for idx in municipios_df.index:
    try:
        municipios_df['Alcalde'][idx] = municipios_df['Alcalde'][idx].split('(')[0].strip()
    except Exception as e:
        print(f"{idx}: {e}")

# In[418]:

# En el caso de las columnas de huso horario, país y entidad los valores son iguales para todos los indices, es decir,
# Uso horario: UTC -5, Entidad: Municipio y País: Colombia. Se usa esta información para rellenar los valores nulos.

municipios_df['Huso horario'].fillna("UTC -5", inplace=True)
municipios_df['Entidad'].fillna("Municipio", inplace=True)
municipios_df['País'].fillna("Colombia", inplace=True)

# In[433]:

# Cambiar el orden de las columnas dependiendo de la relevancia de la información contenida.

municipios_df = municipios_df[['Municipio', 'Departamento', 'Entidad', 'País', 'Fundación', 'Erección', 'Superficie (km²)',
                              'Altitud Media (m.s.n.m)', 'Población Total (hab)', 'Pobl. Urbana (hab)', 'Densidad (hab/km²)',
                              'Coordenadas', 'Clima', 'Gentilicio', 'Alcalde', 'Código Postal', 'Huso horario']]

# ##### ** ¡Trabajo finalizado! Ahora, guardar los datos en formato CSV **

# In[21]:

# Guardar datos en formato CSV

municipios_df.to_csv("municipios_colombia_df.csv")


import uproot
import os
import pandas as pd
import numpy as np
import awkward
from tqdm import tqdm

def extract_scalar_data(events, branches, entrystop=None, progressbar=False):
    data = {}

    data["event"] = events.array("event", entrystop=entrystop)

    for br in tqdm(branches, disable=not progressbar):
        data[br] = events.array(br, entrystop=entrystop).flatten()

    return pd.DataFrame(data)

def extract_vector_data(events, branches, entrystop=10, progressbar=False):
    def get(branch, flat=True):
        a = events.array(branch, entrystop=entrystop)
        if flat:
            return a.flatten()
        else:
            return a

    if len(branches) == 0:
        return {}

    first_branch_jagged = get(branches[0], flat=False)
    first_branch_flat = first_branch_jagged.flatten()

    event_jagged = get("event") + awkward.JaggedArray(
        first_branch_jagged.starts, first_branch_jagged.stops, np.zeros(len(first_branch_flat), dtype=np.int)
    )

    data = {}

    data["event"] = event_jagged.flatten()

    data[branches[0]] = first_branch_flat

    for br in tqdm(branches, disable=not progressbar):
        if br == branches[0]:
            continue

        if br in events:
            data[br] = get(br)
        else:
            print('Warning! Branch "' + br + '" not found in input file and skipped.')

    return pd.DataFrame(data)

def nanoaod_to_parquet(input_files, out_dir, entrystop=None, input_prefix="", progressbar=False):
    def save_parquet(df, name):
        df.to_parquet(os.path.join(out_dir, name + ".parquet.gzip"), compression="gzip", index=False)

    out_dir = os.path.expanduser(out_dir)

    if not os.path.exists(out_dir):
        os.makedirs(out_dir)

    n_input_files = len(input_files)

    for i, input_file in enumerate(input_files):

        log_prefix = ""
        if i > 0:
            log_prefix = f"[{i+1}/{n_input_files}] "

        def log(s):
            print(log_prefix + s)

        log("Opening input file " + input_file)
        root_file = uproot.open(input_prefix + input_file)

        events = root_file["Events"]

        branches = [br.decode("ascii") for br in events.keys()]

        n_events = len(events[branches[0]])

        prefixes = list(set([b.split("_")[0] for b in branches]))

        vector_groups_present = [p for p in prefixes if "n" + p in events]

        scalar_branches = ["n" + s for s in vector_groups_present]

        for br in branches:
            if br == "event":
                continue
            if not br.split("_")[0] in vector_groups_present:
                scalar_branches.append(br)

        basename = os.path.basename(input_file)[:-5]

        log("Loading DataFrame for scalar branches")
        df_scalar = extract_scalar_data(events, scalar_branches, entrystop=entrystop, progressbar=progressbar)
        log("Saving DataFrame parquet Scalar")
        save_parquet(df_scalar, basename + "_Scalar")
        del df_scalar

        processed_branches = scalar_branches + ["event"]

        for group in vector_groups_present:
            log("Loading DataFrame for object group " + group)
            filtered_branches = list(filter(lambda br: br == group or br.startswith(group + "_"), branches))
            df = extract_vector_data(events, filtered_branches, entrystop=entrystop, progressbar=progressbar)
            log("Saving DataFrame parquet " + group)
            save_parquet(df, basename + "_" + group)
            processed_branches += filtered_branches
            del df

        # make sure we considered all the branches
        assert [b for b in branches if not b in processed_branches] == []

from geeksw.nanocache import list_files

import os
import subprocess

def convert_files_to_parquet(input_files, base_out_dir=".", server="root://polgrid4.in2p3.fr/"):

    n = len(input_files)

    for i, input_file in enumerate(input_files):

        out_dir = base_out_dir + os.path.dirname(input_file)

        tmp_file_name = os.path.join("/tmp", os.path.basename(input_file))

        if os.path.exists(tmp_file_name):
            subprocess.call(["rm", tmp_file_name])

        print(f"File {i+1} of {n}: copying " + input_file + " to " + tmp_file_name)
        subprocess.call(["xrdcp", server + input_file, tmp_file_name])

        nanoaod_to_parquet([tmp_file_name], out_dir, entrystop=None, progressbar=False)

        subprocess.call(["rm", tmp_file_name])
        print("Deleted " + tmp_file_name)

def convert_dataset_to_parquet(dataset, base_out_dir=".", server="root://polgrid4.in2p3.fr/"):

    input_files = list_files(dataset)

    convert_files_to_parquet(input_files, base_out_dir=base_out_dir, server=server)


# 取消任务

import schedule

i = 0

def some_task():
    global i
    i += 1
    print(i)
    if i == 10:
        schedule.cancel_job(job)
        print('cancel job')
        exit(0)

job = schedule.every().second.do(some_task)
while True:
    schedule.run_pending()


from django.db import transaction
from django.utils.translation import ugettext, ugettext_lazy as _
from django import forms

from gf.gas.forms.order.plan import AddPlannedOrderForm
from gf.gas.forms.order.base import AddOrderForm

from gf.gas.models import GAS, GASSupplierOrder, GASSupplierSolidalPact

import logging, copy

log = logging.getLogger(__name__)

#--------------------------------------------------------------------------------

class AddInterGASOrderForm(AddOrderForm):
    """Form to manage InterGAS orders.

    This works only for context resources GAS and Pact
    """

    #WAS: INTERGAS 0
    intergas = forms.BooleanField(label=_('Is this an InterGAS order?'), required=False)
    intergas_grd = forms.ModelMultipleChoiceField(
        label=_('InterGAS order with'), queryset=GAS.objects.none(),
        required=False,
        widget=forms.CheckboxSelectMultiple
    )

    #WAS: INTERGAS 1
    def __init__(self, *args, **kw):

        super(AddInterGASOrderForm, self).__init__(*args, **kw)

        pacts_count = self._pacts.count()
        self._gas = self._pacts[0].gas

        # Check needed because exception will be shown just by the end of the execution
        # of the __init__
        if pacts_count:

            intergas_gas_qs = self._gas.config.intergas_connection_set.all()
            if pacts_count == 1:

                #log.debug("AddOrderForm only one pact %s" % pacts)
                intergas_gas_qs = intergas_gas_qs & self._pacts[0].supplier.gas_list

        self.fields['intergas_grd'].queryset = intergas_gas_qs

    def clean(self):
        """InterGAS clean() checks for selected GAS and set _involved_extra_pacts attributes."""

        #WAS: INTERGAS 2 (rewritten. It can be factorized along with ordinary GASSupplierOrder clean)

        cleaned_data = super(AddInterGASOrderForm, self).clean()
        self._involved_extra_pacts = set()
        self._intergas_requested = cleaned_data.get('intergas')
        _involved_gas_list = cleaned_data.get('intergas_grd',[])
        supplier = cleaned_data['pact'].supplier

        for gas in _involved_gas_list:
            if gas != self._gas:
                log.debug("AddOrderForm intergas finding another PACT for GAS %s..." % gas)

                #retrieve the existing pact for this gas for this supplier. If exist.
                #TODO: Matteo: if not exists -> POST forged -> we should raise specific exception
                extra_pact = None
                try:
                    extra_pact = gas.pacts.get(supplier=supplier)
                    log.debug("Pact %s found." % extra_pact)
                    self._involved_extra_pacts.add(extra_pact)
                except GASSupplierSolidalPact.DoesNotExist as e:
                    #WAS: self._messages['error'].append(ugettext("Please select a gas which has a pact with the same supplier of the pact this order refers to"))
                    log.debug("Not valid InterGAS order: one or more GAS involeved do not have pacts with the same supplier")
                    raise forms.ValidationError("Not valid InterGAS order: one or more GAS involeved do not have pacts with the same supplier")

        if self._intergas_requested and not self._involved_extra_pacts:
            log.debug("Not valid InterGAS order: at least 2 GAS needed")
            raise forms.ValidationError("Not valid InterGAS order: at least 2 GAS needed")

        return cleaned_data

    def create_order_for_another_pact(self, other_pact):
        """This relates to InterGAS."""

        new_obj = self.instance.clone()
        new_obj.pact = other_pact

        obj = self.instance

        # retrieve the first referrer_person
        refs = other_pact.referrers_people
        if len(refs):
            log.debug("refs %s found for pact %s" % (refs, other_pact))
            new_obj.referrer_person = refs[0]
            new_obj.delivery_referrer_person = new_obj.referrer_person
            new_obj.withdrawal_referrer_person = new_obj.referrer_person
        else:
            #FIXME: 'NoneType' object has no attribute 'user'. Cannot be real. But model permitted
            #Cannot create the order.
            log.warning("no referrers for pact %s" % other_pact)

        #Delivery set to default delivery Place if gas is configured accordingly
        if obj.delivery:

            dd_place = other_pact.gas.config.default_delivery_place or \
                other_pact.gas.headquarter
            new_obj.delivery, created = Delivery.objects.get_or_create(
                    date=obj.delivery.date, place=dd_place
            )

        return new_obj

    @transaction.atomic
    def save(self):

        #WAS: INTERGAS 4 (rewritten. Set correctly group_id in base order to not repeat in planned orders)

        if self._intergas_requested:
            self.instance.group_id = GASSupplierOrder.objects.get_new_intergas_group_id()

            # Create complementary InterGAS orders

            for extra_pact in self._involved_extra_pacts:
                other_intergas_order = self.create_order_for_another_pact(extra_pact)
                try:
                    other_intergas_order.save()
                    log.debug("created another intergas order: %s " % other_intergas_order)
                except Exception as e:
                    log.debug("repeat another NOT created: pact %s, start %s , end %s , delivery %s" % (
                        other_intergas_order.pact, other_intergas_order.datetime_start, other_intergas_order.datetime_end,
                        other_intergas_order.delivery.date
                    ))
                    raise

        super(AddInterGASOrderForm, self).save()

    class Meta(AddOrderForm.Meta):

        #WAS: INTERGAS 6
        gf_fieldsets = copy.deepcopy(AddOrderForm.Meta.gf_fieldsets)
        gf_fieldsets[0][1]['fields'].append(('intergas', 'intergas_grd'))

class AddInterGASPlannedOrderForm(AddInterGASOrderForm, AddPlannedOrderForm):
    """Form to manage InterGAS and order planning at the same time."""

    @transaction.atomic
    def save(self):

        AddInterGASOrderForm.save(self)
        AddPlannedOrderForm.save(self)

    class Meta(AddPlannedOrderForm.Meta):

        gf_fieldsets = copy.deepcopy(AddPlannedOrderForm.Meta.gf_fieldsets)
        gf_fieldsets[0][1]['fields'].append(('intergas', 'intergas_grd'))


from rich.console import Console
from gallow import gallow_frames
from ending import game_over
from the_word import word

c = Console()

def make_gallow(wrong_guesses):
    c.print(gallow_frames[wrong_guesses], style="white")

def enter_word():
    return input("Enter Word: ").upper()

def display_guesses(word, guesses):
    letters = ""
    for letter in word:
        if letter in guesses:
            letters += letter + " "
        elif letter == " ":
            letters += "  "
        else:
            letters += "_ "
    c.print(letters, style="cyan")

def display_wrong_guesses(word, guesses):
    wrong_guesses = ""
    for letter in guesses:
        if letter not in word:
            wrong_guesses += f"{letter} "

    c.print(wrong_guesses, style="magenta")

def get_new_guess(guesses):
    new_guess = input("Make guess: ").upper()
    while new_guess in guesses or new_guess == " ":
        new_guess = input(f"{new_guess} already guessed. Make new guess: ").upper()
    guesses += new_guess[0]
    return guesses

def count_wrong_guesses(word, guesses):
    wrong_guesses = 0
    for letter in guesses:
        if letter not in word:
            wrong_guesses += 1
    return wrong_guesses

def main():
    word_no_dup = "".join(set(word))
    guesses = ""
    wrong_guesses = 0
    playing = True
    while playing:
        make_gallow(wrong_guesses)
        display_guesses(word, guesses)
        display_wrong_guesses(word, guesses)
        guesses = get_new_guess(guesses)
        wrong_guesses = count_wrong_guesses(word,guesses)
        if wrong_guesses >= 6:
            make_gallow(6)
            c.print(game_over, style="red")
            playing = False
        misses = 0
        for letter in word:
            if letter not in guesses and letter != " ":
                misses += 1
        if misses == 0:
            make_gallow(7)
            c.print("Congratulations!\nYou Win!", style="yellow")
            playing = False

if __name__ == '__main__':
    main()


from typing import Any

import database
import models
import repos.queries

def one_of(**params) -> models.Question | None:
    """
    Get models.Question object from database by "specifiers" params

    Arguments:
        **params: specifiers, if it needs to perform data filtering
    """
    specifiers = " ".join(params.keys())
    table_fields = [
        field for field in vars(models.Question).get("__annotations__")
        if not field.startswith("_")
    ]
    query = repos.queries.select_query(
        table_name=models.Question._table_name,
        table_fields=table_fields,
        specifiers=specifiers,
    )
    with database.Connection() as db:
        if params.values():
            param_list = [p for p in params.values()]
            question_record = db.connection.execute(query, param_list).fetchone()
        else:
            question_record = db.connection.execute(query).fetchone()
    if not question_record:
        return None
    return models.Question.from_dict(question_record)

def all_of(**params) -> list[models.Question]:
    """
    Get models.Question objects from database by "specifiers" params

    Arguments:
        **params: specifiers, if it needs to perform data filtering
    """
    specifiers = " ".join(params.keys())
    table_fields = [
        field for field in vars(models.Question).get("__annotations__")
        if not field.startswith("_")
    ]
    query = repos.queries.select_query(
        table_name=models.Question._table_name,
        table_fields=table_fields,
        specifiers=specifiers,
    )
    with database.Connection() as db:
        if params.values():
            param_list = [p for p in params.values()]
            question_records = db.connection.execute(query, param_list).fetchall()
        else:
            question_records = db.connection.execute(query).fetchall()
    return [models.Question.from_dict(record) for record in question_records]

def new(
    question: str,
    asking_user: models.User | None = None,
    asking_user_id: int | None = None,
    expert: models.User | None = None,
    expert_id: int | None = None,
    answer: str | None = None,
) -> models.Question:
    """
    Creates and returns new question

    Arguments:
        question: text of question
        asking_user: user asking. Excludes asking_user_id
        asking_user_id: user's asking. Excludes asking_user
        expert: answering user. Excludes expert_id
        expert_id: answering user. Excludes expert
        answer: text of answer
    """
    if not (asking_user or asking_user_id):
        raise ValueError("Neither 'asking_user' was provided, nor 'asking_user_id'")
    if not (expert or expert_id):
        raise ValueError("Neither 'expert' was provided, nor 'expert_id'")

    table_fields = [
        field for field in vars(models.Question).get("__annotations__")
        if not field.startswith("_") and not field == "id"
    ]
    query = repos.queries.insert_query(
        table_name=models.Question._table_name,
        table_fields=table_fields,
    )
    asking_user_id = asking_user_id or asking_user.id
    expert_id = expert_id or expert.id
    with database.Connection() as db:
        db.cursor.execute(query, [question, asking_user_id, expert_id, answer])
        db.connection.commit()
        new_question_id = db.cursor.lastrowid
    return models.Question(
        id=new_question_id,
        question=question,
        asking_user_id=asking_user_id,
        expert_id=expert_id,
        answer=answer,
    )

def change(
    updating_data: dict[str, Any],
    question: models.Question | None = None,
    **where_params,
) -> None:
    """
    Updates users of provided users of by where_params

    Arguments
        updating_data: dictionary with field and val to update
        question: if you have Question object, and you need to
                change it, you can pass this obj with this arg
        **where_params: WHERE conditions
    """
    table_fields = [
        field for field in vars(models.Question).get("__annotations__")
        if not field.startswith("_")
    ]
    specifiers = None
    params = None
    if question:
        specifiers = "id"
        params = [question.id]
    elif where_params:
        specifiers = " ".join(where_params.keys())
        params = [val for val in where_params.values()]
    query = repos.queries.update_query(
        table_name=models.Question._table_name,
        table_fields=table_fields,
        updating_data=updating_data,
        specifiers=specifiers
    )
    with database.Connection() as db:
        if specifiers:
            db.cursor.execute(query, params)
        else:
            db.cursor.execute(query)
        db.connection.commit()

def all_unanswered_questions(expert_id: int) -> list[models.Question]:
    query = "SELECT * FROM questions WHERE expert_id = ? AND answer IS NULL;"
    with database.Connection() as db:
        questions = db.cursor.execute(query, [expert_id]).fetchall()
    return [models.Question.from_dict(question) for question in questions]

def all_answered_questions() -> list[models.Question]:
    query = "SELECT * FROM questions WHERE answer IS NOT NULL;"
    with database.Connection() as db:
        questions = db.cursor.execute(query).fetchall()
    return [models.Question.from_dict(question) for question in questions]


import pprint
from typing import Dict
from pydantic import BaseModel, Field
from kubernetes import client

class InputSchema(BaseModel):
    namespace: str = Field(
        title='Namespace',
        description='Kubernetes namespace')
    pod_name: str = Field(
        title='Pod',
        description='Name of the pod')

def k8s_get_pod_status_printer(data):
    if data is None:
        return
    pprint.pprint(data)

def k8s_get_pod_status(handle, namespace: str, pod_name: str) -> Dict:
    """k8s_get_pod_status get pod status

        :type handle: object
        :param handle: Object returned from the Task validate method

        :type namespace: str
        :param namespace: Kubernetes namespace.

        :type pod_name: str
        :param pod_name: Name of the pod.

        :rtype: Dict
    """
    coreApiClient = client.CoreV1Api(api_client=handle)

    status = coreApiClient.read_namespaced_pod_status(
        namespace=namespace, name=pod_name)

    res = {}

    ready_containers_number = 0
    containers_number = 0
    restarts_number = 0

    for container in status.status.container_statuses:
        if container.ready:
            ready_containers_number += 1
        if container.restart_count:
            restarts_number = restarts_number + container.restart_count
        containers_number += 1
    res["NAME"] = pod_name
    res['READY'] = f"Ready {ready_containers_number}/{containers_number}"
    res['STATUS'] = status.status.phase
    res['RESTARTS'] = restarts_number
    res['START_TIME'] = status.status.start_time.strftime("%m/%d/%Y, %H:%M:%S")
    return res


from functools import wraps

from pycompss.util.context import CONTEXT
from pycompss.api.commons.constants import INTERNAL_LABELS
from pycompss.api.commons.constants import LABELS
from pycompss.api.commons.constants import LEGACY_LABELS
from pycompss.api.commons.decorator import CORE_ELEMENT_KEY
from pycompss.api.commons.decorator import keep_arguments
from pycompss.api.commons.decorator import process_computing_nodes
from pycompss.api.commons.decorator import resolve_fail_by_exit_value
from pycompss.api.commons.decorator import resolve_working_dir
from pycompss.api.commons.error_msgs import not_in_pycompss
from pycompss.api.commons.implementation_types import IMPLEMENTATION_TYPES
from pycompss.runtime.task.definitions.core_element import CE
from pycompss.util.arguments import check_arguments
from pycompss.util.exceptions import NotInPyCOMPSsException
from pycompss.util.typing_helper import typing

if __debug__:
    import logging

    logger = logging.getLogger(__name__)

MANDATORY_ARGUMENTS = {LABELS.df_script}
SUPPORTED_ARGUMENTS = {
    LABELS.computing_nodes,
    LABELS.working_dir,
    LABELS.runner,
    LABELS.df_executor,
    LABELS.df_lib,
    LABELS.df_script,
    LABELS.fail_by_exit_value,
}
DEPRECATED_ARGUMENTS = {
    LEGACY_LABELS.computing_nodes,
    LEGACY_LABELS.working_dir,
    LEGACY_LABELS.df_executor,
    LEGACY_LABELS.df_lib,
    LEGACY_LABELS.df_script,
}

class Decaf:  # pylint: disable=too-few-public-methods
    """Decaf decorator class.

    This decorator also preserves the argspec, but includes the __init__ and
    __call__ methods, useful on decaf task creation.
    """

    __slots__ = [
        "decorator_name",
        "args",
        "kwargs",
        "scope",
        "core_element",
        "core_element_configured",
    ]

    def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None:
        """Store arguments passed to the decorator.

        self = itself.
        args = not used.
        kwargs = dictionary with the given constraints.

        :param args: Arguments.
        :param kwargs: Keyword arguments.
        """
        decorator_name = "".join(("@", Decaf.__name__.lower()))
        # super(Decaf, self).__init__(decorator_name, *args, **kwargs)
        self.decorator_name = decorator_name
        self.args = args
        self.kwargs = kwargs
        self.scope = CONTEXT.in_pycompss()
        self.core_element = None  # type: typing.Optional[CE]
        self.core_element_configured = False
        if self.scope:
            # Check the arguments
            check_arguments(
                MANDATORY_ARGUMENTS,
                DEPRECATED_ARGUMENTS,
                SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS,
                list(kwargs.keys()),
                decorator_name,
            )

            # Get the computing nodes
            process_computing_nodes(decorator_name, self.kwargs)

    def __call__(self, user_function: typing.Callable) -> typing.Callable:
        """Parse and set the decaf parameters within the task core element.

        :param user_function: Function to decorate.
        :return: Decorated function.
        """

        @wraps(user_function)
        def decaf_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any:
            if not self.scope:
                raise NotInPyCOMPSsException(not_in_pycompss("decaf"))

            if __debug__:
                logger.debug("Executing decaf_f wrapper.")

            if (
                CONTEXT.in_master() or CONTEXT.is_nesting_enabled()
            ) and not self.core_element_configured:
                # master code - or worker with nesting enabled
                self.__configure_core_element__(kwargs)

            # Set the computing_nodes variable in kwargs for its usage
            # in @task decorator
            kwargs[LABELS.computing_nodes] = self.kwargs[
                LABELS.computing_nodes
            ]

            with keep_arguments(args, kwargs, prepend_strings=False):
                # Call the method
                ret = user_function(*args, **kwargs)

            return ret

        decaf_f.__doc__ = user_function.__doc__
        return decaf_f

    def __configure_core_element__(self, kwargs: dict) -> None:
        """Include the registering info related to @decaf.

        IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].

        :param kwargs: Keyword arguments received from call.
        :return: None
        """
        if __debug__:
            logger.debug("Configuring @decaf core element.")

        # Resolve @decaf specific parameters
        runner = "mpirun"
        if LABELS.runner in self.kwargs:
            runner = self.kwargs[LABELS.runner]

        if LEGACY_LABELS.df_script in self.kwargs:
            df_script = self.kwargs[LEGACY_LABELS.df_script]
        else:
            df_script = self.kwargs[LABELS.df_script]

        if LABELS.df_executor in self.kwargs:
            df_executor = self.kwargs[LABELS.df_executor]
        elif LEGACY_LABELS.df_executor in self.kwargs:
            df_executor = self.kwargs[LEGACY_LABELS.df_executor]
        else:
            df_executor = (
                INTERNAL_LABELS.unassigned
            )  # Empty or INTERNAL_LABELS.unassigned

        if LABELS.df_lib in self.kwargs:
            df_lib = self.kwargs[LABELS.df_lib]
        elif LEGACY_LABELS.df_lib in self.kwargs:
            df_lib = self.kwargs[LEGACY_LABELS.df_lib]
        else:
            df_lib = (
                INTERNAL_LABELS.unassigned
            )  # Empty or INTERNAL_LABELS.unassigned

        # Resolve the working directory
        resolve_working_dir(self.kwargs)
        # Resolve the fail by exit value
        resolve_fail_by_exit_value(self.kwargs)

        impl_type = IMPLEMENTATION_TYPES.decaf
        impl_signature = ".".join((impl_type, df_script))
        impl_args = [
            df_script,
            df_executor,
            df_lib,
            self.kwargs[LABELS.working_dir],
            runner,
            self.kwargs[LABELS.fail_by_exit_value],
        ]

        if CORE_ELEMENT_KEY in kwargs:
            # Core element has already been created in a higher level decorator
            # (e.g. @constraint)
            kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
            kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature)
            kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
        else:
            # @binary is in the top of the decorators stack.
            # Instantiate a new core element object, update it and include
            # it into kwarg
            core_element = CE()
            core_element.set_impl_type(impl_type)
            core_element.set_impl_signature(impl_signature)
            core_element.set_impl_type_args(impl_args)
            kwargs[CORE_ELEMENT_KEY] = core_element

        # Set as configured
        self.core_element_configured = True

# ########################################################################### #
# #################### DECAF DECORATOR ALTERNATIVE NAME ##################### #
# ########################################################################### #

decaf = Decaf  # pylint: disable=invalid-name


import argparse
import os
import copy
import logging

import torch
from torch.distributed.distributed_c10d import _get_default_group
from torch.distributed.utils import _sync_module_states

from torchvision.models import resnet18

from pippy.IR import Pipe
from pippy import split_into_equal_size

from easydist import easydist_setup, mdconfig
from easydist.torch.api import easydist_compile
from easydist.torch import set_device_mesh
from easydist.utils.testing import TorchMockDeviceMesh

def broadcast_module(model):
    _sync_module_states(model,
                        _get_default_group(),
                        broadcast_bucket_size=int(250 * 1024 * 1024),
                        src=0,
                        params_and_buffers_to_ignore=set())
    return model

def train_submodule(train_step, name, module, *prev_mod_input):
    with torch.device('cuda'):
        opt = torch.optim.Adam(module.parameters(),
                               lr=0.001, foreach=True, capturable=True)
        module_2 = copy.deepcopy(module)
        opt_2 = torch.optim.Adam(
            module_2.parameters(), lr=0.001, foreach=True, capturable=True)

        torch_step_1_result = train_step.original_func(
            module, opt, *prev_mod_input)
        torch_step_2_result = train_step.original_func(
            module, opt, *prev_mod_input)

    md_step_1_result = train_step(module_2, opt_2, *prev_mod_input)
    md_step_2_result = train_step(module_2, opt_2, *prev_mod_input)

    warp_fn = lambda item: [item] if isinstance(item, torch.Tensor) else item
    local_rank = int(os.environ["LOCAL_RANK"])
    for i, (torch_res, md_res) in enumerate(zip(warp_fn(torch_step_1_result), warp_fn(md_step_1_result))):
        # assert torch.allclose(torch_res, md_res), f"{name} training test step 1 failed."
        if not torch.allclose(torch_res, md_res):
            print(
                '\033[31m'
                f'rank {local_rank} {name} step 1 {i}-th output failed.'
                f'\n\tabs sum: {torch.sum(torch.abs(torch_res - md_res))}'
                '\033[0m'
                )

    for i, (torch_res, md_res) in enumerate(zip(warp_fn(torch_step_2_result), warp_fn(md_step_2_result))):
        # assert torch.allclose(torch_res, md_res), f"{name} training test step 2 failed."
        if not torch.allclose(torch_res, md_res):
            print(
                '\033[31m'
                f'rank {local_rank} {name} step 2 {i}-th output failed.'
                f'\n\tabs sum: {torch.sum(torch.abs(torch_res - md_res))}'
                '\033[0m'
                )

    return torch_step_2_result

@easydist_compile(tracing_mode="fake", cuda_graph=False)
def train_step_submod0(model, opt, *input):
    out = model(*input)
    _out = out[0].mean() + out[1].mean()
    _out.backward()
    opt.step()
    opt.zero_grad(True)
    return out

@easydist_compile(tracing_mode="fake", cuda_graph=False)
def train_step_submod1(model, opt, *input):
    out = model(*input)
    _out = out.mean()
    _out.backward()
    opt.step()
    opt.zero_grad(True)
    return out

def train_example():
    world_size = int(os.environ["WORLD_SIZE"])
    local_rank = int(os.environ["LOCAL_RANK"])
    model = resnet18().cuda()
    model.train()
    # broadcast the parameter and input
    model = broadcast_module(model)

    split_policy = split_into_equal_size(world_size)
    pipe = Pipe.from_tracing(model, split_policy=split_policy).cuda()

    input = [torch.randn(16, 3, 224, 224).cuda()]
    torch.distributed.broadcast(input[0], src=0)
    input = train_submodule(train_step_submod0, "submod_0", pipe.split_gm.submod_0, *input)

    input = [t.detach() for t in input]
    train_submodule(train_step_submod1, "submod_1", pipe.split_gm.submod_1, *input)

    print(f"rank {local_rank} training test finished.")

def inference_example():
    raise NotImplementedError()

def main():
    parser = argparse.ArgumentParser(
        description="Simple example of parallelize model.")
    parser.add_argument("--mode",
                        type=str,
                        default=None,
                        choices=["train", "inference"],
                        required=True)
    args = parser.parse_args()

    mdconfig.log_level = logging.ERROR
    easydist_setup(backend="torch", device="cuda", allow_tf32=False)
    world_size = int(os.environ["WORLD_SIZE"])
    assert world_size == 2, "This example only support 2 GPUs."
    mock_mesh = TorchMockDeviceMesh(1, 2, debug_only=True)
    set_device_mesh(mock_mesh)

    torch.distributed.init_process_group(backend="nccl")
    local_rank = int(os.environ["LOCAL_RANK"])
    torch.cuda.set_device(local_rank)

    if args.mode == "train":
        train_example()
    else:
        inference_example()

if __name__ == "__main__":
    os.chdir(os.path.dirname(os.path.abspath(__file__)))
    main()


import sys
from time import sleep

from common.db_utils import (
    ConnFactory,
)
from common.orm.repository import PoktInfoRepository
from common.utils import get_last_block_height

from rewards_calc import run_rewards, record_rewards, SERVICE_CLASS

SAVE_STATE = True

if __name__ == "__main__":
    mode = str(sys.argv[1])

    # generate_valid_urls()

    # python3 run_rewards.py history 500 50000
    if mode == "history":
        skip_recorded = False
        # Historical mode - gets rewards for addresses between heights.
        from_height, to_height = int(sys.argv[2]), int(sys.argv[3])
        as_test = False
        run_rewards(
            from_height,
            to_height,
            as_test,
            save_state=SAVE_STATE,
            skip_recorded=skip_recorded,
        )
    # python3 run_rewards.py live (optional to add height to start from)
    elif mode == "live":
        # Live mode - checks if new block has been created and if so, get rewards.
        as_test = False
        with ConnFactory.poktinfo_conn() as session:
            last_height = (
                PoktInfoRepository.get_last_recorded_reward_height(session)
                if len(sys.argv) < 3 or not sys.argv[2].isdigit()
                else int(sys.argv[2])
            )
        while True:
            try:
                height = get_last_block_height()
                if height - 1 > last_height:
                    record_rewards(last_height, as_test, save_state=SAVE_STATE)
                    last_height += 1
            except Exception as e:
                print(e)
            sleep(60)
    # python3 run_rewards.py complete
    elif mode == "complete":
        as_test = False
        # Retrieves from db failed blocks and reruns them
        with ConnFactory.poktinfo_conn() as session:
            heights = PoktInfoRepository.get_failed_blocks_of_service(
                session, SERVICE_CLASS
            )
        run_rewards(0, 0, as_test, heights=heights, save_state=SAVE_STATE)


import numpy as np
from sklearn.metrics import confusion_matrix, precision_recall_fscore_support, \
    accuracy_score

def zip_data(data, target):
    return {study: (data[study], target[study]) for study in data}

def unzip_data(data):
    return {study: data[study][0] for study in data}, \
           {study: data[study][1] for study in data}

def compute_metrics(preds, targets, target_encoder):
    f1_dict = {}
    prec_dict = {}
    recall_dict = {}
    confusion_dict = {}
    accuracy_dict = {}
    bacc_dict = {}
    for study in preds:
        these_preds = preds[study]['contrast']
        these_targets = targets[study]['contrast']
        accuracy_dict[study] = accuracy_score(these_preds, these_targets)
        precs, recalls, f1s, support = precision_recall_fscore_support(
            these_preds, these_targets, warn_for=())
        contrasts = target_encoder.le_[study]['contrast'].classes_
        prec_dict[study] = {contrast: prec for contrast, prec in
                           zip(contrasts, precs)}
        recall_dict[study] = {contrast: recall for contrast, recall in
                             zip(contrasts, recalls)}
        f1_dict[study] = {contrast: f1 for contrast, f1 in
                         zip(contrasts, f1s)}
        confusion = confusion_matrix(these_preds, these_targets)
        confusion_dict[study] = confusion.tolist()

        baccs = baccs_from_confusion(confusion)
        bacc_dict[study] = {contrast: bacc for contrast, bacc in
                           zip(contrasts, baccs)}

    return {'confusion': confusion_dict,
            'prec': prec_dict,
            'recall': recall_dict,
            'f1': f1_dict,
            'bacc': bacc_dict,
            'accuracy': accuracy_dict,
            }

def baccs_from_confusion(C):
    baccs = []
    total = np.sum(C)
    for i in range(len(C)):
        t = np.sum(C[i])
        p = np.sum(C[:, i])
        n = total - p
        tp = C[i, i]
        fp = p - tp
        fn = t - tp
        tn = total - fp - fn - tp
        baccs.append(.5 * (tp / p + tn / n))
    return baccs

class ScoreCallback:
    def __init__(self, Xs, ys, score_function):
        self.Xs = Xs
        self.ys = ys
        self.score_function = score_function
        self.n_iter_ = []
        self.scores_ = []
        self.ranks_ = []

    def __call__(self, estimator, n_iter):
        preds = estimator.predict(self.Xs)
        scores = {}
        study_scores = {}
        study_contrast_scores = {}
        for study in self.ys:
            scores[study] = self.score_function(preds[study]['contrast'],
                                                self.ys[study]['contrast'])
            study_scores[study] = self.score_function(preds[study]['study'],
                                                      self.ys[study]['study'])
            study_contrast_scores[study] = self.score_function(
                preds[study]['study_contrast'],
                self.ys[study]['study_contrast'])
        self.n_iter_.append(n_iter)
        self.scores_.append(scores)
        scores_str = ' '.join('%s: %.3f' % (study, score)
                              for study, score in scores.items())
        scores_str = 'Score: ' + scores_str
        return scores_str

class MultiCallback:
    def __init__(self, callbacks):
        self.callbacks = callbacks

    def __call__(self, *args, **kwargs):
        for name, callback in self.callbacks.items():
            output = callback(*args, **kwargs)
            print('[%s] %s' % (name, output))


# basic
import argparse
import os,datetime

# data input
import pandas as pd
import numpy as np
import scipy.sparse
import pickle
# metric and loss function
import tensorflow as tf
import tensorflow.keras.backend as K
from tensorflow.keras.metrics import categorical_accuracy, binary_accuracy, top_k_categorical_accuracy
# embedding
from tensorflow.keras.initializers import Constant
from tensorflow.keras.layers import Embedding
# models
import tensorflow_hub as hub
from tensorflow.keras.layers import Layer
from tensorflow.keras.layers import Dense, Input, Flatten, Concatenate, Conv1D, MaxPooling1D, Dropout
from tensorflow.keras.layers import CuDNNLSTM, Bidirectional, TimeDistributed, Lambda, Softmax
from tensorflow.keras.initializers import Constant
from tensorflow.keras.models import Model

def Coloured(string):
    return "\033[1;30;43m {} \033[0m".format(string)

# # INPUT

def get_input(in_dir, mode, sparse = False, get_output = [True]*4):
    # get_output: bool of get this var ['x_train','y_trains','x_test','y_tests']
    x_train,y_trains,x_test,y_tests = [None]*4
    if get_output[0]:
        x_train = np.load(os.path.join(in_dir,'x_train.npy'))
    if get_output[1]:
        dirs = [os.path.join(in_dir,d) for d in sorted(os.listdir(in_dir)) if d.startswith('y_train_{}'.format(mode))]
        y_trains = [scipy.sparse.load_npz(d) for d in dirs]
        if not sparse:
            y_trains = [y.toarray() for y in y_trains]
    if get_output[2]:
        x_test = np.load(os.path.join(in_dir,'x_test.npy'))
    if get_output[3]:
        dirs = [os.path.join(in_dir,d) for d in sorted(os.listdir(in_dir)) if d.startswith('y_test_{}'.format(mode))]
        y_tests = [scipy.sparse.load_npz(d) for d in dirs]
        if not sparse:
            y_tests = [y.toarray() for y in y_tests]
    return [x_train],y_trains,[x_test],y_tests

def get_bert_input(in_dir,mode):
    df = pd.read_pickle(os.path.join(in_dir,'bert_x.pkl'))
    train_df = df[df['train/test']=='train']
    test_df = df[df['train/test']=='test']
    train_sequence = train_df['sequence'].to_list()
    train_mask = train_df['mask'].to_list()
    train_segment = [[0]*len(train_mask[0]) ]*len(train_mask)
    test_sequence = test_df['sequence'].to_list()
    test_mask = test_df['mask'].to_list()
    test_segment = [[0]*len(test_mask[0]) ]*len(test_mask)
    x_trains = [train_sequence,train_mask,train_segment]
    x_trains = [np.array(x) for x in x_trains]
    x_tests = [test_sequence,test_mask,test_segment]
    x_tests = [np.array(x) for x in x_tests]
    _,y_trains,_,y_tests = get_input(in_dir, mode, get_output=[0,1,0,1])
    return x_trains, y_trains, x_tests, y_tests

def mask_ys(y_trues,in_dir):
    # slow an stupid way to mask non important values to -1
    ds = pickle.load(open(os.path.join(in_dir,'child_to_siblings.pkl'),'rb'))
    outs = [y_trues[0]]
    for i,y in enumerate(y_trues):
        if i==0:
            continue
        d = ds[i]
        y_true = y.argmax(axis=1).flatten()
        out = -np.ones(shape=y.shape)
        for j,yt in enumerate(y_true):
            out[j,d[yt]]=0
            out[j,yt]=1
        outs.append(out)
    return outs

def get_unbiased_train_val_split(x_trains,y_trains,in_dir,print_progress = True):
    train_inds_dir = os.path.join(in_dir,'train_inds.npy')
    val_inds_dir = os.path.join(in_dir,'val_inds.npy')

    if not os.path.exists(train_inds_dir) or not os.path.exists(val_inds_dir):
        print('SPLIT BY CLASS')
        val_split=0.2
        sparse_ys = [np.argmax(y_train,axis=1) for y_train in y_trains]
        yt = np.column_stack(sparse_ys)
        unique_h,unique_inverse = np.unique(yt,return_inverse=True,axis=0)
        too_small = []
        train_inds = np.array([])
        val_inds = np.array([])
        for s in range(len(unique_h)):
            inds = np.argwhere(unique_inverse==s)
            if len(inds)<=1:
                too_small.append(inds[0])
                train_inds = np.append(train_inds,inds)
                val_inds = np.append(val_inds,inds)
            else:
                split = int(len(inds)*val_split)
                np.random.seed(s)
                np.random.shuffle(inds)
                train_inds = np.append(train_inds,inds[max(1,split):])
                val_inds = np.append(val_inds,inds[:max(1,split)])
            if print_progress and s%(len(unique_h)//10)==0:
                print("{:.0f}%".format((s+1)/len(unique_h)*100),end='\r')
        print("Duplicated inds: {}".format(len(too_small)))
        train_inds = train_inds.astype(int)
        val_inds = val_inds.astype(int)
        np.save(train_inds_dir,train_inds)
        np.save(val_inds_dir,val_inds)
    else:
        print('LOAD EXISTING VAL INDS')
        train_inds = np.load(train_inds_dir)
        val_inds = np.load(val_inds_dir)

    x_vs = [x[val_inds,:] for x in x_trains]
    y_vs = [y[val_inds,:] for y in y_trains]
    x_ts = [x[train_inds,:] for x in x_trains]
    y_ts = [y[train_inds,:] for y in y_trains]

    return x_ts,y_ts,x_vs,y_vs

# # EMBEDDING LAYER

def get_embedding_layer(in_dir):
    embedding_matrix = np.load(os.path.join(in_dir,'embedding_matrix.npy'))
    num_words, embedding_dim = embedding_matrix.shape
    embedding_layer = Embedding(num_words,
                                embedding_dim,
                                embeddings_initializer=Constant(embedding_matrix),
                                trainable=False)
    return embedding_layer

## BERT LAYER

class BertLayer(Layer):
    def __init__(
        self,
        n_fine_tune_layers=10,
        pooling="first",
        bert_path="https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1",
        **kwargs,
    ):
        self.n_fine_tune_layers = n_fine_tune_layers
        self.trainable = True
        self.output_dim = 768
        self.pooling = pooling
        self.bert_path = bert_path
        if self.pooling not in ["first", "mean"]:
            raise NameError(
                f"Undefined pooling type (must be either first or mean, but is {self.pooling}"
            )

        super(BertLayer, self).__init__(**kwargs)

    def build(self, input_shape):
        self.bert = hub.Module(
            self.bert_path, trainable=self.trainable, name=f"{self.name}_module"
        )

        # Remove unused layers
        trainable_vars = self.bert.variables
        if self.pooling == "first":
            trainable_vars = [var for var in trainable_vars if not "/cls/" in var.name]
            trainable_layers = ["pooler/dense"]

        elif self.pooling == "mean":
            trainable_vars = [
                var
                for var in trainable_vars
                if not "/cls/" in var.name and not "/pooler/" in var.name
            ]
            trainable_layers = []
        else:
            raise NameError(
                f"Undefined pooling type (must be either first or mean, but is {self.pooling}"
            )

        # Select how many layers to fine tune
        for i in range(self.n_fine_tune_layers):
            trainable_layers.append(f"encoder/layer_{str(11 - i)}")

        # Update trainable vars to contain only the specified layers
        trainable_vars = [
            var
            for var in trainable_vars
            if any([l in var.name for l in trainable_layers])
        ]

        # Add to trainable weights
        for var in trainable_vars:
            self._trainable_weights.append(var)

        for var in self.bert.variables:
            if var not in self._trainable_weights:
                self._non_trainable_weights.append(var)

        super(BertLayer, self).build(input_shape)

    def call(self, inputs):
        inputs = [K.cast(x, dtype="int32") for x in inputs]
        input_ids, input_mask, segment_ids = inputs
        bert_inputs = dict(
            input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids
        )
        if self.pooling == "first":
            pooled = self.bert(inputs=bert_inputs, signature="tokens", as_dict=True)[
                "pooled_output"
            ]
        elif self.pooling == "mean":
            result = self.bert(inputs=bert_inputs, signature="tokens", as_dict=True)[
                "sequence_output"
            ]

            mul_mask = lambda x, m: x * tf.expand_dims(m, axis=-1)
            masked_reduce_mean = lambda x, m: tf.reduce_sum(mul_mask(x, m), axis=1) / (
                    tf.reduce_sum(m, axis=1, keepdims=True) + 1e-10)
            input_mask = tf.cast(input_mask, tf.float32)
            pooled = masked_reduce_mean(result, input_mask)
        else:
            raise NameError(f"Undefined pooling type (must be either first or mean, but is {self.pooling}")

        return pooled

    def compute_output_shape(self, input_shape):
        return (input_shape[0], self.output_dim)

def initialize_vars(sess):
    sess.run(tf.local_variables_initializer())
    sess.run(tf.global_variables_initializer())
    sess.run(tf.tables_initializer())
    K.set_session(sess)

# # MODELS

def apply_attention(inputs):
    input1, input2 = inputs
    outer_product = tf.einsum('ghj, ghk -> gjk', input1, input2)
    return outer_product

def get_model(model_name, max_sequence_length, labels_dims, embedding_layer):
    sequence_input = Input(shape=(max_sequence_length,), dtype='int32')
    embedded_sequences = embedding_layer(sequence_input)
    if model_name == 'xmlcnn':
        filter_sizes = [2,4,8]
        pooling_units = 32
        bottle_neck = 512
        convs = []
        for fsz in filter_sizes:
            l = Conv1D(filters = 128, kernel_size = fsz, strides = 2, activation = 'relu')(embedded_sequences)
            s = int(l.shape[-2])
            pool_size = s//pooling_units
            l = MaxPooling1D(pool_size,padding = 'same')(l)
            l = Flatten()(l)
            convs.append(l)
        x = Concatenate(axis=-1)(convs)
        x = Dense(bottle_neck, activation = 'relu')(x)
        x = Dropout(0.5)(x)
        outs = []
        for i,labels_dim in enumerate(labels_dims):
            outs.append(Dense(labels_dim, activation = None, name = 'H{}'.format(i))(x))
    elif model_name in ['attentionxml','attention']:
        labels_dim = sum(labels_dims)
        if model_name == 'attentionxml':
            # with lstm
            x = Bidirectional(CuDNNLSTM(512, return_sequences=True))(embedded_sequences)
        else:
            # without lstm
            x = embedded_sequences
        attention = Dense(labels_dim,activation=None,name='attention_dense')(x)
        attention = Softmax(axis=1,name='attention_softmax')(attention)
        x = Lambda(apply_attention,name = 'apply_attention')([x, attention])
        x = Lambda(lambda x:K.permute_dimensions(x,(0,2,1)),name='transpose')(x)
        x = TimeDistributed(Dense(512,activation='relu'))(x)
        x = Dropout(0.5)(x)
        x = TimeDistributed(Dense(256,activation='relu'))(x)
        x = Dropout(0.5)(x)
        x = TimeDistributed(Dense(1,activation=None))(x)
        x = Lambda(lambda x:K.squeeze(x,axis=-1))(x)
        outs = []
        start = 0
        for i,labels_dim in enumerate(labels_dims):
            outs.append(Lambda(lambda x:x[:,start:start+labels_dim],name = 'H{}'.format(i))(x))
            start+=labels_dim
    else:
        raise Exception('Invalid model_name : {}'.format(model_name))
    return Model(sequence_input, outs)

def get_bert_model(max_sequence_length, labels_dims, bottle_neck, trainable_layers, sess):
    bert_inputs = [
        Input(shape=(max_sequence_length,), name="input_sequence"),
        Input(shape=(max_sequence_length,), name="input_mask"),
        Input(shape=(max_sequence_length,), name="input_segment"),
    ]
    x = BertLayer(n_fine_tune_layers = trainable_layers, pooling="first")(bert_inputs)
    if bottle_neck:
        x = Dense(bottle_neck, activation='relu')(x)
        x = Dropout(0.5)(x)
    outs = []
    for i,labels_dim in enumerate(labels_dims):
        outs.append(Dense(labels_dim, activation = None, name = 'H{}'.format(i))(x))
    initialize_vars(sess)
    return Model(inputs=bert_inputs, outputs=outs)
# # LOSSES

def binary_cross_entropy_with_logits(y_true, y_pred):
    return K.mean(K.binary_crossentropy(y_true,y_pred,from_logits=True),axis=-1)
def categorical_cross_entropy_with_logits(y_true, y_pred):
    return K.mean(K.categorical_crossentropy(y_true,y_pred,from_logits=True),axis=-1)
def masked_categorical_cross_entropy_with_logits(y_true, y_pred):
    y_pred = tf.where(K.not_equal(y_true, -1), y_pred, -1e7*tf.ones_like(y_pred))
    loss = K.categorical_crossentropy(tf.maximum(y_true,0.) ,y_pred, from_logits=True)
    return K.mean(loss,axis=-1)

# # METRICS

def binary_accuracy_with_logits(y_true, y_pred):
    return K.mean(K.equal(y_true, K.tf.cast(K.less(0.0,y_pred), y_true.dtype)))
def pAt1(y_true,y_pred):
    return categorical_accuracy(y_true, y_pred)
def pAt5(y_true,y_pred):
    return top_k_categorical_accuracy(y_true, y_pred, k=5)

# # MISC

def save_predictions(model,x_test,y_tests,out_dir):
    print('SAVE PREDICTIONS TO : {}'.format(out_dir))
    out_probs = model.predict(x_test,verbose=1)
    if len(y_tests)==1:
        out_probs = [out_probs]
    ind_dirs = [os.path.join(out_dir,'pred_outputs{}.txt'.format(i)) for i in range(len(y_tests))]
    log_dirs = [os.path.join(out_dir,'pred_logits{}.txt'.format(i)) for i in range(len(y_tests))]
    f_ind = [open(ind_dir,'ab') for ind_dir in ind_dirs]
    f_log = [open(log_dir,'ab') for log_dir in log_dirs]
    for i,out_prob in enumerate(out_probs):
        ind = np.argsort(out_prob,axis=1)[:,:-11:-1]
        logits = np.take_along_axis(out_prob, ind, axis=1)
        np.savetxt(f_ind[i],ind,fmt='%d')
        np.savetxt(f_log[i],logits,fmt='%1.3f')
    for f in f_ind + f_log:
        f.close()
def save_hs_predictions(model,x_test,y_tests,out_dir,IN_DIR):
    print('SAVE HS PREDICTIONS TO : {}'.format(out_dir))
    out_logits = model.predict(x_test,verbose=1)
    out_probs = get_cascade_sm(out_logits,IN_DIR)
    if len(y_tests)==1:
        out_probs = [out_probs]
    ind_dirs = [os.path.join(out_dir,'pred_outputs{}.txt'.format(i)) for i in range(len(y_tests))]
    prob_dirs = [os.path.join(out_dir,'pred_probs{}.txt'.format(i)) for i in range(len(y_tests))]
    for i,out_prob in enumerate(out_probs):
        ind = np.argsort(out_prob,axis=1)[:,:-11:-1]
        probs = np.take_along_axis(out_prob, ind, axis=1)
        np.savetxt(ind_dirs[i],ind,fmt='%d')
        np.savetxt(prob_dirs[i],probs,fmt='%1.3f')

# function
def get_cascade_sm(y_preds,IN_DIR):
    child_to_siblings = pickle.load(open(os.path.join(IN_DIR,'child_to_siblings.pkl'),'rb'))
    parent_to_child = pickle.load(open(os.path.join(IN_DIR,'parent_to_child.pkl'),'rb'))
    cascade_sm = [softmax(y_preds[0],axis=1)]
    for i in range(len(y_preds)-1):
        data = []
        row_ind = []
        col_ind = []
        sms = cascade_sm[-1]
        for key,val in parent_to_child[i].items():
            child_sm = softmax(y_preds[i+1][:,val],axis=1)
            data.append(np.multiply(child_sm,sms[:,key,np.newaxis]).reshape(-1))
            row_ind.append(np.repeat(np.arange(sms.shape[0]),len(val)))
            col_ind.append(np.tile(val,sms.shape[0]))
        data = np.concatenate(data)
        row_ind = np.concatenate(row_ind)
        col_ind = np.concatenate(col_ind)
        cascade_sm.append(sp.csr_matrix((data,(row_ind,col_ind))).toarray())
    return cascade_sm


import cv2
import numpy as np

face_classifier = cv2.CascadeClassifier("haar_face.xml")

def face_extractor(img):

    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    face = face_classifier.detectMultiScale(gray,1.3,5)

    if len(face) == 0:
      return None

    for(x,y,w,h) in face:
        cropped_face = img[y:y+h, x:x+w]

    return cropped_face

cap = cv2.VideoCapture(0)
count = 0

while True:
    ret, frame = cap.read()
    faces = face_classifier.detectMultiScale(frame, scaleFactor=1.3, minNeighbors=5)

    if len(faces) > 0:
        count = count + 1
        for (x, y, w, h) in faces:
            cropped_face = frame[y:y+h, x:x+w]
            face = cv2.resize(cropped_face, (200, 200))
            face = cv2.cvtColor(face, cv2.COLOR_BGR2GRAY)

            file_name_path = r"C:\Users\hiii\OneDrive\Desktop\face detection\dataset\\" + str(count) + '.jpg'
            cv2.imwrite(file_name_path, face)

            cv2.putText(frame, str(count), (50, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 2)
            cv2.imshow('Face Cropper', face)

    else:
        print("Face not found")

    if cv2.waitKey(1) == 13 or count == 100:
        break

cap.release()
cv2.destroyAllWindows()
print("Dataset collection completed.")


import sys

import cv2
from PyQt5 import uic, QtGui
from PyQt5.QtCore import QTimer
from PyQt5.QtGui import QImage
from PyQt5.QtWidgets import QApplication, QMainWindow, QFileDialog
import NS, BDview
from BDview import Ui_MainWindow

class MyProject(QMainWindow):
    def __init__(self):
        super().__init__()
        uic.loadUi('Go1.ui', self)
        self.cap = None
        self.cap2 = None
        self.pushButton.clicked.connect(self.getVideo)
        self.pushButton_2.clicked.connect(self.HandleVideo)#!!!!!!
        self.pushButton_3.clicked.connect(self.OpenBD)

        self.timer = QTimer(self)
        self.timer.timeout.connect(self.updateFrame)
        self.timer2 = QTimer(self)
        self.timer2.timeout.connect(self.updateFrame2)
        # BDview.Ui_MainWindow.main(self)

    def OpenBD(self):
        self.BDview = Ui_MainWindow()
        self.BDview.show()

    def HandleVideo(self):
        if self.cap is None:
            self.label.setText("Видео не загружено.")
            return
        NS.main(self.cap)
        self.cap2 = cv2.VideoCapture('yolo/examles_pictor/Video5.avi')
        self.timer2.start(33)

    def getVideo(self):
        path = QFileDialog.getOpenFileName(self,
                                            "Выбрать видео",
                                            "",
                                            "Файлы видео (*)")[0]

        if not path:
            return
        self.cap = cv2.VideoCapture(path)
        self.timer.start(33)

    def updateFrame(self):
        ret, img = self.cap.read()
        if not ret:
            self.timer.stop()
            self.cap.release()
            return
        h, w, ch = img.shape
        if h > w:
            w = int(w * 501 / h)
            h = 501
        else:
            h = int(h * 501 / w)
            w = 501
        img = cv2.resize(img, (w, h))
        bpl = 3 * w
        qImg = QImage(img.data, w, h, bpl, QImage.Format_BGR888)
        self.label.setPixmap(QtGui.QPixmap.fromImage(qImg))

    def updateFrame2(self):
        ret, img = self.cap2.read()
        if not ret:
            self.timer2.stop()
            self.cap2.release()
            return
        h, w, ch = img.shape
        if h > w:
            w = int(w * 501 / h)
            h = 501
        else:
            h = int(h * 501 / w)
            w = 501
        img = cv2.resize(img, (w, h))
        bpl = 3 * w
        qImg = QImage(img.data, w, h, bpl, QImage.Format_BGR888)
        self.label_2.setPixmap(QtGui.QPixmap.fromImage(qImg))

if __name__ == '__main__':

    app = QApplication(sys.argv)
    ex = MyProject()
    ex.show()
    sys.exit(app.exec_())


#!/usr/bin/python3
# -*- coding:utf-8 -*-
import requests
from flask import Flask, request, jsonify, make_response
from flask_restx import Api, Resource, fields
from werkzeug.middleware.proxy_fix import ProxyFix
app = Flask(__name__)
app.wsgi_app = ProxyFix(app.wsgi_app)
api = Api(app, version='1.0', title='RPA机器人API'
          )
# flask原生api页面
ns = api.namespace('rpa/api', description="RPA机器人")
response_model = api.model('Model', {
    'code': fields.Integer, 'data': fields.String, 'message': fields.String
    # 'code':fields.Integer,'data': fields.String,'message':fields.String,'type':fields.String,'company_id':fields.String
})
aaa = api.namespace('rpaapi', description="存入账号信息")
@aaa.route('/api/', methods=['post'])
@aaa.response(404, '接口调用失败')
@aaa.response(200, 'Success', response_model)
class GuessGender(Resource):
    @aaa.expect(aaa.model('Resource', {
        'type': fields.String(required=True, description='报税类型'),
        'declareId': fields.String(required=True, description='申报ID')
    }), validate=True)
    def post(self):
        '''添加待报税公司id信息信息'''
        todos = request.get_json()
        typeq = todos['type']
        company_id = todos['declareId']
        if company_id == '' and typeq == '':
            response = make_response(
                jsonify({'data': None, "message": '未接收到数据', 'code': 1}))
            return response
        if typeq == '':
            response = make_response(
                jsonify({'data': None, "message": '未接收到类型数据', 'code': 1}))
            return response
        if company_id == '':
            response = make_response(
                jsonify({'data': None, "message": '未接收到公司id数据', 'code': 1}))
            return response
        url = "http://127.0.0.1:2229"
        payload = "{'type':'%s','company_id':'%s'}" % (typeq, company_id)  # 添加待报税公司id信息信息
        print(payload)
        payload = payload.encode('utf-8')
        headers = {
            'Accept-Language':'zh-CN,zh;q=0.9',
        }
        res = requests.request("POST",url,headers=headers,data=payload).text.encode('latin-1').decode('utf8').replace('company_id','declareId')
        response = make_response(jsonify({'data': res, "message": '接口调用成功', 'code': 0}))
        return response
if __name__ == '__main__':
    app.run(host='0.0.0.0', port=48050)


import os

os.environ['http_proxy'] = 'http://192.41.170.23:3128'
os.environ['https_proxy'] = 'http://192.41.170.23:3128'

import torchtext
import torch
from torch import nn
import math
import numpy as np
import torch.optim as optim
# device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
device = 'cuda:1'
print(device)

from torchtext.datasets import IMDB

#training hyperparameters
batch_size = 4
num_epochs = 2
lr=0.0001

from transformers import BertTokenizer, BertForSequenceClassification
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertForSequenceClassification.from_pretrained('bert-base-uncased')

optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)

train_iter = IMDB(split='train')
test_iter = IMDB(split='test')

trainloader = torch.utils.data.DataLoader(train_iter, batch_size=batch_size)
test_loader = torch.utils.data.DataLoader(test_iter, batch_size=batch_size)

label_maps = {"neg":0,"pos":1}

train_loss = []
test_loss = []

# create optimzer
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
model = model.to(device)

for idx, batch in enumerate(trainloader):

    optimizer.zero_grad()
    labels, sample = batch
    #print("labels string  :",labels)

    inputs = tokenizer(sample,padding=True,truncation=True,return_tensors="pt")

    # move parameter to device
    inputs = {k:v.to(device) for k,v in inputs.items()}

    labels = [label_maps[stringtoId] for stringtoId in labels]

    labels = torch.tensor(labels).unsqueeze(0)

    labels = labels.to(device)
    #print("label Id :",labels)

    outputs = model(**inputs,labels=labels)
    loss, logits = outputs[:2]

    loss.backward()
    optimizer.step()

    print("loss :" ,loss.item())

    train_loss.append(loss)

    break

    #print(loss)


# -*- mode: python -*-

import utils
from build import build_app
Import('env')

target = 'reconstructPar'

sources = Split("""
reconstructPar.cpp
""")

caelus_inc = [
'lagrangian/basic/lnInclude',
'parallel/reconstruct/reconstruct/lnInclude',
'regionModels/regionModel/lnInclude',
'dynamicMesh/dynamicMesh/lnInclude'
]

caelus_libs = [
'core',
'lagrangian',
'reconstruct',
'regionModels',
'dynamicMesh'
]

ostype = utils.ostype()
prepend_args = {}
if ostype == 'windows':
   prepend_args = {'LINKFLAGS' :
         '-u genericPatchFieldsLoad'
}

build_app(env,
          target=target,
          sources=sources,
          caelus_inc=caelus_inc,
          caelus_libs=caelus_libs,
          prepend_args=prepend_args)