# -*- coding: utf-8 -*-
"""
Created on Sun Aug  9 16:08:51 2020

@author: Pablo Saldarriaga
"""
import os
import json
import time
import random
import funciones
import numpy as np
import pandas as pd
import datetime as dt
from tqdm import tqdm
import sklearn.metrics as metrics
# %%
# Define el archivo en el que se guararan los logs del codigo
import logging
from logging.handlers import RotatingFileHandler

file_name = "Algoritmo_genetico"
logger = logging.getLogger()
dir_log = f"../logs/{file_name}.log"

# Crea la carpeta de logs
if not os.path.isdir("../logs"):
    os.makedirs("../logs", exist_ok=True)

handler = RotatingFileHandler(dir_log, maxBytes=2000000, backupCount=10)
logging.basicConfig(
    level=logging.DEBUG,
    format="%(asctime)s - %(process)d - %(name)s - %(levelname)s - %(message)s",
    handlers=[handler],
)
# %%
# Esta funcion obtiene la metrica para cada uno de los modelos entrenados
# con las diferentes soluciones

def evaluar_fitness_poblacion(X, Y, X_val, Y_val, population, k=3):

    poblacion = population.copy()
    fitness = []
    best_score = 0
    best_mod = None
    for i in range(len(poblacion)):

        if not "fitness" in poblacion[i].keys():
            _, score = funciones.entrenar_NN(X, Y, X_val, Y_val, k, poblacion[i])
            fitness.append(score)
            poblacion[i]["fitness"] = score
        else:
            score = poblacion[i]["fitness"]
            fitness.append(score)

        if score > best_score:
            best_score = score
            best_mod = poblacion[i]

    fitness = np.array(fitness)

    return poblacion, fitness, best_mod, best_score

def seleccion(poblacion):

    ### Seleccion por metodo de torneo
    ### Definicion de cantidad de individuos a participar en el torneo
    k = 5
    fitness = []
    participantes = random.sample(poblacion, k)

    for i in range(k):
        fitness.append(participantes[i]['fitness'])

    indices_best2worst = np.argsort(fitness)[::-1]

    ### Se seleccionan los dos mejores individuos acorde a su fitness
    padre_1 = participantes[indices_best2worst[0]]
    padre_2 = participantes[indices_best2worst[1]]

    return padre_1, padre_2

def get_bestPoblacion(fitness, poblacion, cant_seleccion):

    indices_best2worst = np.argsort(fitness)[::-1]
    # fitness_best2worst = fitness[indices_best2worst]

    poblacion_best2worst = []
    for ind in indices_best2worst:
        poblacion_best2worst.append(poblacion[ind])

    best_pop = poblacion_best2worst[:cant_seleccion]

    return best_pop

def crossover(X, Y, X_val, Y_val, k, padre_1a, padre_2a, bin_cont = False):
    ### Cruce por dos puntos

    padre_1 = padre_1a.copy()
    padre_2 = padre_2a.copy()

    array_p1 = funciones.sol2array(padre_1, bin_cont)
    array_p2 = funciones.sol2array(padre_2, bin_cont)

    array_cruce1 = list(np.zeros(len(array_p1)))
    array_cruce2 = list(np.zeros(len(array_p1)))

    ### Encuentro los dos puntos en los cuales se hace el cruce
    aa = -1
    bb = -1

    while (aa == bb) or (abs(aa - bb) < 2):
        aa = random.randint(1, len(array_p1) - 2)
        bb = random.randint(1, len(array_p1) - 2)

    a = min(aa, bb)
    b = max(aa, bb)

    for i in range(len(array_p1)):

        if i < a:
            array_cruce1[i] = array_p1[i]
            array_cruce2[i] = array_p2[i]

        elif i < b:
            array_cruce1[i] = array_p2[i]
            array_cruce2[i] = array_p1[i]

        else:
            array_cruce1[i] = array_p1[i]
            array_cruce2[i] = array_p2[i]

    hijo_1 = funciones.array2sol(array_cruce1, bin_cont)
    hijo_2 = funciones.array2sol(array_cruce2, bin_cont)

    _, score_h1 = funciones.entrenar_NN(X, Y, X_val, Y_val, k, hijo_1)
    _, score_h2 = funciones.entrenar_NN(X, Y, X_val, Y_val, k, hijo_2)

    if score_h1 >= score_h2:
        best_h = hijo_1
        best_h['fitness'] = score_h1
    else:
        best_h = hijo_2
        best_h['fitness'] = score_h2

    return best_h

def mutacion(hijo_a, min_lay, max_lay, min_neu, max_neu, min_alpha, max_alpha,
             min_lr, max_lr, escenario=1,bin_cont = False):

    hijo = hijo_a.copy()
    ops = ["layer_sizes", "activation", "alpha", "learning_rate_init"]
    if escenario == 1:
        ### Este escenario selecciona de forma aleatoria uno de los 4 hiperparametros a variar
        gen = random.choice(ops)

        if gen == "layer_sizes":

            layer = funciones.change_layer(hijo, min_lay, max_lay, min_neu, max_neu)

            hijo["layer_sizes"] = tuple(layer)

        elif gen == "activation":

            hijo["activation"] = funciones.change_activation(hijo)

        elif gen == "alpha":
            hijo["alpha"], hijo["alpha_bin"] = funciones.change_alpha(hijo, min_alpha, max_alpha, bin_cont = bin_cont)

        else:
            hijo["learning_rate_init"], hijo["learning_rate_init_bin"] = funciones.change_learningRate(hijo, min_lr, max_lr, bin_cont = bin_cont)
    elif escenario == 2:
        ### Este escenario le asigna una probabilidad de mutacion a cada hiperparametro
        probabilidades = [0.7, 0.6, 0.4, 0.4]

        ### layers
        if random.random() < probabilidades[0]:

            layer = funciones.change_layer(hijo, min_lay, max_lay, min_neu, max_neu)

            hijo["layer_sizes"] = tuple(layer)

        ### activation
        if random.random() < probabilidades[1]:

            hijo["activation"] = funciones.change_activation(hijo)

        ### alpha
        if random.random() < probabilidades[2]:
            hijo["alpha"], hijo["alpha_bin"] = funciones.change_alpha(hijo, min_alpha, max_alpha, bin_cont = bin_cont)

        ### Learning rate
        if random.random() < probabilidades[3]:
            hijo["learning_rate_init"], hijo["learning_rate_init_bin"] = funciones.change_learningRate(hijo, min_lr, max_lr, bin_cont = bin_cont)

    elif escenario == 3:
        ### Este escenario selecciona de forma aleatoria uno de los 4 hiperparametros a variar.
        ### la diferencia con el escenario 1, es que  se le asignan pesos a cada uno de los hiperparametros
        ### por lo que la seleccion no es uniforme

        gen_n = np.random.choice([0, 1, 2, 3], 1, p=[0.6, 0.2, 0.1, 0.1], replace=False)
        gen = ops[gen_n[0]]
        if gen == "layer_sizes":

            layer = funciones.change_layer(hijo, min_lay, max_lay, min_neu, max_neu)

            hijo["layer_sizes"] = tuple(layer)

        elif gen == "activation":

            hijo["activation"] = funciones.change_activation(hijo)

        elif gen == "alpha":
            hijo["alpha"], hijo["alpha_bin"] = funciones.change_alpha(hijo, min_alpha, max_alpha, bin_cont = bin_cont)

        else:
            hijo["learning_rate_init"], hijo["learning_rate_init_bin"] = funciones.change_learningRate(hijo, min_lr, max_lr, bin_cont = bin_cont)

    return hijo

def myFunc(e):
  return e[0]

def update_poblacion(X_train, Y_train, X_val, Y_val, poblacion_prev, fitness, hijos_prev, k, bin_cont = False):

    poblacion = poblacion_prev.copy()
    hijos = hijos_prev.copy()

    N = len(poblacion)

    ### Agrego tanto hijos como individuos de la poblacion a una sola lista
    res = []
    for ind in poblacion:
        res.append([ind['fitness'].copy(),ind.copy()])

    for hijo in hijos:
        res.append([hijo['fitness'].copy(),hijo.copy()])

    ### organizo los individuos por su fitness
    res.sort(reverse = True, key=myFunc)

    N2 = len(res)
    ### inicializo variables para comenzar a actualizar los individuos de la
    ### poblacion buscando que los individuos sean lo mas diferente posible
    ids_check = []
    new_population = []

    ### Agrego a la nueva poblacion el mejor individuo
    ids_check.append(0)
    new_population.append(res[0][1].copy())

    ### Calculo la matriz de distancias de los individuos de la poblacion
    mat_dist = np.zeros(shape = (N2,N2))

    for i in range(N2):
        for j in range(N2):
            sol1 = res[i][1].copy()
            sol2 = res[j][1].copy()

            sol1_arr = funciones.sol2array(sol1, bin_cont = bin_cont)
            sol2_arr = funciones.sol2array(sol2, bin_cont = bin_cont)

            sol1_arr_z = np.array(funciones.estandatizar_sol(sol1_arr))
            sol2_arr_z = np.array(funciones.estandatizar_sol(sol2_arr))

            mat_dist[i,j] = np.linalg.norm(sol1_arr_z-sol2_arr_z)

    ### comenzamos a actualizar la poblacion con base en la matriz de distancias
    for _ in range(1,N):

        ### inicializo diccionario
        dic_cercanos = {}
        for k in range(N2):
            dic_cercanos[k] = 0

        for index in ids_check:
            mas_cercanos = np.argsort(mat_dist[index,:])[1:]

            for v in range(len(mas_cercanos)):
                if mas_cercanos[v] not in ids_check:
                    dic_cercanos[mas_cercanos[v]] = dic_cercanos[mas_cercanos[v]] + v

        max_sum = -1
        a_agregar = 0
        for key in dic_cercanos.keys():
            if max_sum < dic_cercanos[key]:
                max_sum = dic_cercanos[key]
                a_agregar = key

        ids_check.append(a_agregar)
        new_population.append(res[a_agregar][1].copy())

    return new_population, fitness, new_population[0].copy(), new_population[0]['fitness'].copy()

def busqueda_exahustiva(X_train, Y_train, X_val, Y_val, k, best_mod):
    logger.info('Inicia busqueda exahustiva')
    best_sol = best_mod.copy()

    sol_aux = best_mod.copy()
    for f_activation in ["identity", "logistic"]:

        for n_layers in range(1,5):

            if n_layers <= len(sol_aux['layer_sizes']):
                new_layers = sol_aux['layer_sizes'][:n_layers]
            else:
                ones = []
                cant_ones = n_layers - len(sol_aux['layer_sizes'])
                for _ in range(cant_ones):
                    ones.append(1)

                new_layers = tuple([*list(sol_aux['layer_sizes']), *ones])

            new_sol = best_mod.copy()
            new_sol['layer_sizes'] = new_layers
            new_sol['activation'] = f_activation

            _, score = funciones.entrenar_NN(X_train, Y_train, X_val, Y_val, k, new_sol)

            new_sol["fitness"] = score

            if best_sol['fitness']< new_sol["fitness"]:
                best_sol = new_sol.copy()
                logger.info(f'Mejora la solucion con la busqueda exahustiva: {best_sol}')

    return best_sol

def busqueda_local(X_train, Y_train, X_val, Y_val, k, min_alpha, max_alpha,
                                min_lr, max_lr, best_mod, vec_size =5,
                                bin_cont = False):
    ### inicializo soluciones a considerar
    best_sol = best_mod.copy()

    lr = best_sol['learning_rate_init']
    alpha = best_sol['alpha']

    ### discretizo los posibles valores de learning_rate y alpha
    step_size = 20
    h_lr = (max_lr-min_lr)/step_size
    h_alpha = (max_alpha-min_alpha)/step_size

    lr_list = []
    alpha_list = []
    for i in range(step_size+1):
        lr_list.append(min_lr+h_lr*i)
        alpha_list.append(min_alpha+h_alpha*i)

    lr_list = np.array(lr_list)
    alpha_list = np.array(alpha_list)

    ### obtengo binarizacion de las dos variables continuas
    bin_lr = funciones.binarize_parameter(lr, lr_list)
    bin_alpha = funciones.binarize_parameter(alpha, alpha_list)

    if bin_cont:
        logger.info('Entra a bin cont')
        ### creacion del vecindario.
        vecindario = funciones.create_neighborhood(best_mod['learning_rate_init_bin'], best_mod['alpha_bin'], vec_size, bin_cont)
        ### iterar sobre el vecindario
        for vecino in vecindario:
            lr_vecino = vecino[0]
            alpha_vecino = vecino[1]

            new_sol = best_mod.copy()
            new_sol['learning_rate_init_bin'] = lr_vecino
            new_sol['learning_rate_init'] = funciones.bin2cont(lr_vecino, 1)
            new_sol['alpha_bin'] = alpha_vecino
            new_sol['alpha'] = funciones.bin2cont(alpha_vecino, 10)

            ### obtengo el fitness del vecino en la iteracion
            _, score = funciones.entrenar_NN(X_train, Y_train, X_val, Y_val, k, new_sol)
            new_sol["fitness"] = score

            ### conserva el primer vecino que mejore la FO
            if best_sol['fitness']< new_sol["fitness"]:
                best_sol = new_sol.copy()
                logger.info(f'Mejora la solucion con la busqueda local: {best_sol}')
                print('Mejora')
                return best_sol
    else:
        vecindario = funciones.create_neighborhood(bin_lr, bin_alpha, vec_size, bin_cont)
        ### iterar sobre el vecindario
        for vecino in vecindario:
            lr_vecino = vecino[0]
            alpha_vecino = vecino[1]

            new_sol = best_mod.copy()
            new_sol['learning_rate_init'] = lr_list[lr_vecino==1][0]
            new_sol['alpha'] = alpha_list[alpha_vecino==1][0]

            ### obtengo el fitness del vecino en la iteracion
            _, score = funciones.entrenar_NN(X_train, Y_train, X_val, Y_val, k, new_sol)
            new_sol["fitness"] = score

            ### conserva el primer vecino que mejore la FO
            if best_sol['fitness']< new_sol["fitness"]:
                best_sol = new_sol.copy()
                logger.info(f'Mejora la solucion con la busqueda local: {best_sol}')
                print('Mejora')
                return best_sol

    logger.info('NO HAY MEJORA EN EL VECINDARIO')
    return best_sol

# %%

def AG(X_train,
       Y_train,
       X_val,
       Y_val,
       k=3,
       num_generacion=5,
       population_size=10,
       prob_seleccion=0.5,
       prob_mutacion=0.2,
       escenario=1,
       iteracion = 0,
       desc = 'run',
       BH = False,
       LS = False,
       bin_cont = False,
       vec_size = 5,
       threshold_roc = 0.77):

    time_ini = time.time()

    logger.info("***" * 20)
    logger.info("INICIA ALGORITMO GENETICO")
    logger.info("***" * 20)

    # Fijar semillas
    SEED = 42
    random.seed(SEED)
    np.random.seed(SEED)

    # Definicion de parametros del AG
    cant_seleccion = int(population_size * prob_seleccion)

    # Creacion de la poblacion
    min_lay = 1
    max_lay = 4
    min_neu = 1
    max_neu = 128
    min_alpha = 0.0001
    max_alpha = 10
    min_lr = 0.001
    max_lr = 1

    historia_score = []

    poblacion = funciones.crear_soluciones(
        population_size,
        min_lay,
        max_lay,
        min_neu,
        max_neu,
        min_alpha,
        max_alpha,
        min_lr,
        max_lr,
        bin_cont = bin_cont)

    poblacion, fitness, best_mod, best_score = evaluar_fitness_poblacion(X_train, Y_train, X_val, Y_val, poblacion, k=k)

    historia_score.append(best_score)
    last_sol = ''
    for i in tqdm(range(num_generacion)):
        logger.info(f'generacion {i} de {num_generacion}')
        hijos = []
        for j in range(population_size - cant_seleccion):

            padre_1, padre_2 = seleccion(poblacion)
            hijo = crossover(X_train, Y_train, X_val, Y_val, k, padre_1, padre_2, bin_cont = bin_cont)

            if random.random() < prob_mutacion:
                ### solamente se va a mutar un hijo si es malo, de lo contrario
                ### no se muta
                if hijo['fitness']< threshold_roc:
                    logger.info(f'MUTA HIJO NO TAN BUENO')
                    hijo = mutacion(hijo,
                                    min_lay,
                                    max_lay,
                                    min_neu,
                                    max_neu,
                                    min_alpha,
                                    max_alpha,
                                    min_lr,
                                    max_lr,
                                    escenario=escenario,
                                    bin_cont = bin_cont)

            hijos.append(hijo)

        poblacion, fitness, mod, score = update_poblacion(X_train,
                                                          Y_train,
                                                          X_val,
                                                          Y_val,
                                                          poblacion,
                                                          fitness,
                                                          hijos,
                                                          k=k)

        if score > best_score:
            best_score = score
            best_mod = mod.copy()

        ### BUSQUEDA EXAHUSTIVA
        if BH:
            if not last_sol==str(best_mod):
                logger.info(f'Inicia busqueda exahustiva en la generacion al final de la generacion {i}')
                ### consideramos una busqueda exahustiva en la mejor solucion encontrada
                best_sol = busqueda_exahustiva(X_train, Y_train, X_val, Y_val, k, best_mod)

                last_sol = str(best_mod.copy())
                last_mod = best_mod.copy()
                ### actualiza la mejor solucion enconrada
                if best_sol['fitness'] > best_score:

                    logger.info('Encuentra mejor solucion en busqueda exahustiva')
                    best_score = best_sol['fitness']
                    best_mod = best_sol.copy()

                    logger.info('Agrega a la poblacion el mejor individuo de la BH')
                    ### ingresamos a la poblacion el mejor individuo encontrado
                    if not best_mod in poblacion:
                        logger.info(f'AGREGADO:{best_mod}')
                        print(f'AGREGADO:{best_mod}')
                        poblacion[poblacion.index(last_mod)] = best_mod.copy()

            else:
                logger.info(f"YA SE EVALUO LA SOLUCION")
                logger.info(f"ANTES: {last_mod}")
                logger.info(f"AHORA: {best_mod}")
        ### BUSQUEDA LOCAL
        if LS:
            if not last_sol==str(best_mod):
                logger.info(f'Inicia busqueda local en la generacion al final de la generacion {i}')
                ### consideramos una busqueda exahustiva en la mejor solucion encontrada
                best_sol = busqueda_local(X_train, Y_train, X_val, Y_val, k,
                                          min_alpha, max_alpha, min_lr, max_lr,
                                          best_mod, vec_size = vec_size, bin_cont = bin_cont)

                last_sol = str(best_mod.copy())
                last_mod = best_mod.copy()
                ### actualiza la mejor solucion enconrada
                if best_sol['fitness'] > best_score:

                    logger.info('Encuentra mejor solucion en busqueda local')
                    best_score = best_sol['fitness']
                    best_mod = best_sol.copy()

                    logger.info('Agrega a la poblacion el mejor individuo de LS')
                    if not best_mod in poblacion:
                        logger.info(f'AGREGADO:{best_mod}')
                        print(f'AGREGADO:{best_mod}')
                        ### ingresamos a la poblacion el mejor individuo encontrado
                        poblacion[poblacion.index(last_mod)] = best_mod.copy()
    #                    poblacion, fitness, mod, score = update_poblacion(X_train,
    #                                                                      Y_train,
    #                                                                      X_val,
    #                                                                      Y_val,
    #                                                                      poblacion,
    #                                                                      fitness,
    #                                                                      [best_mod],
    #                                                                      k=k)
            else:
                logger.info(f"YA SE EVALUO LA SOLUCION")
                logger.info(f"ANTES: {last_mod}")
                logger.info(f"AHORA: {best_mod}")

        historia_score.append(best_score)
        logger.info(f'Historia: {historia_score}')

    bes_mod_entrenado, _ = funciones.entrenar_NN(X_train, Y_train, X_val, Y_val, k, best_mod)

    time_exec = time.time() - time_ini

    ### Guardo resultados
    if bin_cont:
        best_mod['learning_rate_init_bin'] = str(list(best_mod['learning_rate_init_bin']))
        best_mod['alpha_bin'] = str(list(best_mod['alpha_bin']))
    else:
        best_mod['learning_rate_init_bin'] = str([])
        best_mod['alpha_bin'] = str([])

    res = {
        'population_size': population_size,
        'num_generaciones': num_generacion,
        'prob_selecicon': prob_seleccion,
        'prob_mutacion': prob_mutacion,
        'best_score': best_score,
        'best_mod': best_mod,
        'execution_time_mins': time_exec,
        'history': historia_score
    }

    now = dt.datetime.now().strftime('%Y-%m-%d_%H_%M_%S')
    with open(f'../data/output/output_{escenario}_{iteracion}_{now}_{desc}.json', 'w') as json_file:
        json.dump(res, json_file)

    logger.info("***" * 20)
    logger.info(f"FIN ALGORITMO GENETICO: tiempo {round(time_exec,4)} mins")
    logger.info("***" * 20)

    logger.info("La mejor solucion fue")
    logger.info(str(best_mod))

    return best_mod, best_score, bes_mod_entrenado, historia_score

## %%
#if __name__ == "__main__":
#
#    logger.info("CARGA DE DATOS")
#    # Carga la informacion para entrenar el modelo
#    train = pd.read_csv("../data/train_z.csv").drop(columns=["TW", "BARRIO"])
#    validation = pd.read_csv("../data/validation_z.csv").drop(columns=["TW", "BARRIO"])
#    test = pd.read_csv("../data/test_z.csv").drop(columns=["TW", "BARRIO"])
#
#    # Divide conjunto de datos en entrenamiento y en prueba
#    X_train = train.drop(columns="Accidente")
#    Y_train = train["Accidente"]
#
#    X_val = validation.drop(columns="Accidente")
#    Y_val = validation["Accidente"]
#
#    X_test = test.drop(columns="Accidente")
#    Y_test = test["Accidente"]
#
#    # Realiza la ejecucion del algoritmo genetico
#    best_mod, best_score, bes_mod_entrenado, historia_score = AG(X_train,
#                                                                  Y_train,
#                                                                  X_val,
#                                                                  Y_val,
#                                                                  num_generacion=5,
#                                                                  population_size=10,
#                                                                  prob_seleccion=0.5,
#                                                                  prob_mutacion=0.2,
#                                                                  escenario=1,
#                                                                  BH = False,
#                                                                  LS = True,
#                                                                  bin_cont = True)
#
#    # realiza las predicciones en el conjutno de prueba
#    proba_test = bes_mod_entrenado.predict_proba(X_test)[:, 1]
#    preds_test = bes_mod_entrenado.predict(X_test)
#
#    # Calcula las metricas del modelo en el conjunto de prueba
#    PR_auc = funciones.precision_recall_auc_score(Y_test, proba_test)
#    ROC_auc = metrics.roc_auc_score(Y_test, proba_test)
#
#    bAccuracy = metrics.balanced_accuracy_score(Y_test, preds_test)
#    precision = metrics.precision_score(Y_test, preds_test)
#    recall = metrics.recall_score(Y_test, preds_test)
#    fscore = metrics.f1_score(Y_test, preds_test)
#
#    logger.info("Desempeño en el conjunto de prueba:")
#    logger.info(f"PR-AUC:{PR_auc}")
#    logger.info(f"ROC-AUC:{ROC_auc}")
#    logger.info(f"Balanced accuracy:{bAccuracy}")
#    logger.info(f"Precision:{precision}")
#    logger.info(f"Recall:{recall}")
#    logger.info(f"F1 Score:{fscore}")

# %%


# scroll.py

from source.common import scroll
from source.maps import create_field_matrix

import curses

def main(screen):
    curses.curs_set(0)
    world = create_field_matrix(100, 50, '#')
    w, h = len(world[0]), len(world)
    px, py = 0, 0

    while True:
        camera_x = scroll(px, 80, w)
        camera_y = scroll(py, 25, h)
        for y in range(camera_y, camera_y + 25):
            for x in range(camera_x, camera_x + 80):
                c = world[y][x]
                try:
                    screen.addch(y - camera_y, x - camera_x, c)
                except:
                    screen.insch(y - camera_y, x - camera_x, c)

        try:
            screen.addch(py - camera_y, px - camera_x, '@')
        except:
            screen.insch(px - camera_y, py - camera_x, '@')

        screen.refresh()
        c = screen.getch()
        if c == ord('q') or c == 27:
            break
        elif c == curses.KEY_UP:
            py -= 1
        elif c == curses.KEY_DOWN:
            py += 1
        elif c == curses.KEY_LEFT:
            px -= 1
        elif c == curses.KEY_RIGHT:
            px += 1
        px = min(99, max(0, px))
        py = min(49, max(0, py))

if __name__ == "__main__":
    curses.wrapper(main)


import pytest

from gelanis import Row
from gelanis.sql._ast.ast_to_python import parse_expression
from gelanis.sql.types import IntegerType, StructField, StructType

ROW = Row(a=1, b=2, c=3)
SCHEMA = StructType([
    StructField("a", IntegerType()),
    StructField("b", IntegerType()),
])

SCENARIOS = {
    'Least(-1,0,1)': ('least', 'least(-1, 0, 1)', -1),
    'GREATEST(-1,0,1)': ('greatest', 'greatest(-1, 0, 1)', 1),
    'shiftRight ( 42, 1 )': ('shiftright', 'shiftright(42, 1)', 21),
    'ShiftLeft ( 42, 1 )': ('shiftleft', 'shiftleft(42, 1)', 84),
    "concat_ws('/', a, b )": ('concat_ws', 'concat_ws(/, a, b)', "1/2"),
    'instr(a, a)': ('instr', 'instr(a, a)', 1),  # rely on columns
    'instr(a, b)': ('instr', 'instr(a, b)', 0),  # rely on columns
    "instr('abc', 'c')": ('instr', 'instr(abc, c)', 3),  # rely on lit
}

@pytest.mark.parametrize('string, expected', SCENARIOS.items())
def test_functions(string, expected):
    operator, expected_parsed, expected_result = expected
    actual_parsed = parse_expression(string, True)

    assert expected_parsed == str(actual_parsed)
    assert operator == actual_parsed.pretty_name

    actual_result = actual_parsed.eval(ROW, SCHEMA)

    assert expected_result == actual_result


#!/usr/bin/env python
# encoding: utf-8
from __future__ import unicode_literals

import re

from markdown.preprocessors import Preprocessor

class NormalizeImg(Preprocessor):
    """ add forgotten {attach}. """

    def run(self, lines):
        source = '\n'.join(lines)
        source = re.sub(r'\!\[([^\]]*)\]\(images\/([^)]*)\)', '![\g<1>]({attach}images/\g<2>)', source)
        return source.split('\n')

class NormalizeLink(Preprocessor):
    """ add forgotten {filename}. """

    def run(self, lines):
        source = '\n'.join(lines)
        source = re.sub(r'\[(.*)\]\((.*\.md)\)', '[\g<1>]({filename}\g<2>)', source)
        return source.split('\n')


from setuptools import setup, find_packages
import puanchen

with open("README.md", "r") as f:
    long_description = f.read()

setup(
    name=puanchen.__name__,
    version=puanchen.__version__,
    description="a Python implementation of message queue based on pika",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/perfeelab/puanchen",
    author=puanchen.__author__,
    author_email=puanchen.__author_email__,
    packages=find_packages(),
    include_package_data=True,
    install_requires=["pika>=1.0.1"],
    classifiers=[
        "Programming Language :: Python :: 2",
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ]
)


from django.contrib.auth.decorators import login_required
from django.contrib.auth.mixins import LoginRequiredMixin
from django.db.models import Q
from django.http import JsonResponse
from django.shortcuts import render
from django.views.generic import ListView

from PlayFilms import settings
from PlayFilms.mixins import premium_required, PremiumRequiredMixin
from catalogue.models import Content, Episode

# Create your views here.
from userprofiles.models import History

def mejor_calificados():
    return Content.objects.all().order_by('-score')[0:5]

def cargar_info_usuario(request):
    is_auth = False
    username = None
    if request.user.is_authenticated():
        is_auth = True
        username = request.user.username
    data = {
        'is_auth': is_auth,
        'username': username,
    }
    return data

def buscar(request):
    content = None
    name = None
    es_busqueda = False
    if request.method == 'GET':
        name = request.GET.get('q', None)
        if name is not None:
            print(name)
            es_busqueda = True
            content = Content.objects.filter(
                Q(title__contains=name) | Q(genre__name__startswith=name) | Q(genre__name__endswith=name)).distinct()
    if name is None:
        content = Content.objects.all()
    return {'contenido': content, 'es_busqueda': es_busqueda, 'parametro': name}

@login_required(login_url='/user/login/')
@premium_required
def content_list(request):
    data = cargar_info_usuario(request)
    content = buscar(request)
    return render(request, 'catalogo.html', {'catalogue': content['contenido'],
                                             'data': data,
                                             'es_busqueda': content['es_busqueda'],
                                             'parametro': content['parametro']})

class ContentListView(PremiumRequiredMixin, LoginRequiredMixin, ListView):
    model = Content
    context_object_name = 'catalogue'
    template_name = 'catalogo.html'
    login_url = '/user/login'

    def get_queryset(self):
        order = self.request.GET.get('orden', 'title')
        filtro = int(self.request.GET.get('filtro', 1))
        if filtro == 1:
            queryset = self.model.objects.order_by(order)
        else:
            queryset = self.model.objects.order_by(order).reverse()
        return queryset

    def get_context_data(self, **kwargs):
        context = super(ContentListView, self).get_context_data(**kwargs)
        data = cargar_info_usuario(self.request)
        context['data'] = data
        return context

@login_required(login_url='/user/login/')
@premium_required
def movies_view(request):
    data = cargar_info_usuario(request)
    content = Content.objects.filter(type_of_content=1)
    return render(request, 'catalogo.html', {'catalogue': content, 'data': data})

@login_required(login_url='/user/login/')
@premium_required
def series_view(request):
    data = cargar_info_usuario(request)
    content = Content.objects.filter(type_of_content=2)
    return render(request, 'catalogo.html', {'catalogue': content, 'data': data})

@login_required(login_url='/user/login/')
@premium_required
def film_detail(request, pk):
    film = Content.objects.get(pk=pk)
    data = cargar_info_usuario(request)
    temp_user = request.user.userprofile
    content = mejor_calificados()
    if History.objects.filter(user_profile=temp_user, content=film, is_favorite=True).exists():
        data['fav_submit'] = 'Quitar de favoritos'
        data['class_fav'] = 'btn-danger'
    else:
        data['fav_submit'] = 'Agregar a favoritos'
        data['class_fav'] = 'btn-default'
    try:
        aux_score = History.objects.get(user_profile=temp_user, content=film)
    except History.DoesNotExist:
        aux_score = None

    if aux_score is not None:
        data['mi_puntaje'] = aux_score.score

    return render(request, 'pelicula.html', {'film': film, 'data': data, 'content': content})

@login_required(login_url='/user/login/')
@premium_required
def serie_detail(request, pk):
    serie = Content.objects.get(pk=pk)
    data = cargar_info_usuario(request)
    content = mejor_calificados()
    temp_user = request.user.userprofile
    if History.objects.filter(user_profile=temp_user, content=serie, is_favorite=True).exists():
        data['fav_submit'] = 'Quitar de favoritos'
        data['class_fav'] = 'btn-danger'
    else:
        data['fav_submit'] = 'Agregar a favoritos'
        data['class_fav'] = 'btn-default'

    try:
        aux_score = History.objects.get(user_profile=temp_user, content=serie)
    except History.DoesNotExist:
        aux_score = None

    if aux_score is not None:
        data['mi_puntaje'] = aux_score.score

    return render(request, 'serie.html', {'serie': serie, 'data': data, 'content': content})

@login_required(login_url='/user/login/')
@premium_required
def episode_detail(request, pk):
    episode = Episode.objects.get(pk=pk)
    data = cargar_info_usuario(request)
    content = mejor_calificados()
    return render(request, 'episode.html', {'episode': episode, 'data': data, 'content': content})

def agregar_favorito_view(request):
    data = {'error': 'error'}
    if request.method == 'GET':
        pk = int(request.GET.get('film', None))
        if pk is not None:
            film = Content.objects.get(id=pk)
            temp_user = request.user.userprofile
            if temp_user.favorites.filter(id=pk).exists():
                history = History.objects.get(user_profile=temp_user, content=film)
                history.is_favorite = not history.is_favorite
                data = {'favorito': history.is_favorite}
                history.save()
            else:
                data = {'favorito': True}
                history = History.objects.create(user_profile=temp_user, content=film, is_favorite=True)
                history.save()
    return JsonResponse(data)

def puntuar_view(request):
    data = {'ok': False}
    if request.method == 'GET':
        pk = int(request.GET.get('film', None))
        score = int(request.GET.get('calificacion', None))
        if (pk and score) is not None:
            film = Content.objects.get(id=pk)
            temp_user = request.user.userprofile
            if temp_user.favorites.filter(id=pk).exists():
                history = History.objects.get(user_profile=temp_user, content=film)
                history.score = score
                history.save()
            else:
                history = History.objects.create(user_profile=temp_user, content=film, score=score)
                history.save()
            data['puntaje'] = Content.objects.get(id=pk).score
            data['ok'] = True
    return JsonResponse(data)

class GenreListView(PremiumRequiredMixin, LoginRequiredMixin, ListView):
    model = Content
    context_object_name = 'catalogue'
    template_name = 'catalogo.html'
    login_url = '/user/login'

    def get_queryset(self):
        if self.kwargs.get('genre'):
            queryset = self.model.objects.filter(genre__name__contains=self.kwargs['genre'])
        else:
            queryset = super(GenreListView, self).get_queryset()
        return queryset

    def get_context_data(self, **kwargs):
        context = super(GenreListView, self).get_context_data(**kwargs)
        data = cargar_info_usuario(self.request)
        context['data'] = data
        return context


import numpy as np
import matplotlib.pyplot as plt

x, y= np.mgrid[-5:5:201j, -5:5:101j]

z = np.zeros_like(x)

for n in range(10):
    x0, y0 = np.random.randn(2)
    z += np.exp( -(x-x0)**2 - (y-y0)**2)

plt.pcolor(x, y, z, cmap=plt.cm.Reds)
plt.axis('tight')
plt.show()

fig = plt.figure(figsize=(9,4))
ax = fig.add_axes([0.1, 0.1, 0.8, 0.8])
pc = ax.pcolor(x, y, z, cmap=plt.cm.Reds)

cb = plt.colorbar(pc)
cb.set_label(r'Energy [m$^3$ s$^{-2}$]')

ax.set_xlabel('Distance [km]')
ax.set_ylabel('Distance [km]')
ax.set_title('one plot to rule them all')
ax.set_aspect(1.0)
ax.set_ylim(-5, 5)
ax.set_xlim(-5, 5)

plt.show()


# -*- coding: utf-8 -*-
__author__ = 'kyota'
import utilities
import data_utilities
import numpy.linalg
import mysql.connector
import numpy as np
# import scipy
from scipy import linalg
import math
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import social_force
import logging
import cProfile

class Kriging:
    def __init__(self, prediction_person_id=None, nagget=0, sill=5, range=5, keys=[], all_Z=None):
        ### see nagget, range, sill figure explanation.
        ### https://drive.google.com/file/d/0B0sVf5AmJhwJSlhVT3U5RWhlZ0E/view?usp=sharing
        self.nagget = nagget    ### offset
        self.sill     = sill    ### steady state value, it can be got calculating variance of vecZ[i].
        self.range     = range    ### x-range of zero to steady state.
        self.prediction_person_id = prediction_person_id
        self.keys     = keys
        self.next_prediction_point = self.empty()
        self.prediction_point = self.empty()
        self.variograms = []    # [variogram, distance]
        self.last_prediction_point = None
        self.all_Z = all_Z
        self.real_point = self.empty()

    def krige_all_place(self, vecZ, point_range=(0, 100), x_range=range(-8, 14), y_range=range(-4, 14),\
                          prediction_type='v', destination=0, plot_type="all", frame=-1):
        predicted_points = {}
        # xmax = x_range[-1]
        for y in y_range:
            for x in x_range:
                hash = self.empty()
                hash['x'], hash['y'] = x, y
                self.prediction_point = hash
                prediction_point, vec_omega =  self.krige(vecZ)
                # self.matC, self.vecD, invC = self.get_matrixes(vecZ, self.prediction_point)
                # vec_omega = self.get_weight(self.vecD, invC)
                # prediction_point = self.predict(self.prediction_point, vec_omega, prediction_type)
                prediction_point['x'], prediction_point['y'] = x, y
                predicted_points[y*(100) + x] = prediction_point
        return predicted_points

    ##### return vec_omega, lamda
    ##### args: C=np.ndarray(), D=np.ndarray
    ##### cf : kriging example https://drive.google.com/file/d/0B0sVf5AmJhwJd2VaRDNoU0UwWm8/view?usp=sharing
    ##### point_selection は 'predicted': 前回の予測点を使う, 'real': 実際の点を使う.
    ##### support_persons -> 'current': use current persons only, 'past': use current and past persons both
    def krige(self, _vecZ, Z=None, prediction_type='v', point_selection='predicted', prediction_point=None,
              tracked_prediction_point=None, support_persons='current', use_self_flag=False, frame=None):
        """
        :param _vecZ:
        :param Z:
        :param prediction_type:
        :param point_selection:
        :param prediction_point: 予測対象の現在の x,y座標 を持っている. 将来の地点へ行く速度の予測を行う.
        :param support_persons:
        :param use_self_flag:
        :param frame:
        :return:
        """
        persons = _vecZ.copy()
        self.prediction_point, real_point, other_persons = self.get_prediction_point(
            persons, self.prediction_person_id, tracked_prediction_point, point_selection=point_selection,
            use_self_flag=use_self_flag)
        # self.next_point = self.get_prediction_point(next_vecZ, self.prediction_person_id, self.prediction_point)

        if not self.prediction_point or other_persons.__len__() == 0:
            return self.empty(), self.empty(), np.zeros(len(other_persons)), other_persons, {}
        ### get invC, one, oneT, vecD
        if (support_persons == 'current'):
            self.matC, self.vecD, invC = self.get_matrixes(other_persons, self.prediction_point)
        else:
            support_persons = self.get_support_people(self.prediction_point, Z, frame)
            self.matC, self.vecD, invC = self.get_matrixes(support_persons, self.prediction_point)

        vec_omega = self.get_weight(self.vecD, invC)
        dict_omega = self.get_dict_omega(vec_omega, other_persons)
        ### クリギングによる予測
        prediction_point = self.predict(self.prediction_point, vec_omega, other_persons, prediction_type)
        self.last_prediction_point = prediction_point ### 次の周で参照のために保存
        return prediction_point, real_point, vec_omega, other_persons, dict_omega

    # 信頼度の入力を得る. 同一方向に向かう人物の数, range内の予測に使える人の数, range内の人の近さ.

    # 予測に使用する人物との距離を得る.
    def get_input_distances(self, other_persons, prediction_point):
        distances = []
        for person in other_persons.values():
            distance = utilities.get_distance(prediction_point, person)
            distances.append(distance)
        return distances

    def get_support_people(self, prediction_person, Z, frame):
        support_people = {}
        if (Z == None):
            return {}
        # _Z = Z[0:50]
        _Z = Z[(frame - 100):frame]
        for frame, vecZ in enumerate(_Z):
            for person_id, person in vecZ.iteritems():
                ## key は一意になれば何でも良い
                support_people[person_id * 10000 + frame ] = person
        return support_people

    def get_distance_summary(self, distances):
        in_range_num     = 0
        people_num         = distances.__len__()
        out_range_num     = 0
        for distance in distances:
            if (distance < self.range):
                in_range_num += 1
        if (in_range_num is 0 and people_num is not 0):
            ### range に含まる人がいないときの range外の人の数.
            out_range_num = people_num
        return people_num, in_range_num, out_range_num

    ### plot用に重みをdictに入れる.
    def get_dict_omega(self, vec_omega, persons):
        logging.debug("in get_dict_omega: persons = "+str(persons))
        dict_omega = {}
        for i, person_id in enumerate(self.vecZ.keys()):
            dict_omega[person_id] = vec_omega[i]
        return dict_omega

    ### get vector of omega which is weight of points.
    def get_weight(self, vecD, invC):
        oneT = np.ones((self.vecD.size))
        one = oneT[np.newaxis].T
        ### lambda = (D.T*invC*ones - 1)*(ones.T*invC*ones)
        lamda = (np.dot(np.dot(self.vecD.T, invC), one) - 1) / np.dot(np.dot(oneT, invC), one)
        ### omegas = invC*(vecD - lambda*ones)
        ### ∵ σ2E
        ###     = E{(Z0(r) − ˆ Z0(r))2}2
        ###     = V {Z0(r) − ˆ Z0(r)}
        ###     = C_00 + w.T*C*w − 2w.T*D
        vec_omega = np.dot(invC, (self.vecD - lamda * one)).flatten()
        logging.debug("in get_weight: omega = "+str(vec_omega))
        return vec_omega

    def get_matrixes(self, vecZ, prediction_point):
        " get kriging matrixes vecD, invC"
        sample_size = vecZ.__len__()
        logging.debug("in get_matrixes: vecZ = "+str(vecZ))
        self.vecZ = vecZ
        self.matC = self.make_matC(vecZ)
        logging.debug("in get_matrixes: matC = "+str(self.matC))
        self.vecD = self.make_vecD(vecZ, prediction_point)
        logging.debug("in get_matrixes: vecD = "+str(self.vecD))
        invC = self.matC
        try:
            invC = linalg.pinv(self.matC)
        except np.linalg.LinAlgError as e:
            print(e)
        except ValueError as e:
            print(e)
        return self.matC, self.vecD, invC

    def empty(self):
        return dict.fromkeys(self.keys, 0)

    @classmethod
    def get_dest_tablename(self, destination):
        if (destination < 0):
            table = "person_destination_minus"
        elif (destination == 0):
            table = "person_destination_both"
        elif (destination > 0):
            table = "person_destination_plus"
        return table

    def get_dest_dirname(self, destination, data_set):
        " データ出力するディレクトリを返す, destination, data_set によって決まる.  "
        if (destination < 0):
            dest_dir = "destination_minus"
        elif (destination == 0):
            dest_dir = "destination_both"
        elif (destination > 0):
            dest_dir = "destination_plus"
        dest_dir = data_set+'_'+dest_dir
        return dest_dir

    def predict(self, ex_prediction_point, vec_omega, other_persons, type='v'):
        prediction = self.empty()
        for i, hash in enumerate(self.vecZ.values()):
            for j, k in enumerate(self.keys):
                prediction[k] += hash[k] * vec_omega[i]
            try:
                if (math.isnan(vec_omega[0])):
                    raise IndexError("vec_omega has NaN.")
            except IndexError as e:
                if (self.last_prediction_point):
                    prediction['vx'] = self.last_prediction_point['vx']
                    prediction['vy'] = self.last_prediction_point['vy']
                else:
                    prediction['vx'] = self.prediction_point['vx']
                    prediction['vy'] = self.prediction_point['vy']
    # print 'vx = '+ str(prediction['vx']) + '\tvy = ' + str(prediction['vy'])
        prediction = self.set_velocity(prediction)
        # prediction = self.direction_adjust(ex_prediction_point, prediction)
        prediction['person_id'] = ex_prediction_point['person_id']
        if not ex_prediction_point:
            return self.empty()
        prediction['x'] = ex_prediction_point['x'] # 移動前なので x, y は変化させない。
        prediction['y'] = ex_prediction_point['y']
        if (type == 'dv'):
            # 二階微分使用時
            prediction['vx'] = ex_prediction_point['vx']
            prediction['vy'] = ex_prediction_point['vy']
        return prediction

    def move_point(self, predicted_point, type='v'):
        next_point = self.empty()
        if (type=='v'):
            next_point['vx'] = predicted_point['vx']
            next_point['vy'] = predicted_point['vy']
            next_point['x']= predicted_point['x'] + predicted_point['vx']
            next_point['y']= predicted_point['y'] + predicted_point['vy']
        elif (type=='dv'):
            next_point['vx'] = predicted_point['vx'] + predicted_point['dvx']
            next_point['vy'] = predicted_point['vy'] + predicted_point['dvy']
            next_point['x']= predicted_point['x'] + next_point['vx']
            next_point['y']= predicted_point['y'] + next_point['vy']
        self.prediction_point = next_point
        return next_point

    def set_velocity(self, prediction):
        vx, vy = prediction['vx'], prediction['vy']
        velocity, theta = data_utilities.get_v_theta(vx, vy)
        prediction['velocity'], prediction['theta'] = velocity, theta
        return prediction

    def direction_adjust(self, ex_prediction_point, prediction_point):
        cnx = mysql.connector.connect(user='kyota', host='localhost', database='kriging')
        cursor = cnx.cursor()
        # y, x = int(prediction_point['x']), int(prediction_point['y'])
        y, x = math.floor(prediction_point['x']), math.floor(prediction_point['y'])
        table = 'vx_plus' if (prediction_point['vx'] > 0) else 'vx_minus'
        query     = ("select `mean(dtheta)`, `std(dtheta)` from "+table+" where `floor(y)` = "+str(y)+" and    `floor(x)` = "+str(x)+" ; ")
        cursor.execute(query)
        row = cursor.fetchall()
        if (len(row) != 0):
            dtheta_avg, dtheta_std = float(row[0][0]), float(row[0][1])
            if (dtheta_avg > 0):
                dtheta_threshold = (dtheta_avg + 1.97*dtheta_std)*(math.pi/180)
            else:
                dtheta_threshold = (dtheta_avg - 1.97*dtheta_std)*(math.pi/180)
        else:
            dtheta_threshold = 30*(math.pi/180)
        cos_threshold = math.cos(dtheta_threshold)
        theta_diff = prediction_point['theta'] - ex_prediction_point['theta']
        if math.cos(theta_diff) > cos_threshold:
            return prediction_point
        ## adjust theta
        ### if current theta > ex theta
        if math.sin(theta_diff) > 0:
            prediction_point['theta'] = ex_prediction_point['theta'] + dtheta_threshold
        else:
            prediction_point['theta'] = ex_prediction_point['theta'] - dtheta_threshold
        prediction_point = data_utilities.set_vx_vy(prediction_point)
        return prediction_point

    def get_prediction_point(self, vecZ, prediction_person_id, tracked_prediction_point, point_selection='predicted', use_self_flag=False):
        """ point_selection は 'predicted': 前回の予測点を使う, 'real': 実際の点を使う. 'given': 与えられた点を使う"""
        ### if vecZ has prediction_person: copy it and delete from vecZ.
        real_point = vecZ.pop(prediction_person_id)
        _prediction_point = real_point  # 前回の予測結果がなければ実測値が入ったまま返る.
        # when current vecZ has target person
        if point_selection == 'predicted':
            if tracked_prediction_point and tracked_prediction_point != self.empty():
                # if previous predicted point exist, overwrite _prediction point with that.
                _prediction_point = tracked_prediction_point
            if use_self_flag:
                # 前回予測(追跡)された値をクリギングの予測に使用する.
                vecZ[prediction_person_id] = _prediction_point
        elif point_selection == 'real': pass
        return _prediction_point, real_point, vecZ

    def make_matC(self, persons):
        sample_size = persons.__len__()
        if (0 < sample_size <= 1):
            return np.zeros((1,1))
        self.matC = np.ndarray((sample_size, sample_size))
        # if (isinstance(persons, list)): ### personsのlist が与えられたらpersonsに分解する.
        #     people = persons
        #     for persons in people:

        for row, hash1 in enumerate(persons.values()):
                    for col, hash2 in enumerate(persons.values()):
                        # val = self.variogram(hash1, hash2, self.nagget, self.sill, self.range)
                        val = self.cosineogram(hash1, hash2, self.nagget, self.sill, self.range)
                        if (math.isnan(val) or math.isinf(val)):
                            self.matC[row][col] = 0
                        else:
                            self.matC[row][col] = val

        return self.matC

    def make_vecD(self, persons, prediction_point):
        sample_size = persons.__len__()
        self.vecD = np.ndarray((sample_size, 1))
        # for i in range(0, self.sample_size):
        ### personsのlist が与えられたらpersonsに分解する.
        # if (isinstance(persons, list)):
        #     people = persons
        #     for persons in people:
        for i, hash in enumerate(persons.values()):
                    # self.vecD[i] = self.variogram(hash, prediction_point, self.nagget, self.sill, self.range)
                    self.vecD[i] = self.cosineogram(hash, prediction_point, self.nagget, self.sill, self.range)
        return self.vecD

    ### return spherical variogram
    def variogram(self, sample1, sample2, nagget, sill, range):
        distance = utilities.get_distance(sample1, sample2)
        # if distance > range:
        #     distance = range
        variogram = nagget + sill * ((3 / 2) * (distance / range) - (1 / 2) * (distance / range) ** 3)
        if distance > range:
            variogram = nagget + sill
        # variogram = sill*(1 - (3/2)*(distance/range) + (1/2)*math.pow(distance/range, 3))
        self.variograms.append([variogram, distance])
        return variogram

    def cosineogram(self, sample1, sample2, nagget, sill, range):
        distance = utilities.get_distance(sample1, sample2)
        variogram = ( (1-nagget) - ( ( (1-nagget) - (sill**2))*((3.0/2.0)*(distance/range) - (1.0/2.0)*((distance/range)**3))))
        if distance > range:
            variogram = nagget + sill
        self.variograms.append([variogram, distance])
        return variogram

    def get_means(self, xdata, ydata):
        dict_y = {}
        for i, x in enumerate(xdata):
            if (not isinstance(dict_y.get(x), list)):
                dict_y[x] = []
            if (not math.isnan(ydata[i])):
                dict_y[x].append(ydata[i])
        xs, ymeans = [], []
        for x, ys in dict_y.iteritems():
            ys = np.array(ys)
            # ymeans[x] = ys.mean()
            ymeans.append(ys.mean())
            xs.append(x)
        return xs, ymeans

    def plot_diffs(self, person_id, velocity_diffs, distances_list, dirname):
        xdata1, xdata2, xdata3, ydata = [], [], [], []
        for i, distances in enumerate(distances_list):
            person_num, in_range_num, out_range_num= self.get_distance_summary(distances)
            x1 = in_range_num
            x2 = person_num
            x3 = out_range_num
            try:
                y = velocity_diffs[i][1]
            except:
                print("y data index error.")
            xdata1.append(x1)
            xdata2.append(x2)
            xdata3.append(x3)
            ydata.append(y)

        utilities.mkdir(dirname + "velocity_diff")
        x1s, ymeans = self.get_means(xdata1, ydata)
        plt.clf()
        plt.scatter(xdata1, ydata)
        plt.scatter(x1s, ymeans, color='r')
        plt.xlabel("in range people num")
        plt.ylabel("difference of velocity")
        plt.savefig(dirname + "velocity_diff/in_range_diff_"+str(person_id)+".png")

        x2s, ymeans = self.get_means(xdata2, ydata)
        plt.clf()
        plt.scatter(xdata2, ydata)
        plt.scatter(x2s, ymeans, color='r')
        plt.xlabel("people num")
        plt.ylabel("difference of velocity")
        plt.savefig(dirname + "velocity_diff/people_diff_"+str(person_id)+".png")
        # plt.close()

        xs, ymeans = self.get_means(xdata3, ydata)
        plt.clf()
        plt.scatter(xdata3, ydata)
        plt.scatter(xs, ymeans, color='r')
        plt.xlabel("out range people num")
        plt.ylabel("difference of velocity")
        plt.savefig(dirname + "velocity_diff/out_range_diff_"+str(person_id)+".png")
        # plt.close()

if __name__ == "__main__":
    main()


from typing import Optional
from zlib import crc32
from datetime import datetime
from busio import I2C
from board import SCL, SDA
import adafruit_mcp9808
from .measurements import Measurement, MeasurementError

I2C_ADDRESSES = [ 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F ]

def enumerate_sensors():
  sensors = []
  bus = I2C(SCL, SDA)
  for i2c_address in I2C_ADDRESSES:
    try:
      sensor = mcp9808(i2c_dev=bus, i2c_addr=i2c_address)
    except (OSError, AttributeError, ValueError) as error:
      # If no device is found, an AttributeError is raised (possibly ValueError if no device at address?)
      print("Error initialising MCP9808 sensor at I2C address {:#x}: {}".format(i2c_address, str(error)))
    else:
      print("Found MCP9808 sensor with ID {:x} at I2C address {:#x}".format(sensor.serial_number, i2c_address))
      sensors.append(sensor)
  return sensors

class mcp9808(adafruit_mcp9808.MCP9808):

  manufacturer = 'Microchip Technology'
  model = 'MCP9808'
  supported_measurements = [Measurement.TEMPERATURE]

  def __init__(self, i2c_dev=None,
               i2c_addr: Optional[int] = I2C_ADDRESSES[0]):
    self.i2c_address = i2c_addr
    super().__init__(i2c_bus=i2c_dev, address=i2c_addr)

  def update_sensor(self):
    try:
      self._temperature = super().temperature
    except ValueError as error:
      raise MeasurementError(str(error))
    else:
      self.timestamp = datetime.now().isoformat(timespec='seconds')

  # Override super class property for temperature
  # because otherwise there's a name clash
  @property
  def temperature(self):
    return self._temperature

  @property
  def id(self):
    """A unique identifier for the device."""
    return "{model:s}--{serial:08x}".format(model=self.model.replace('-',''), serial=self.serial_number).lower()

  @property
  def serial_number(self):
    """The hardware identifier (serial number) for the device."""
    # Doesn't look like the device supports reading a unique
    # identifier, this cobbles something together from the
    # sensor type and I2C address
    unique_string = ''.join("{}{}{:#x}".format(self.manufacturer, self.model, self.i2c_address).lower().split())
    return crc32(unique_string.encode())


# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.macosx-10.12-x86_64/egg/tss/serving.py
# Compiled at: 2018-05-01 15:21:11
import tensorflow as tf
from tensorflow.contrib.saved_model.python.saved_model import reader
from tensorflow.contrib.saved_model.python.saved_model import signature_def_utils
import logging

def load_meta_graph(model_path, tags, graph, signature_def_key=None):
    saved_model = reader.read_saved_model(model_path)
    the_meta_graph = None
    for meta_graph_def in saved_model.meta_graphs:
        if sorted(meta_graph_def.meta_info_def.tags) == sorted(tags):
            the_meta_graph = meta_graph_def

    signature_def_key = signature_def_key or tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
    try:
        signature_def = signature_def_utils.get_signature_def_by_key(the_meta_graph, signature_def_key)
    except ValueError as ex:
        try:
            formatted_key = ('default_input_alternative:{}').format(signature_def_key)
            signature_def = signature_def_utils.get_signature_def_by_key(the_meta_graph, formatted_key)
        except ValueError:
            raise ValueError(('Got signature_def_key "{}". Available signatures are {}. Original error:\n{}').format(signature_def_key, list(the_meta_graph.signature_def), ex))

    input_names = {k:v.name for k, v in signature_def.inputs.items()}
    output_names = {k:v.name for k, v in signature_def.outputs.items()}
    feed_tensors = {k:graph.get_tensor_by_name(v) for k, v in input_names.items()}
    fetch_tensors = {k:graph.get_tensor_by_name(v) for k, v in output_names.items()}
    return (feed_tensors, fetch_tensors)

def get_model(model_base_path, version=None):
    versions = {}
    model_base_path = model_base_path.rstrip('/')
    for version in tf.gfile.ListDirectory(model_base_path):
        version = version.rstrip('/')
        if version.isdigit():
            versions[int(version)] = ('/').join([model_base_path, version])

    if version is None:
        version = max(versions.keys())
    logging.info('Choose model_path %s with version %s' % (model_base_path, version))
    return versions[int(version)]

def load_model(model_base_path, model_version):
    model_path = get_model(model_base_path, model_version)
    with tf.Graph().as_default() as (graph):
        sess = tf.Session(graph=graph)
        tags = [tf.saved_model.tag_constants.SERVING]
        tf.saved_model.loader.load(sess, tags, model_path)
        feed_tensors, fetch_tensors = load_meta_graph(model_path, tags, graph)
        return (graph, feed_tensors, fetch_tensors)

def rename_by_mapping(df, tensor_mapping, reverse=False):
    for name, tensor in tensor_mapping.items():
        renames = (
         name, tensor.op.name)
        if reverse:
            renames = reversed(renames)
        df = df.withColumnRenamed(*renames)

    return df

def tf_serving_with_dataframe(df, model_base_path, model_version=None):
    """

    :param df: spark dataframe, batch input for the model
    :param model_base_path: str, tensorflow saved Model model base path
    :param model_version: int, tensorflow saved Model model version, default None
    :return: spark dataframe, with predicted result.
    """
    import tensorframes as tfs
    g, feed_tensors, fetch_tensors = load_model(model_base_path, model_version)
    with g.as_default():
        df = rename_by_mapping(df, feed_tensors)
        df = tfs.analyze(df)
        df = tfs.map_blocks(fetch_tensors.values(), df)
        df = rename_by_mapping(df, feed_tensors, reverse=True)
        return rename_by_mapping(df, fetch_tensors, reverse=True)


import socket, time, json, datetime, platform, psutil, requests, pprint, uuid
from jira import JIRA

# Setup this variables:
Jira_login = 'MPA2'
Jira_pass = '19QAZ08wsx'
Jira_issue = 'DS7429-122'
Jira_cert_path = 'E:\Work\Shell\_Monitoring System\Server-Monitoring-Script-master\jiraaccelryscom.pem'

def main():

    # Hostname Info
	hostname = socket.gethostname()
	print("Hostname:", hostname)

    # CPU Info
	cpu_count = psutil.cpu_count()
	cpu_usage = psutil.cpu_percent(interval=1)
	print("CPU:\n\tCount:", cpu_count, "\n\tUsage:", cpu_usage)

    # Memory Info
	memory_stats = psutil.virtual_memory()
	memory_total = memory_stats.total
	memory_used = memory_stats.used
	memory_used_percent = memory_stats.percent
	print("Memory:\n\tPercent:", memory_used_percent, "\n\tTotal:", memory_total / 1e+6, "MB", "\n\tUsed:", memory_used / 1e+6, "MB")

    # Disk Info
	disk_info = psutil.disk_partitions()
	print("Disks:")
	disks = []
	for x in disk_info:
		# Try fixes issues with connected 'disk' such as CD-ROMS, Phones, etc.
		try:
			disk = {
				"name" : x.device,
				"mount_point" : x.mountpoint,
				"type" : x.fstype,
				"total_size" : psutil.disk_usage(x.mountpoint).total,
				"used_size" : psutil.disk_usage(x.mountpoint).used,
				"percent_used" : psutil.disk_usage(x.mountpoint).percent
            }

			disks.append(disk)

			print("\tDisk name",disk["name"], "\tMount Point:", disk["mount_point"], "\tType",disk["type"], "\tSize:", disk["total_size"] / 1e+9,"\tUsage:", disk["used_size"] / 1e+9, "\tPercent Used:", disk["percent_used"])
		except:
			print("")

    # Bandwidth Info
	network_stats = get_bandwidth()
	print("Network:\n\tTraffic in:",network_stats["traffic_in"] / 1e+6,"\n\tTraffic out:",network_stats["traffic_out"] / 1e+6)

    # Network Info
	nics = []
	print("NICs:")
	for name, snic_array in psutil.net_if_addrs().items():
        # Create NIC object
		nic = {
            "name": name,
            "mac": "",
            "address": "",
            "address6": "",
            "netmask": ""
        }
        # Get NiC values
		for snic in snic_array:
			if snic.family == -1:
				nic["mac"] = snic.address
			elif snic.family == 2:
				nic["address"] = snic.address
				nic["netmask"] = snic.netmask
			elif snic.family == 23:
				nic["address6"] = snic.address
		nics.append(nic)
		print("\tNIC:",nic["name"], "\tMAC:", nic["mac"], "\tIPv4 Address:",nic["address"], "\tIPv4 Subnet:", nic["netmask"], "\tIPv6 Address:", nic["address6"])

    # Platform Info
	system = {
		"name" : platform.system(),
		"version" : platform.release()
	}
	print("OS:\n\t",system["name"],system["version"])

    # Time Info
	timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S+00:00")
	uptime = int(time.time() - psutil.boot_time())
	print("System Uptime:\n\t",uptime)

    # System UUID
	sys_uuid = uuid.getnode()

    # Set Machine Info
	machine = {
		"hostname" : hostname,
		"uuid" : sys_uuid,
		"system" : system,
		"uptime" : uptime,
		"cpu_count" : cpu_count,
		"cpu_usage" : cpu_usage,
		"memory_total" : memory_total,
		"memory_used" : memory_used,
		"memory_used_percent" : memory_used_percent,
		"drives" : disks,
		"network_up" : network_stats["traffic_out"],
		"network_down" : network_stats["traffic_in"],
		"network_cards": nics,
		"timestamp" : timestamp
		}

	data = json.dumps(machine)
	print("\nData:")
	pprint.pprint(machine, indent=4)

	send_data(machine)

def get_bandwidth():
    # Get net in/out
	net1_out = psutil.net_io_counters().bytes_sent
	net1_in = psutil.net_io_counters().bytes_recv

	time.sleep(1)

    # Get new net in/out
	net2_out = psutil.net_io_counters().bytes_sent
	net2_in = psutil.net_io_counters().bytes_recv

    # Compare and get current speed
	if net1_in > net2_in:
		current_in = 0
	else:
		current_in = net2_in - net1_in

	if net1_out > net2_out:
		current_out = 0
	else:
		current_out = net2_out - net1_out

	network = {"traffic_in" : current_in, "traffic_out" : current_out}
	return network

def send_data(machine):
    # Attempt to send data up to 30 times
	for attempt in range(1):
		try:

			#Jira comment posting
			#jira = JIRA()

			jira_options = {
				'server': 'https://jira.accelrys.com',
				'verify': False,
				'cert': Jira_cert_path
				}

			jira = JIRA(options=jira_options, basic_auth=(Jira_login, Jira_pass))

			Issue_number = Jira_issue;

			issue = jira.issue(Issue_number, fields='comment')

			drives = machine.get("drives")

			comment_body = ('||ServerName||Check Parameter Name||Result value|' + '\n' +
							'|' + machine.get("hostname") + '|cpu_usage|' + str(machine.get("cpu_usage")) + '|' + '\n' +
							'|' + machine.get("hostname") + '|memory_used_percent|' + str(machine.get("memory_used_percent")) + '|' + '\n' +
							'|' + machine.get("hostname") + '|drives.name|' + str(drives[0].get("name")) + '\n'  +
							'|' + machine.get("hostname") + '|drives.percent_used|' + str(drives[0].get("percent_used")) + '|' + '\n' +
							'|' + machine.get("hostname") + '|drives.name|' + str(drives[1].get("name"))  + '\n'  +
							'|' + machine.get("hostname") + '|drives.percent_used|' + str(drives[1].get("percent_used")) + '|' + '\n' +
							'|' + machine.get("hostname") + '|drives.name|' + str(drives[2].get("name"))  + '\n'  +
							'|' + machine.get("hostname") + '|drives.percent_used|' + str(drives[2].get("percent_used")) + '|' + '\n' )

			comment = jira.add_comment(Issue_number, comment_body)

		except OSError as err:
			print("OS error: {0}".format(err))

            # Sleep 1 minute before retrying
			#time.sleep(60)
	else:
        # If no connection established for half an hour, kill script
		exit(0)
while True:
	main()
	print("-----------------------------------------------------------------")
	time.sleep(3)


from __future__ import annotations

import typing as ty
import aurflux
from aurflux.response import Response
import discord
import asyncio
import subprocess

if ty.TYPE_CHECKING:
    import datetime

VERSION = subprocess.check_output(["poetry", "version"]).decode().split(" ")[1]

class Interface(aurflux.AurfluxCog):
    listening_channels = set()

    def route(self):
        @self.aurflux.router.endpoint(":message", decompose=True)
        async def _(message: discord.Message):
            if message.author == self.aurflux.user:
                return
            if isinstance(message.channel, discord.channel.DMChannel):
                embed = discord.Embed(
                    title=f"Pinbot V{VERSION}",
                    description="Use ..help in your server for command details\n" \
                                f"[Click me to add to your server](https://discord.com/oauth2/authorize?client_id={self.aurflux.user.id}&scope=bot&permissions=76816)"
                )
                await Response(embed=embed).execute(aurflux.MessageContext(bot=self.aurflux, message=message))

        @aurflux.CommandCheck.has_permissions(discord.Permissions(manage_guild=True))
        @self._commandeer(name="setup", parsed=False)
        async def setup(ctx: aurflux.MessageContext, args: str):
            """
            setup
            :param ctx:
            :param args:
            :return:
            """
            configs = self.aurflux.CONFIG.of(ctx)
            try:
                print("setup!")
                yield Response("Beginning setup... Type `done` to end setup", delete_after=30)

                header_resp = Response("Please select a pair of channels `#from-channel` to `#to-channel`\n"
                                       "Once there are `max` pinned messages in `#from-channel`, the next pin in `#from-channel` will cause the oldest pin to be converted to an embed and sent in `#to-channel`\n"
                                       # f"You can do this in the future without `{ctx.full_command}` via `{configs['prefix']}map #from #to`"
                                       )
                yield header_resp

                async def check_pair(ev: aurflux.AurfluxEvent) -> bool:
                    m: discord.Message = ev.args[0]
                    print(m)
                    if m.author == m.guild.me:
                        return False
                    if m.author != ctx.author:
                        return False
                    if m.content in ("cancel", "done"):
                        return True
                    if not m.channel_mentions:
                        return False
                    s = aurflux.utils.find_mentions(m.content)
                    return len(s) == 2 or m.content == "done"

                assignment: discord.Message = (await self.aurflux.router.wait_for(":message", check_pair, timeout=45)).args[0]
                if assignment.content == "cancel":
                    yield Response(f"Finished setup {aurflux.utils.EMOJIS['white_check_mark']}")
                    return

                if assignment.content == "done":
                    yield Response(f"Finished setup {aurflux.utils.EMOJIS['white_check_mark']}")
                    return

                from_channel, to_channel = aurflux.utils.find_mentions(assignment.content)
                from_channel, to_channel = self.aurflux.get_channel(from_channel), self.aurflux.get_channel(to_channel)
                # try:
                #     await assignment.delete()
                # except discord.errors.Forbidden:
                #     raise aurflux.errors.BotMissingPermissions(discord.Permissions())
                resp = Response(f"Mapping pins from {from_channel.mention} to embeds in {to_channel.mention}", delete_after=30)
                yield resp

                async with self.aurflux.CONFIG.writeable_conf(ctx.config_identifier) as cfg:
                    cfg["pinmap"] = {**cfg.get("pinmap", {}), from_channel.id: to_channel.id}
                await resp.message.add_reaction(aurflux.utils.EMOJIS["white_check_mark"])

                resp = Response(f"What is the number of native discord pins you would like in {from_channel.mention}? [0-49]\n"
                                f"If you set this to 0, all pins will be immediately turned into embeds\n"
                                f"If you set this to any other number <= 49, {from_channel.mention} will hold that many native discord pins, "
                                f"after which the **oldest** pin will be turned into an embed")
                yield resp

                async def check_max(ev: aurflux.AurfluxEvent) -> bool:
                    m: discord.Message = ev.args[0]
                    if m.author == m.guild.me:
                        return False
                    if m.author != m.author:
                        return False
                    return True

                while True:
                    max_resp: discord.Message = (await self.aurflux.router.wait_for(":message", check_max, timeout=45)).args[0]
                    if max_resp.author == self.aurflux.user:
                        continue
                    try:
                        max_pins = int(max_resp.content)
                        if not 0 <= max_pins <= 49:
                            raise ValueError
                        break
                    except ValueError:
                        yield Response("Please enter a number <= 49 and >= 0", delete_after=10)

                async with self.aurflux.CONFIG.writeable_conf(ctx.config_identifier) as cfg:
                    cfg["maxmap"] = {**cfg.get("maxmap", {}), from_channel.id: max_pins}

                await max_resp.add_reaction(aurflux.utils.EMOJIS["white_check_mark"])
                if max_pins == 0:
                    yield Response(f"Done! When a message is pinned in {from_channel.mention}, it will be converted into an embed in {to_channel.mention}")
                else:
                    yield Response(f"Done! Once there are {max_pins} pins in {from_channel.mention}, the oldest pin will be converted to an embed in {to_channel.mention}")
            except asyncio.exceptions.TimeoutError:
                yield Response(f"Timed out! Stopping setup process. `{configs['prefix']}setup` to restart")

        @self.register
        @self.aurflux.commandeer(name="maps", parsed=False)
        async def _(ctx: aurflux.MessageContext, _):
            """
            Prints out the current set of maps
            :param ctx:
            :return:
            """
            configs = self.aurflux.CONFIG.of(ctx)

            embed = discord.Embed(title="Pinbot Map")
            pinmap = configs["pinmap"].items()
            print(len(pinmap))
            if len(pinmap) == 0:
                embed.description = "No maps set!"
                return Response(embed=embed)

            embed.add_field(name="Map", value="\n".join(f"<#{f}> \u27F6 <#{t}>" for f, t in pinmap), inline=True)
            embed.add_field(name="Max", value="\n".join(str(configs["maxmap"][f]) for f, _ in pinmap), inline=True)

            return Response(embed=embed)

        @self.register
        @self.aurflux.commandeer(name="pinall", parsed=False)
        async def _(ctx: aurflux.MessageContext, _):
            """
            Forces a re-check of all mapped channels for pins
            :param ctx:
            :param _:
            :return:
            """
            configs = self.aurflux.CONFIG.of(ctx)
            print(configs["pinmap"])
            for channel_id in configs["pinmap"]:
                await self.router.submit(event=aurflux.AurfluxEvent(self.aurflux, "aurflux:guild_channel_pins_update", self.aurflux.get_channel(int(channel_id)), None))
            return Response()


import numpy as np

import torch

from im_utils import binarize, to_image, make_grid
from cma_optimizer import CMA
from misc import to_numpy, progress_print
from optimization_core import step

def propagate_z(variables, z_sigma):
    """
    The variable information is re-used to make new samples. This is used to
    speed up search time. Using a moving average performs similarly.
    """
    z_mu = torch.stack(variables.z.data)
    z_noise = torch.randn_like(z_mu).float().cuda()
    z_mu = z_mu.clamp_(-1.0, 1.0).mean(0, keepdim=True)
    z = z_mu + z_sigma * z_noise
    return z

def search_transform(model, transform_fn, var_manager, loss_fn, meta_steps=30,
                     grad_steps=30, z_sigma=0.5, t_sigma=0.05, log=False,
                     pbar=None):
    """
    Searches for transformation parameter to apply to the target image such
    that it is better invertible. The transformation is optimized in a
    BasinCMA like fashion. Outer loop optimizes transformation using CMA and
    the inner loop is optimized using gradient descent.

    TODO: Clean up the code and integrate with optimzers.py

    Args:
        model:
            generative model
        transform_fn:
            the transformation search function
        var_manager:
            variable manager
        loss_fn:
            loss function to optimize with
        meta_steps:
            Number of CMA updates for transformation parameter
        grad_steps:
            Number of ADAM updates for z, c
        z_sigma:
            Sigma for latent variable resampling
        t_sigma:
            Sigma for transformation parameter
        log:
            Returns intermediate transformation result if True
        pbar:
            Progress bar such TQDM or st.progress()
    Returns
        Transformation parameters
    """

    # -- setup CMA -- #
    var_manager.optimize_t = True
    t_outs, outs, step_iter = [], [], 0
    total_steps = meta_steps * grad_steps
    t_cma_opt = CMA(mu=var_manager.t.cpu().numpy()[0], sigma=t_sigma)

    if t_cma_opt.batch_size() > var_manager.num_seeds:
        import nevergrad as ng
        print('Number of seeds is less than that required by PyCMA ' +
              'transformation search. Using Nevergrad CMA instead.')
        batch_size = var_manager.num_seeds
        opt_fn = ng.optimizers.registry['CMA']
        p = ng.p.Array(init=var_manager.t.cpu().numpy()[0])
        p = p.set_mutation(sigma=t_sigma)
        t_cma_opt = opt_fn(parametrization=p, budget=meta_steps)
        variables = var_manager.init(var_manager.num_seeds)
        using_nevergrad = True
    else:
        batch_size = t_cma_opt.batch_size()
        variables = var_manager.init(batch_size)
        using_nevergrad = False

    # Note: we will use the binarized weight for the CMA loss.
    mask = binarize(variables.weight.data[0]).unsqueeze(0)
    target = variables.target.data[0].unsqueeze(0)

    for i in range(meta_steps):

        # -- initialize variable -- #
        if using_nevergrad:
            _t = [t_cma_opt.ask() for _ in range(var_manager.num_seeds)]
            t = np.concatenate([np.array(x.args) for x in _t])
        else:
            t = t_cma_opt.ask()

        t_params = torch.Tensor(t)
        z = propagate_z(variables, z_sigma)
        variables = var_manager.init(num_seeds=batch_size,
                                     z=z.detach().float().cuda(),
                                     t=t_params.detach().float().cuda())

        for x in variables.t.data:
            x.requires_grad = False

        # -- inner update -- #
        losses = []
        for j in range(grad_steps):

            out, _, other = step(model, variables,
                                 loss_fn=loss_fn,
                                 transform_fn=transform_fn,
                                 optimize=True)

            step_iter += 1

            # Compute loss after inverting
            t_params = torch.stack(variables.t.data)
            inv_out = transform_fn(out, t_params, invert=True)

            loss = loss_fn(inv_out,
                           target.repeat(inv_out.size(0), 1, 1, 1),
                           mask.repeat(inv_out.size(0), 1, 1, 1))

            losses.append(to_numpy(loss))
            outs.append(to_image(make_grid(out), cv2_format=False))

            if pbar is not None:
                pbar.progress(step_iter / total_steps)
            else:
                if (step_iter + 1) % 50 == 0:
                    progress_print('transform', step_iter + 1, total_steps, 'c')

        # -- update CMA -- #
        if using_nevergrad:
            for z, l in zip(_t, np.min(losses, 0)):
                t_cma_opt.tell(z, l)
        else:
            t_cma_opt.tell(t, np.min(losses, 0))

        # -- log for visualization -- #
        if log and 'target' in other.keys():
            t_out = binarize(other['weight'], 0.3) * other['target']
            t_outs.append(to_image(make_grid(t_out), cv2_format=False))

    if using_nevergrad:
        t_mu = np.array(t_cma_opt.provide_recommendation().value)
    else:
        t_mu = t_cma_opt.mean()
    return torch.Tensor([t_mu]), t_outs, outs


from __future__ import division
import keras
import os
import sys
import numpy as np
import scipy.io as sio

from keras import optimizers
from keras import initializers
from keras.models import load_model
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Permute, Flatten, Dense, BatchNormalization, Activation, Dropout

def trainOnMatfile(matfile,modelfile,windowSize,targetdelays):
	if not os.path.isfile(modelfile):
		x_train,y_train,x_val,y_val,x_test,x_trainnc = load_traindata(matfile,windowSize,targetdelays)
		numChannels = x_train.shape[2];
		model = train(modelfile,x_train,y_train,x_val,y_val)

		pred = classify(model,x_test)
		np.savetxt(matfile[:-4]+"_train_pred.csv", pred, delimiter=",")
		pred = classify(model,x_trainnc)
		np.savetxt(matfile[:-4]+"_trainnc_pred.csv", pred, delimiter=",")
	else:
		numChannels = get_channels(matfile)
		model = load_model(modelfile)
	return model,numChannels

def train(filename,x_train,y_train,x_val,y_val):
	if not os.path.isfile(filename):
		# model parameter
		lr = 0.001
		batchsize = 256
		epochs = 25

		model = construct_model(x_train.shape[1],x_train.shape[2])
		adam = keras.optimizers.Adam(lr=lr)
		model.compile(loss='categorical_crossentropy',optimizer=adam,metrics=['accuracy'])
		history = model.fit(x_train, y_train, batch_size=batchsize, epochs=epochs, validation_data=(x_val, y_val), callbacks = [keras.callbacks.ModelCheckpoint(filename, monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=False, mode='auto', period=1)])

	model = load_model(filename)

	return model;

# input: keras model, and test data (windowSize x Channels)
def classify(model,x_test):
	x_test = x_test.reshape(-1,x_test.shape[1],x_test.shape[2],1)
	pred = model.predict(x_test)
	return pred[:,0];

def get_channels(filename):
	print('Load mat file')
	mat_contents = sio.loadmat(filename)
	usedchannels = np.array(mat_contents['usedchannels'])[0]
	numChannels = usedchannels.size
	return numChannels

def load_traindata(filename,windowSize,targetdelays):
	print('Load mat file')
	mat_contents = sio.loadmat(filename)

	# extract matlab struct
	train_data_x = np.array(mat_contents['train_data_x'])
	train_data_y = np.array(mat_contents['train_data_y'])
	trainnc_data_x = np.array(mat_contents['trainnc_data_x'])
	trainnc_data_y = np.array(mat_contents['trainnc_data_y'])
	usedchannels = np.array(mat_contents['usedchannels'])[0]
	targetdelays = np.array(targetdelays)

	print('Splitting train data to windows using the following channels')
	print(usedchannels)
	data_x_train = []
	data_y_train = []
	data_x_test = []
	for ii in range(train_data_x.shape[0]):
		realtarget = ii % 32
		targetshift = targetdelays[realtarget]
		trialdata = train_data_x[ii,usedchannels-1,:].squeeze().transpose()
		trialdatashifted = np.roll(trialdata,-targetshift,axis=0)

		data_x_timelag = np.zeros((trialdata.shape[0],windowSize,len(usedchannels)))
		data_x_timelagshifted = np.zeros((trialdatashifted.shape[0],windowSize,len(usedchannels)))
		for t in range(windowSize):
			data_x_timelag[:,t,:] = np.roll(trialdata,-t,axis=0)
			data_x_timelagshifted[:,t,:] = np.roll(trialdatashifted,-t,axis=0)

		data_x_timelag = data_x_timelag[0:(data_x_timelag.shape[0]-windowSize),:,:]
		data_x_timelagshifted = data_x_timelagshifted[0:(data_x_timelagshifted.shape[0]-windowSize),:,:]
		bitdata = train_data_y[ii,realtarget,:].squeeze().transpose()
		for t in range(data_x_timelag.shape[0]):
			data_x_train.append(data_x_timelagshifted[t,:,:].squeeze())
			data_x_test.append(data_x_timelag[t,:,:].squeeze())
			data_y_train.append([bitdata[t],abs(bitdata[t]-1)])

	# split train and validation set
	(x_train,y_train,x_val,y_val) = split_data(np.array(data_x_train),np.array(data_y_train))
	data_x_test = np.array(data_x_test)

	data_x_trainnc = []
	for ii in range(trainnc_data_x.shape[0]):
		realtarget = ii % 32
		targetshift = targetdelays[realtarget]
		trialdata = trainnc_data_x[ii,usedchannels-1,:].squeeze().transpose()

		data_x_timelag = np.zeros((trialdata.shape[0],windowSize,len(usedchannels)))
		for t in range(windowSize):
			data_x_timelag[:,t,:] = np.roll(trialdata,-t,axis=0)

		data_x_timelag = data_x_timelag[0:(data_x_timelag.shape[0]-windowSize),:,:]
		for t in range(data_x_timelag.shape[0]):
			data_x_trainnc.append(data_x_timelag[t,:,:].squeeze())

	data_x_trainnc = np.array(data_x_trainnc)

	print('done.')

	return x_train,y_train,x_val,y_val,data_x_test,data_x_trainnc

def split_data(x_train,y_train):
	print ('Splitting train data to train set and validation set')
	x_train = x_train.reshape(-1,x_train.shape[1],x_train.shape[2],1)
	y_train = y_train.reshape(-1,2)
	# Split data to train and validation set
	if (len(x_train) % 2) != 0:
		x_train = x_train[:-1]
		y_train = y_train[:-1]
	x_split = np.split(x_train, 2)
	y_split = np.split(y_train, 2)
	# train set
	x_train = x_split[0]
	y_train = y_split[0]
	# validation set
	x_val = x_split[1]
	y_val = y_split[1]
	print ('done.')

	return x_train,y_train,x_val,y_val

def construct_model(windowSize,numChannels):
	# construct sequential model
	model = Sequential()
	# permute input so that it is as in EEG Net paper
	model.add(Permute((3,2,1), input_shape=(windowSize,numChannels,1)))
	# layer1
	model.add(Conv2D(16, kernel_size=(numChannels, 1), padding='valid', strides=(1, 1), data_format='channels_first', activation='relu'))
	model.add(BatchNormalization(axis=1, scale=False, center=False))
	#model.add(Permute((2,1,3)))
	model.add(Activation('relu'))
	model.add(MaxPooling2D(pool_size=(2, 2),strides=(2, 2),padding='same'))
	# layer2
	model.add(Conv2D(8,kernel_size=(1, 64),data_format='channels_first',padding='same'))
	model.add(BatchNormalization(axis=1,scale=False, center=False))
	model.add(Activation('relu'))
	model.add(MaxPooling2D(pool_size=(2, 2),strides=(2, 2),padding='same'))
	model.add(Dropout(0.5))
	# layer3
	model.add(Conv2D(4,kernel_size=(5, 5),data_format='channels_first',padding='same'))
	model.add(BatchNormalization(axis=1,scale=False,center=False))
	model.add(Activation('relu'))
	model.add(MaxPooling2D(pool_size=(2, 2), data_format='channels_first',padding='same'))
	model.add(Dropout(0.5))
	# layer4
	model.add(Flatten())
	model.add(Dense(1024, activation='relu'))
	model.add(Dropout(0.5))
	# layer5
	model.add(Dense(2, activation='softmax'))
	#print(model.summary())
	return model


import os
import pytest
from buildtest.menu.compilers import BuildtestCompilers
from buildtest.exceptions import BuildTestError

here = os.path.dirname(os.path.abspath(__file__))

class TestBuildtestCompilers:

    bc = BuildtestCompilers()

    def test_init(self):

        assert hasattr(self.bc, "compilers")
        assert isinstance(self.bc.names, list)

    def test_print(self):
        self.bc.print_yaml()
        self.bc.print_json()
        self.bc.print_compilers()

    def test_invalid_moduletool(self):
        settings_file = os.path.join(here, "test_compilers", "invalid_moduletool.yml")
        print(f"Using settings file: {settings_file} for loading compilers")
        bc = BuildtestCompilers(settings_file=settings_file)
        with pytest.raises(BuildTestError):
            bc.find_compilers()

    def test_missing_compiler_find(self):
        settings_file = os.path.join(
            here, "test_compilers", "missing_compiler_find.yml"
        )
        print(f"Using settings file: {settings_file} for loading compilers")
        bc = BuildtestCompilers(settings_file=settings_file)
        with pytest.raises(BuildTestError):
            bc.find_compilers()


from tools.scraper import Scraper
from tools.state import State

def reply(activity, bot, data):
    print(data.get_entities())
    credentials = activity["message"]["text"].split(" ")
    rollno = credentials[0]
    wak = credentials[1]
    if len(wak) > 5:
        bot.send_text_activity(activity, "Please check your roll no. and web access key again.")
        bot.send_text_activity(activity, "Enter roll no. and web access key seperated by a single space.")
        return
    scraper = Scraper()
    scraper.authenticate(rollno,wak)
    if scraper.authenticated:
        state = State()
        studentdata = scraper.get_studentdata()
        studentdata["wak"] = wak
        studentdata["RowKey"] = activity["sender"]["id"]
        state.insertOrUpdateStudent(studentdata)
        bot.send_text_activity(activity, "Authentication Successful!")
    else:
        bot.send_text_activity(activity, "Please check your roll no. and web access key again.")
        bot.send_text_activity(activity, "Enter roll no. and web access key seperated by a single space.")