import warnings
import operator
import numpy as np
import matplotlib.pyplot as plt

from functools import partial
from itertools import product
from scipy.linalg import pinvh
from collections import Sequence

from sklearn.utils import check_random_state, check_array
from sklearn.utils.extmath import squared_norm, fast_logdet
from sklearn.covariance import empirical_covariance
from sklearn.datasets import make_sparse_spd_matrix
from sklearn.preprocessing import normalize
from sklearn.model_selection import check_cv
from sklearn.utils import Parallel, delayed
from sklearn.exceptions import ConvergenceWarning

from regain.covariance.graph_lasso_ import graph_lasso

from network_inference.datasets import is_pos_def
from network_inference.utils import log_likelihood, BIC, EBIC, EBIC_m
from network_inference.utils import convergence, l1_od_norm

def par_max(emp_cov):
    A = np.copy(emp_cov)
    A.flat[::A.shape[0] + 1] = 0
    return np.max(np.abs(A))

def _penalized_nll(K, S=None, regularizer=None):
    res = - fast_logdet(K) + np.sum(K*S)
    res += np.linalg.norm(regularizer*K, 1)
    return res

def flgl_path(X_train, mask=None, lambdas=[0.1], mus=[0.1], etas=[0.1],
              X_test=None, random_search=True,  tol=1e-3, max_iter=200,
              update_rho=False, verbose=0, score='ebic',
              random_state=None, save_all=False, penalize_latent=True):

    score_func = {'likelihood': log_likelihood,
                  'bic': BIC,
                  'ebic': partial(EBIC, n=X_test.shape[0]),
                  'ebicm': partial(EBIC_m, n=X_test.shape[0])}
    try:
        score_func = score_func[score]
    except KeyError:
        warnings.warn("The score type passed is not available, using log likelihood.")
        score_func = log_likelihood

    emp_cov = empirical_covariance(X_train)
    covariance_ = emp_cov.copy()

    covariances_ = list()
    precisions_ = list()
    observeds_ =  list()
    scores_ = list()

    if X_test is not None:
        test_emp_cov = empirical_covariance(X_test)

    if random_search:
        couples = zip(lambdas, mus, etas)
    else:
        couples = product(lambdas, mus, etas)
    for i, params in enumerate(couples):
        print(params)
        try:
            # Capture the errors, and move on
            prec_, cov_, _ = EM_extension(
                emp_cov, mask, params[0], params[1], params[2], max_iter=max_iter,
                return_n_iter=False, penalize_latent=penalize_latent)
            if save_all:
                covariances_.append(cov_)
                precisions_.append(prec_)

            h = mask.shape[1]
            observed = prec_[h:,h:] - prec_[h:, :h].dot(pinvh(prec_[:h, :h])).dot(prec_[:h, h:])
            if X_test is not None:
                this_score = score_func(test_emp_cov, observed)
                print("Iter: %d, Lambda %.4f, mu %.4f, eta %.2f, score %.4f" %(i, params[0], params[1], params[2], this_score))
            observeds_.append(observed)
        except FloatingPointError:
            this_score = -np.inf
            if save_all:
                covariances_.append(np.nan)
                precisions_.append(np.nan)
                observeds_.append(np.nan)
        if X_test is not None:
            if not np.isfinite(this_score):
                this_score = -np.inf
            scores_.append(this_score)
        if verbose:
            if X_test is not None:
                print('[graphical_lasso_path] lambda_: %.2f, mu: %.2f, eta:%.2f, score: %.2f'
                      % (params[0], params[1], params[2], this_score))
            else:
                print('[graphical_lasso_path] lambda_: %.2f, mu: %.2f' % (lambda_, mu))
    if X_test is not None:
        return covariances_, precisions_, observeds_, scores_
    return covariances_, precisions_, observeds_

def EM_Yuan(emp_cov, lambda_, h, assume_centered=False, tol=1e-3, max_iter=200, verbose=0, compute_objective=False,
            return_n_iter=False, max_iter_graph_lasso=100):
    o = emp_cov.shape[0]
    emp_cov_H = np.zeros((h, h))
    emp_cov_OH = np.zeros((o,h))

    K = np.random.randn(h+o, h+o)
    K = K.dot(K.T)
    K /= np.max(K)

    regularizer = np.zeros((h+o, h+o))
    regularizer[h:, h:]=np.ones((o,o))*lambda_
    regularizer[h:, h:] -= np.diag(np.diag(regularizer[h:, h:]))

    penalized_nll = np.inf
    for iter_  in range(max_iter):

        #expectation step
        sigma = pinvh(K)
        sigma_o_inv = pinvh(sigma[h:, h:])
        sigma_ho = sigma[:h, h:]
        sigma_oh = sigma[h:, :h]
        emp_cov_H = (sigma[:h, :h] - sigma_ho.dot(sigma_o_inv).dot(sigma_oh)+
                     sigma_ho.dot(sigma_o_inv).dot(emp_cov).dot(sigma_o_inv).dot(sigma_oh))
        emp_cov_OH = emp_cov.dot(sigma_o_inv).dot(sigma_oh)
        S = np.zeros_like(K)
        S[:h,:h] = emp_cov_H
        #emp_cov_O = emp_cov + emp_cov_OH.dot(pinvh(emp_cov_H)).dot(emp_cov_OH.T)
        S[:h, h:] =  emp_cov_OH.T
        S[h:, :h] = emp_cov_OH
        S[h:, h:] = emp_cov
        penalized_nll_old = penalized_nll
        penalized_nll = _penalized_nll(K, S, regularizer)

        # maximization step
        K, _ = graph_lasso(S, alpha=regularizer, return_n_iter=False, max_iter=max_iter_graph_lasso,
                          verbose=0)#int(np.max(verbose-1, 0)))
        nll_diff = penalized_nll_old - penalized_nll
        if verbose:
            print("iter: %d, NLL: %.6f , NLL_diff: %.6f"%(iter_, penalized_nll, nll_diff))
        if  np.abs(nll_diff) < tol:
            break
    else:
        warnings.warn("The optimization of EM did not converged.")
    return K, S, penalized_nll_old

def EM_extension(emp_cov, M, lambda_, mu_, eta_=0,  assume_centered=False, tol=1e-3, max_iter=200, verbose=0, compute_objective=False,
       return_n_iter=False, penalize_latent=True, max_iter_graph_lasso=100):
    h = M.shape[1]
    o = emp_cov.shape[0]
    emp_cov_H = np.zeros((h, h))
    emp_cov_OH = np.zeros_like(M)

    K = np.random.randn(h+o, h+o)
    K = K.dot(K.T)
    K /= np.max(K)

    if penalize_latent:
        regularizer = np.ones((h+o, h+o))
        regularizer -= np.diag(np.diag(regularizer))
        regularizer[:h, :h]*=eta_
        regularizer[h:, h:]*=lambda_
    else:
        regularizer = np.zeros((h+o, h+o))
        regularizer[h:, h:] = lambda_*np.ones((o,o))
        regularizer[h:, h:] -=  np.diag(np.diag(regularizer[h:, h:]))
    regularizer[:h, h:] = mu_*M.T
    regularizer[h:, :h] = mu_*M
    penalized_nll = np.inf
    for iter_  in range(max_iter):

        #expectation step
        sigma = pinvh(K)
        sigma_o_inv = pinvh(sigma[h:, h:])
        sigma_ho = sigma[:h, h:]
        sigma_oh = sigma[h:, :h]
        emp_cov_H = (sigma[:h, :h] - sigma_ho.dot(sigma_o_inv).dot(sigma_oh)+
                     sigma_ho.dot(sigma_o_inv).dot(emp_cov).dot(sigma_o_inv).dot(sigma_oh))
        emp_cov_OH = emp_cov.dot(sigma_o_inv).dot(sigma_oh)
        S = np.zeros_like(K)
        S[:h,:h] = emp_cov_H
        #emp_cov_O = emp_cov + emp_cov_OH.dot(pinvh(emp_cov_H)).dot(emp_cov_OH.T)
        S[:h, h:] =  emp_cov_OH.T
        S[h:, :h] = emp_cov_OH
        S[h:, h:] = emp_cov
        penalized_nll_old = penalized_nll
        penalized_nll = _penalized_nll(K, S, regularizer)

        # maximization step
        K, _ = graph_lasso(S, alpha=regularizer, return_n_iter=False, max_iter=max_iter_graph_lasso,
                          verbose=0)#int(np.max(verbose-1, 0)))
        nll_diff = penalized_nll_old - penalized_nll
        if verbose:
            print("iter: %d, NLL: %.6f , NLL_diff: %.6f"%(iter_, penalized_nll, nll_diff))
        if  np.abs(nll_diff) < tol:
            break
    else:
        warnings.warn("The optimization of EM did not converged.")
    return K, S, penalized_nll_old

class EMLatentGraphLassoCV():
    """
    lambdas: the parameters to test
        If an integer n is passed then n parameters are selected in a (hopefully) suitable interval.
        If a list is passed then the parameters in the list are used to test the method.
        If a tuple of two elements is passed 10 elements between the elements of the tuple are tested.
    mus: the parameters to test
        If an integer n is passed then n parameters are selected in a (hopefully) suitable interval.
        If a list is passed then the parameters in the list are used to test the method.
        If a tuple of two elements is passed 10 elements between the elements of the tuple are tested.
    random_search: boolean, default True
        If True and lambdas and mus are either an integer or a tuple, the values are randomly selected,
        not in a grid search.
    """
    def __init__(self, mask, lambdas=4, mus=4, etas=4, rho=1., score='ebic', cv=None, tol=1e-4,
                 max_iter=100, n_jobs=1, random_search=True,
                 verbose=False, assume_centered=False,
                 update_rho=False, random_state=None, n_params=10, save_all=False, penalize_latent=True):
        self.mask = mask
        self.lambdas = lambdas
        self.mus = mus
        self.rho = 1.
        self.etas= etas
        self.cv = cv
        self.n_jobs = n_jobs
        self.tol = tol
        self.max_iter = max_iter
        self.verbose = verbose
        self.update_rho = update_rho
        self.assume_centered = assume_centered
        self.random_state = random_state
        self.score = score
        self.random_search = random_search
        self.n_params = n_params
        self.save_all = save_all
        self.penalize_latent = penalize_latent

    def grid_scores(self):
        return self.grid_scores_

    def fit(self, X, y=None):
        """Fits the GraphLasso covariance model to X.
        Parameters
        ----------
        X : ndarray, shape (n_samples, n_features)
            Data from which to compute the covariance estimate
        y : (ignored)
        """
        # Covariance does not make sense for a single feature
        self.random_state = check_random_state(self.random_state)
        # check_data_dimensions(X, layers=2)
        X = check_array(X, ensure_min_features=2,
                         ensure_min_samples=2, estimator=self)

        self.X_train = X
        if self.assume_centered:
            self.location_ = np.zeros((X.shape[0],  X.shape[1]))
        else:
            self.location_ = X.mean(0)

        emp_cov = empirical_covariance(
                        X, assume_centered=self.assume_centered)

        X = check_array(X, ensure_min_features=2, estimator=self)
        cv = check_cv(self.cv, y, classifier=False)

        # List of (alpha, scores, covs)
        path = list()
        n_lambdas = self.lambdas
        inner_verbose = max(0, self.verbose - 1)

        if isinstance(n_lambdas, list):
            lambdas = self.lambdas
        else:
            lambda_1 = par_max(emp_cov)
            lambda_0 = 1e-2 * lambda_1
            if self.random_search:
                if isinstance(n_lambdas, tuple):
                     lambdas = np.array([self.random_state.uniform(n_lambdas[0], n_lambdas[1]) for i in range(self.n_params)])
                else:
                    lambdas = np.array([self.random_state.uniform(lambda_0, lambda_1) for i in range(n_lambdas)])
            else:
                lambdas = np.logspace(np.log10(lambda_0), np.log10(lambda_1),
                                      n_lambdas)[::-1]

        n_etas = self.etas
        inner_verbose = max(0, self.verbose - 1)

        if isinstance(n_etas, list):
            etas = self.etas
        else:
            eta_1 = par_max(emp_cov)
            eta_0 = 1e-2 * eta_1
            if self.random_search:
                if isinstance(n_etas, tuple):
                     etas = np.array([self.random_state.uniform(n_etas[0], n_etas[1]) for i in range(self.n_params)])
                else:
                     etas = np.array([self.random_state.uniform(eta_0, eta_1) for i in range(n_etas)])
            else:
                etas = np.logspace(np.log10(eta_0), np.log10(eta_1),
                                      n_etas)[::-1]
        n_mus = self.mus
        inner_verbose = max(0, self.verbose - 1)

        if isinstance(n_mus, list):
            mus = self.mus
        else:
            mu_1 = par_max(emp_cov)
            mu_0 = 1e-2 * mu_1
            if self.random_search:
                if isinstance(n_mus, tuple):
                    mus = np.array([self.random_state.uniform(n_mus[0], n_mus[1]) for i in range(self.n_params)])
                else:
                    mus = np.array([self.random_state.uniform(mu_0, mu_1) for i in range(n_mus)])
            else:
                mus = np.logspace(np.log10(mu_0), np.log10(mu_1),
                                     n_mus)[::-1]

        with warnings.catch_warnings():
            warnings.simplefilter('ignore', ConvergenceWarning)

        this_path = Parallel(
                n_jobs=self.n_jobs,
                verbose=self.verbose
            )(delayed(flgl_path)(X[train], mask=self.mask, lambdas=lambdas, mus=mus, etas=etas,
                                 X_test=X[test], random_search=self.random_search, tol=self.tol,
                                        max_iter=int(.1 * self.max_iter),
                                        update_rho=self.update_rho,
                                        verbose=0, random_state=self.random_state,
                                score=self.score, save_all=self.save_all, penalize_latent=self.penalize_latent)
              for train, test in cv.split(X, y))

        # Little danse to transform the list in what we need
        covs, precs, hidds, scores = zip(*this_path)

        if self.save_all:
            covs = zip(*covs)
            precs = zip(*precs)
            hidds = zip(*hidds)
        scores = zip(*scores)
        #combinations = zip(lambdas, mus)
        combinations = list(zip(lambdas, mus, etas))
        if self.save_all:
            path.extend(zip(combinations, scores, covs))
        else:
            path.extend(zip(combinations, scores))
        path = sorted(path, key=operator.itemgetter(0), reverse=True)

        # Find the maximum (avoid using built in 'max' function to
        # have a fully-reproducible selection of the smallest alpha
        # in case of equality)
        best_score = -np.inf
        last_finite_idx = 0
        for index, (combination, scores) in enumerate(path):
            this_score = np.mean(scores)
            if this_score >= .1 / np.finfo(np.float64).eps:
                this_score = np.nan
            if np.isfinite(this_score):
                last_finite_idx = index
            if this_score >= best_score:
                best_score = this_score
                best_index = index

        path = list(zip(*path))
        grid_scores = list(path[1])
        parameters = list(path[0])
        # Finally, compute the score with alpha = 0
        best_lambda, best_mu, best_eta = combinations[best_index]
        self.lambda_ = best_lambda
        self.mu_ = best_mu
        self.eta_ = best_eta
        self.cv_parameters_ = combinations

        # Finally fit the model with the selected alpha
        self.precision_, self.covariance_,  self.n_iter_ = EM_extension(
            emp_cov, self.mask, lambda_=best_lambda, mu_=best_mu, eta_=best_eta, tol=self.tol,
            max_iter=self.max_iter,
            verbose=self.verbose,
            compute_objective=True, return_n_iter=True)
        return self


from bs4 import BeautifulSoup

import requests
headers = {'user-agent': 'my-app/0.0.1'}

def waterstonesScrape(isbn):
    #print 'Starting Ben\'s Amazing Waterstones Scraper!'

    url = "https://www.waterstones.com/book/" + isbn

    page = requests.get(url, headers=headers)
    if(page.status_code == 404):
        print("404! Skipping ISBN : " + isbn)
    else:
        html_contents = page.text
        soup = BeautifulSoup(html_contents, 'html.parser')

        if(soup.find("div", id="scope_book_description")):
            return soup.find("div", id="scope_book_description").text.strip()
        else:
            return

    #print('Finished Ben\'s Amazing Waterstones Scraper!')

#synopsis = waterstonesScrape("9780091816971")
#print(synopsis)

#404 test
#print(waterstonesScrape("9788126506750"))


import sys
import os
import h5py
import torch
import scipy.io as sio
import numpy as np
import pickle
import random
import networkx as nx
import matplotlib.pyplot as plt
import NTdatasets.HN.HNdatasets as datasets
import NDNT.utils as utils
import NDNT.NDNT as NDN
from NDNT.modules.layers import ChannelLayer, NDNLayer
from NDNT.networks import FFnetwork
from time import time
from torch import nn
from copy import deepcopy

device = torch.device("cuda:0")  # use the GPU
dtype = torch.float32  # make arrays be type float32

datadir = '../../data/hn/'
dirname = './savedir/'

def define_glm_layer(num_cells, stim_dims, as_layer=True):
    # if this is the only network to train, use a 'softplus' activation function
    # if this is part of a larger network, use a 'linear' activation function
    nl_type = 'lin' if as_layer else 'softplus'

    glm_layer = NDNLayer.layer_dict(
        input_dims=stim_dims,
        num_filters=num_cells,
        bias=True,
        initialize_center=True,
        NLtype=nl_type)

    # regularization method (use d2xt with lambda weight 0.02)
    glm_layer['reg_vals'] = {'d2xt': 0.02, 'bcs': {'d2xt': 1}}

    # only return the FFnetwork wrapped NDNLayer when combining this with other layers
    if as_layer:
        return FFnetwork.ffnet_dict(xstim_n='stim', layer_list=[glm_layer])
    else:
        return glm_layer

def define_comb_layer(num_cells):
    comb_layer = ChannelLayer.layer_dict(
        num_filters=num_cells,
        NLtype='softplus',
        bias=True)
    comb_layer['weights_initializer'] = 'ones'  # initialize to just pass the data through this "network"

    return FFnetwork.ffnet_dict(xstim_n=None, ffnet_n=[0, 1], layer_list=[comb_layer], ffnet_type='add')

def define_latent_layer(num_latents, num_cells, as_layer=True):
    L2reg = 0.01

    nl_type = 'lin' if as_layer else 'softplus'

    # input --> LVs (X --> Z)
    # encoder
    AClin_layer = NDNLayer.layer_dict(
        # what are these input_dims?
        input_dims=[num_cells, 1, 1, 1], num_filters=num_latents,  # num_LVs --> |Z|
        norm_type=1,  # normalization needed to keep the scale
        # this adjusts the activation threshold by making the output spikes more or less
        bias=False,  # don't put a bias on the input
        NLtype='lin')  # 'lin' -> linear, no nonlinearity applied here

    # LVs --> output (Z --> X')
    # decoder
    ACout_layer = NDNLayer.layer_dict(
        num_filters=num_cells,
        bias=True,
        NLtype=nl_type)

    # apply L2 regularization on the output
    # TODO: I don't understand why, though
    ACout_layer['reg_vals'] = {'l2': L2reg}

    # return the autoencoder as an FFnetwork (TODO: do we want to do this?)
    return FFnetwork.ffnet_dict(
        xstim_n='robs',
        layer_list=[AClin_layer, ACout_layer])

def fit_model(model, data):
    # define parameters for the LBFGS optimizer to fit the GLM
    lbfgs_pars = utils.create_optimizer_params(
        optimizer_type='lbfgs',
        tolerance_change=1e-10,
        tolerance_grad=1e-10,
        batch_size=2000,
        history_size=100,
        max_epochs=3,  # do we really only want to use a maximum of 3 epochs?
        max_iter=2000)

    # Fit the model
    model.fit(data, force_dict_training=True, **lbfgs_pars, verbose=1)

    # Evaluate model using null-adjusted log-likelihood
    return model.eval_models(data[data.val_inds], null_adjusted=True)

def train_autoencoder(data, num_latents):
    # define the untrained autoencoder model to learn the latent vars
    latent_layer = define_latent_layer(num_latents, data.NC, as_layer=False)

    latent_model = NDN.NDN(ffnet_list=[latent_layer], loss_type='poisson')

    # fit the model on the data
    lls = fit_model(latent_model, data)

    return lls, latent_model

# TRAIN THE GLM+AUTOENCODER MODEL
def train_autoencoder_and_glm(data, num_latents, pretrained_glm):
    # define the untrained GLM model
    # use a linear output since it gets combined with the autoencoder in the channel layer
    glm_layer = define_glm_layer(data.NC, data.stim_dims, as_layer=True)
    # define the untrained autoencoder model to learn the latent vars
    latent_layer = define_latent_layer(num_latents=num_latents, num_cells=data.NC)
    # define the untrained combined model
    comb_layer = define_comb_layer(data.NC)

    # define a model that combines info from the pretrained GLM and the autoencoder
    pretrained_glm_plus_latent_model = NDN.NDN(ffnet_list=[glm_layer, latent_layer, comb_layer], loss_type='poisson')

    # initialize GLM network with weights from the previously trained GLM
    pretrained_glm_plus_latent_model.networks[0].layers[0].weight = deepcopy(
        pretrained_glm.networks[0].layers[0].weight)

    # fit the model on the data
    lls = fit_model(pretrained_glm_plus_latent_model, data)

    return lls, pretrained_glm_plus_latent_model

def train_drift(data):
    lbfgs_pars = utils.create_optimizer_params(
        optimizer_type='lbfgs',
        tolerance_change=1e-10,
        tolerance_grad=1e-10,
        batch_size=2000,
        history_size=100,
        max_epochs=3,
        max_iter=2000)

    # Fit all cells at once
    Dreg = 0.005
    drift_pars = NDNLayer.layer_dict(
        input_dims=[1, 1, 1, data.Xdrift.shape[1]], num_filters=data.NC, bias=False, norm_type=0, NLtype='lin')
    drift_pars['reg_vals'] = {'d2t': Dreg, 'bcs': {'d2t': 0}}

    # for stand-alone drift model
    drift_parsN = deepcopy(drift_pars)
    drift_parsN['NLtype'] = 'softplus'

    drift_netN = FFnetwork.ffnet_dict(xstim_n='Xdrift', layer_list=[drift_parsN])

    # Stand-alone drift model
    drift_mod = NDN.NDN(ffnet_list=[drift_netN], loss_type='poisson')
    # Check out all the parameters in the model
    # drift_mod.list_parameters()
    drift_mod.fit(data, force_dict_training=True, **lbfgs_pars, verbose=0)
    # evaluate model using null-adjusted log-likelihood
    LLsNULL = drift_mod.eval_models(data[data.val_inds], null_adjusted=False)
    print("Mean LL-null:", np.mean(LLsNULL))

    return LLsNULL, drift_netN, drift_mod

def train_glm_with_drift(data, drift_net, drift_mod):
    lbfgs_pars = utils.create_optimizer_params(
        optimizer_type='lbfgs',
        tolerance_change=1e-10,
        tolerance_grad=1e-10,
        batch_size=2000,
        history_size=100,
        max_epochs=3,
        max_iter=2000)

    glm_layer_lin = NDNLayer.layer_dict(
        input_dims=data.stim_dims,
        num_filters=data.NC,
        bias=False, initialize_center=True,
        NLtype='lin')
    glm_layer_lin['reg_vals'] = {'d2xt': 0.02, 'bcs': {'d2xt': 1}}
    glm_net = FFnetwork.ffnet_dict(xstim_n='stim', layer_list=[glm_layer_lin])

    comb_layer = ChannelLayer.layer_dict(
        num_filters=data.NC, NLtype='softplus', bias=True)
    comb_layer['weights_initializer'] = 'ones'

    comb_net = FFnetwork.ffnet_dict(xstim_n=None, ffnet_n=[0, 1], layer_list=[comb_layer], ffnet_type='add')

    glmD0 = NDN.NDN(ffnet_list=[glm_net, drift_net, comb_net], loss_type='poisson')
    # Copy drift from drift_mod and do not fit
    # stuff the pretrained drift model weights into the drift_net component
    glmD0.networks[1].layers[0].weight.data = deepcopy(drift_mod.networks[0].layers[0].weight.data)
    glmD0.set_parameters(val=False, name='weight', ffnet_target=2)
    glmD0.set_parameters(val=False, ffnet_target=1)
    glmD0.fit(data, force_dict_training=True, **lbfgs_pars, verbose=0)
    # Evaluate model using null-adjusted log-likelihood
    LLs0 = glmD0.eval_models(data[data.val_inds], null_adjusted=False)
    return LLs0, glmD0

def hierarchy_pos(G, root=None, width=1., vert_gap=0.2, vert_loc=0, xcenter=0.5):
    """
    From Joel's answer at https://stackoverflow.com/a/29597209/2966723.
    Licensed under Creative Commons Attribution-Share Alike

    If the graph is a tree this will return the positions to plot this in a
    hierarchical layout.

    G: the graph (must be a tree)

    root: the root node of current branch
    - if the tree is directed and this is not given,
      the root will be found and used
    - if the tree is directed and this is given, then
      the positions will be just for the descendants of this node.
    - if the tree is undirected and not given,
      then a random choice will be used.
    width: horizontal space allocated for this branch - avoids overlap with other branches
    vert_gap: gap between levels of hierarchy
    vert_loc: vertical location of root
    xcenter: horizontal location of root
    """
    if not nx.is_tree(G):
        raise TypeError('cannot use hierarchy_pos on a graph that is not a tree')

    if root is None:
        if isinstance(G, nx.DiGraph):
            root = next(iter(nx.topological_sort(G)))  # allows back compatibility with nx version 1.11
        else:
            root = random.choice(list(G.nodes))

    def _hierarchy_pos(G, root, width=1., vert_gap=0.2, vert_loc=0, xcenter=0.5, pos=None, parent=None):
        """
        see hierarchy_pos docstring for most arguments

        pos: a dict saying where all nodes go if they have been assigned
        parent: parent of this branch. - only affects it if non-directed
        """
        if pos is None:
            pos = {root: (xcenter, vert_loc)}
        else:
            pos[root] = (xcenter, vert_loc)
        children = list(G.neighbors(root))
        if not isinstance(G, nx.DiGraph) and parent is not None:
            children.remove(parent)
        if len(children) != 0:
            dx = width / len(children)
            nextx = xcenter - width / 2 - dx / 2
            for child in children:
                nextx += dx
                pos = _hierarchy_pos(G, child, width=dx, vert_gap=vert_gap,
                                     vert_loc=vert_loc - vert_gap, xcenter=nextx,
                                     pos=pos, parent=root)
        return pos

    return _hierarchy_pos(G, root, width, vert_gap, vert_loc, xcenter)

# TODO: use networkx to draw the network to see what it looks like
def draw_network(model, names):
    # TODO: go through each network, and its layers and connect them into a network
    # TODO: add any attributes to the nodes and edges that make it clearer what they are doing
    assert type(model) == NDN.NDN, "model needs to be of type NDN"

    # define the network
    g = nx.DiGraph()

    # create the labels dictionary
    labels = {idx: name for idx, name in enumerate(names)}

    # go through the networks in the model
    for netidx, network in enumerate(model.networks):
        # add the network as a node
        g.add_node(netidx)

        # add edges between data and networks
        if network.xstim_n is not None:
            g.add_edge(network.xstim_n, netidx)
            # add the stim name to the map of node labels
            labels[network.xstim_n] = network.xstim_n

        # add edges between networks
        if network.ffnets_in is not None:
            for inputidx in network.ffnets_in:
                g.add_edge(inputidx, netidx)
                # attribute the edge if the network type is something
                # TODO: think about how to represent the attrs,
                #       but don't get too deep into this yet.
                #       Maybe I can use D3 for this to make it render nicer.
                if network.network_type is not None:
                    g[inputidx][netidx]['type'] = network.network_type
        # inputs to each network
        # QUESTION: can a network have ffnets_in and xstim_n?

    # draw the network
    # TODO: fix the layout to draw an inverted tree
    # nx.draw_networkx(g, pos=hierarchy_pos(g))
    nx.draw_networkx(g, labels=labels, with_labels=True)


"""
got the program working but accuracy is not getting above 0.333, maybe an
issue with the target data im not sure
"""
import tensorflow as tf
import pandas as pd
import numpy as np
from IPython.display import display
from sklearn import datasets

iris = datasets.load_iris()
df = pd.DataFrame(data= np.c_[iris['data'], iris['target']], columns= iris['feature_names'] + ['target'])
df1 = pd.DataFrame(iris.data, columns=iris.feature_names, )

print("complete df")
display(df)
print("df head")
display(df.head())
print("df types")
display(df.dtypes)
print("df columns")
display(df.columns)
# print("df1 data")
# display(df1.data)

target = df.target
dataset = tf.data.Dataset.from_tensor_slices((df1.values, target.values))
train_dataset = dataset.shuffle(len(df)).batch(1)

def get_compiled_model():
  model = tf.keras.Sequential([
    tf.keras.layers.Dense(4, activation='relu'),
    # tf.keras.layers.Dense(10, activation='relu'),
    tf.keras.layers.Dense(3, activation='sigmoid')
  ])

  model.compile(optimizer='adam',
                loss='binary_crossentropy',
                metrics=['accuracy'])
  return model

model = get_compiled_model()
model.fit(train_dataset, epochs=200)
# model.evaluate(x_test,  y_test, verbose=2)

# (x_train, y_train), (x_test, y_test) = df1.data
# x_train, x_test = x_train / 255.0, x_test / 255.0
# # Build the tf.keras.Sequential model by stacking layers. Choose an optimizer and loss function for training:
#
#
# model = tf.keras.models.Sequential([
#   tf.keras.layers.Flatten(input_shape=(28, 28)),
#   tf.keras.layers.Dense(128, activation='relu'),
#   tf.keras.layers.Dropout(0.2),
#   tf.keras.layers.Dense(10, activation='softmax')
# ])
#
# model.compile(optimizer='adam',
#               loss='sparse_categorical_crossentropy',
#               metrics=['accuracy'])
# # Train and evaluate the model:
#
#
# model.fit(x_train, y_train, epochs=5)
#
# model.evaluate(x_test,  y_test, verbose=2)


import json
import logging
from typing import Any, Dict, Iterable, List, Optional

import dateutil.parser

from datahub.configuration import ConfigModel
from datahub.configuration.common import AllowDenyPattern
from datahub.emitter.mce_builder import DEFAULT_ENV, get_sys_time
from datahub.ingestion.api.common import PipelineContext
from datahub.ingestion.api.source import Source, SourceReport
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.dbt_types import (
    POSTGRES_TYPES_MAP,
    SNOWFLAKE_TYPES_MAP,
    resolve_postgres_modified_type,
)
from datahub.metadata.com.linkedin.pegasus2avro.common import AuditStamp
from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
    DatasetLineageTypeClass,
    UpstreamClass,
    UpstreamLineage,
)
from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot
from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
from datahub.metadata.com.linkedin.pegasus2avro.schema import (
    BooleanTypeClass,
    DateTypeClass,
    MySqlDDL,
    NullTypeClass,
    NumberTypeClass,
    SchemaField,
    SchemaFieldDataType,
    SchemaMetadata,
    StringTypeClass,
    TimeTypeClass,
)
from datahub.metadata.schema_classes import DatasetPropertiesClass

logger = logging.getLogger(__name__)

class DBTConfig(ConfigModel):
    manifest_path: str
    catalog_path: str
    sources_path: Optional[str]
    env: str = DEFAULT_ENV
    target_platform: str
    load_schemas: bool
    node_type_pattern: AllowDenyPattern = AllowDenyPattern.allow_all()

class DBTColumn:
    name: str
    comment: str
    index: int
    data_type: str

    def __repr__(self):
        fields = tuple("{}={}".format(k, v) for k, v in self.__dict__.items())
        return self.__class__.__name__ + str(tuple(sorted(fields))).replace("'", "")

class DBTNode:
    dbt_name: str
    database: str
    schema: str
    dbt_file_path: str
    node_type: str  # source, model
    materialization: str  # table, view, ephemeral
    name: str  # name, identifier
    columns: List[DBTColumn]
    upstream_urns: List[str]
    datahub_urn: str
    max_loaded_at: Optional[str]

    def __repr__(self):
        fields = tuple("{}={}".format(k, v) for k, v in self.__dict__.items())
        return self.__class__.__name__ + str(tuple(sorted(fields))).replace("'", "")

def get_columns(catalog_node: dict) -> List[DBTColumn]:
    columns = []

    raw_columns = catalog_node["columns"]

    for key in raw_columns:
        raw_column = raw_columns[key]

        dbtCol = DBTColumn()
        dbtCol.comment = raw_column["comment"]
        dbtCol.data_type = raw_column["type"]
        dbtCol.index = raw_column["index"]
        dbtCol.name = raw_column["name"]
        columns.append(dbtCol)
    return columns

def extract_dbt_entities(
    nodes: Dict[str, Dict[str, Any]],
    catalog: Dict[str, Dict[str, Any]],
    sources_results: List[Dict[str, Any]],
    load_catalog: bool,
    target_platform: str,
    environment: str,
    node_type_pattern: AllowDenyPattern,
) -> List[DBTNode]:

    sources_by_id = {x["unique_id"]: x for x in sources_results}

    dbt_entities = []
    for key, node in nodes.items():
        dbtNode = DBTNode()

        # check if node pattern allowed based on config file
        if not node_type_pattern.allowed(node["resource_type"]):
            continue
        dbtNode.dbt_name = key
        dbtNode.database = node["database"]
        dbtNode.schema = node["schema"]
        dbtNode.dbt_file_path = node["original_file_path"]
        dbtNode.node_type = node["resource_type"]
        dbtNode.max_loaded_at = sources_by_id.get(key, {}).get("max_loaded_at")
        if "identifier" in node and not load_catalog:
            dbtNode.name = node["identifier"]
        else:
            dbtNode.name = node["name"]

        if "materialized" in node["config"].keys():
            # It's a model
            dbtNode.materialization = node["config"]["materialized"]
            dbtNode.upstream_urns = get_upstreams(
                node["depends_on"]["nodes"],
                nodes,
                load_catalog,
                target_platform,
                environment,
            )
        else:
            # It's a source
            dbtNode.materialization = catalog[key]["metadata"][
                "type"
            ]  # get materialization from catalog? required?
            dbtNode.upstream_urns = []

        if (
            dbtNode.materialization != "ephemeral" and load_catalog
        ):  # we don't want columns if platform isn't 'dbt'
            logger.debug("Loading schema info")
            dbtNode.columns = get_columns(catalog[dbtNode.dbt_name])
        else:
            dbtNode.columns = []

        dbtNode.datahub_urn = get_urn_from_dbtNode(
            dbtNode.database,
            dbtNode.schema,
            dbtNode.name,
            target_platform,
            environment,
        )

        dbt_entities.append(dbtNode)

    return dbt_entities

def loadManifestAndCatalog(
    manifest_path: str,
    catalog_path: str,
    sources_path: Optional[str],
    load_catalog: bool,
    target_platform: str,
    environment: str,
    node_type_pattern: AllowDenyPattern,
) -> List[DBTNode]:
    with open(manifest_path, "r") as manifest:
        dbt_manifest_json = json.load(manifest)

    with open(catalog_path, "r") as catalog:
        dbt_catalog_json = json.load(catalog)

    if sources_path is not None:
        with open(sources_path, "r") as sources:
            dbt_sources_json = json.load(sources)
            sources_results = dbt_sources_json["results"]
    else:
        sources_results = {}

    manifest_nodes = dbt_manifest_json["nodes"]
    manifest_sources = dbt_manifest_json["sources"]

    all_manifest_entities = {**manifest_nodes, **manifest_sources}

    catalog_nodes = dbt_catalog_json["nodes"]
    catalog_sources = dbt_catalog_json["sources"]

    all_catalog_entities = {**catalog_nodes, **catalog_sources}

    nodes = extract_dbt_entities(
        all_manifest_entities,
        all_catalog_entities,
        sources_results,
        load_catalog,
        target_platform,
        environment,
        node_type_pattern,
    )

    return nodes

def get_urn_from_dbtNode(
    database: str, schema: str, name: str, target_platform: str, env: str
) -> str:

    db_fqn = f"{database}.{schema}.{name}".replace('"', "")
    return f"urn:li:dataset:(urn:li:dataPlatform:{target_platform},{db_fqn},{env})"

def get_custom_properties(node: DBTNode) -> Dict[str, str]:
    return {
        "dbt_node_type": node.node_type,
        "materialization": node.materialization,
        "dbt_file_path": node.dbt_file_path,
    }

def get_upstreams(
    upstreams: List[str],
    all_nodes: Dict[str, dict],
    load_catalog: bool,
    target_platform: str,
    environment: str,
) -> List[str]:
    upstream_urns = []

    for upstream in upstreams:
        dbtNode_upstream = DBTNode()

        dbtNode_upstream.database = all_nodes[upstream]["database"]
        dbtNode_upstream.schema = all_nodes[upstream]["schema"]
        if "identifier" in all_nodes[upstream] and not load_catalog:
            dbtNode_upstream.name = all_nodes[upstream]["identifier"]
        else:
            dbtNode_upstream.name = all_nodes[upstream]["name"]

        upstream_urns.append(
            get_urn_from_dbtNode(
                dbtNode_upstream.database,
                dbtNode_upstream.schema,
                dbtNode_upstream.name,
                target_platform,
                environment,
            )
        )

    return upstream_urns

def get_upstream_lineage(upstream_urns: List[str]) -> UpstreamLineage:
    ucl: List[UpstreamClass] = []

    actor = "urn:li:corpuser:dbt_executor"
    sys_time = get_sys_time()

    for dep in upstream_urns:
        uc = UpstreamClass(
            dataset=dep,
            auditStamp=AuditStamp(actor=actor, time=sys_time),
            type=DatasetLineageTypeClass.TRANSFORMED,
        )
        ucl.append(uc)

    return UpstreamLineage(upstreams=ucl)

# See https://github.com/fishtown-analytics/dbt/blob/master/core/dbt/adapters/sql/impl.py
_field_type_mapping = {
    "boolean": BooleanTypeClass,
    "date": DateTypeClass,
    "time": TimeTypeClass,
    "numeric": NumberTypeClass,
    "text": StringTypeClass,
    "timestamp with time zone": DateTypeClass,
    "timestamp without time zone": DateTypeClass,
    "integer": NumberTypeClass,
    "float8": NumberTypeClass,
    **POSTGRES_TYPES_MAP,
    **SNOWFLAKE_TYPES_MAP,
}

def get_column_type(
    report: SourceReport, dataset_name: str, column_type: str
) -> SchemaFieldDataType:
    """
    Maps known DBT types to datahub types
    """
    TypeClass: Any = _field_type_mapping.get(column_type)

    if TypeClass is None:

        # attempt Postgres modified type
        TypeClass = resolve_postgres_modified_type(column_type)

    # if still not found, report the warning
    if TypeClass is None:

        report.report_warning(
            dataset_name, f"unable to map type {column_type} to metadata schema"
        )
        TypeClass = NullTypeClass

    return SchemaFieldDataType(type=TypeClass())

def get_schema_metadata(
    report: SourceReport, node: DBTNode, platform: str
) -> SchemaMetadata:
    canonical_schema: List[SchemaField] = []
    for column in node.columns:
        field = SchemaField(
            fieldPath=column.name,
            nativeDataType=column.data_type,
            type=get_column_type(report, node.dbt_name, column.data_type),
            description=column.comment,
            nullable=False,  # TODO: actually autodetect this
            recursive=False,
        )

        canonical_schema.append(field)

    actor = "urn:li:corpuser:dbt_executor"
    sys_time = get_sys_time()

    last_modified = sys_time

    if node.max_loaded_at is not None:
        last_modified = int(
            dateutil.parser.parse(node.max_loaded_at).timestamp() * 1000
        )

    return SchemaMetadata(
        schemaName=node.dbt_name,
        platform=f"urn:li:dataPlatform:{platform}",
        version=0,
        hash="",
        platformSchema=MySqlDDL(tableSchema=""),
        created=AuditStamp(time=sys_time, actor=actor),
        lastModified=AuditStamp(time=last_modified, actor=actor),
        fields=canonical_schema,
    )

class DBTSource(Source):
    """Extract DBT metadata for ingestion to Datahub"""

    @classmethod
    def create(cls, config_dict, ctx):
        config = DBTConfig.parse_obj(config_dict)
        return cls(config, ctx, "dbt")

    def __init__(self, config: DBTConfig, ctx: PipelineContext, platform: str):
        super().__init__(ctx)
        self.config = config
        self.platform = platform
        self.report = SourceReport()

    def get_workunits(self) -> Iterable[MetadataWorkUnit]:
        platform = self.platform
        nodes = loadManifestAndCatalog(
            self.config.manifest_path,
            self.config.catalog_path,
            self.config.sources_path,
            self.config.load_schemas,
            self.config.target_platform,
            self.config.env,
            self.config.node_type_pattern,
        )

        for node in nodes:

            dataset_snapshot = DatasetSnapshot(
                urn=node.datahub_urn,
                aspects=[],
            )
            custom_properties = get_custom_properties(node)

            dbt_properties = DatasetPropertiesClass(
                description=node.dbt_name,
                customProperties=custom_properties,
                tags=[],
            )
            dataset_snapshot.aspects.append(dbt_properties)

            upstreams = get_upstream_lineage(node.upstream_urns)
            if upstreams is not None:
                dataset_snapshot.aspects.append(upstreams)

            if self.config.load_schemas:
                schema_metadata = get_schema_metadata(self.report, node, platform)
                dataset_snapshot.aspects.append(schema_metadata)

            mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot)
            wu = MetadataWorkUnit(id=dataset_snapshot.urn, mce=mce)
            self.report.report_workunit(wu)

            yield wu

    def get_report(self):
        return self.report

    def close(self):
        pass


# import selenium webdriver
from selenium import webdriver
import pandas as pd
import time
from selenium.webdriver.common.by import By
from selenium_stealth import stealth
# set browser / browser options
from selenium.webdriver.chrome.options import Options

df = pd.read_csv("subscription.csv", usecols=[1])

options = Options()
driver_path = '/usr/local/bin/chromedriver'
options.binary_location = '/Applications/Brave Browser.app/Contents/MacOS/Brave Browser'
options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument('--disable-blink-features=AutomationControlled')
driver = webdriver.Chrome(options=options, executable_path = driver_path)
stealth(driver,
    languages=["en-US", "en"],
    vendor="Google Inc.",
    platform="Win32",
    webgl_vendor="Intel Inc.",
    renderer="Intel Iris OpenGL Engine",
    fix_hairline=True,
    )
# get page
driver.get("https://www.google.com")
time.sleep(30)

for channel in df["Channel Url"]:
    try:
        driver.get(channel)
        time.sleep(1)
        driver.find_element(By.XPATH, '//*[@id="subscribe-button"]/ytd-subscribe-button-renderer/tp-yt-paper-button').click()
    except:
        continue


import pygame
import math
from library import *

ColorList = [VERDE, AZUL, ROJO, BLANCO, MORADO]

if __name__ == '__main__':
	pygame.init()
	pantalla = pygame.display.set_mode([ANCHO, ALTO])
	fin = False
	origen = [ANCHO // 2, ALTO // 2]
	Plano(pantalla, origen)
	x = -50

	while x <= 50:
		y = 5*x**3 +10
		pp = CartesianoAPantalla(origen, [x, y])
		pygame.draw.circle(pantalla, VERDE, pp, 2, 2)
		x += 1

	while not fin:
		for event in pygame.event.get():
			if event.type == pygame.QUIT:
				fin = True

		pygame.display.flip()


import paho.mqtt.client as mqtt
from channels import ChannelMgr
from pprint import pprint
import time
from json import dumps, loads
from conf import load_config
import requests
import sys

def send_to_elk(client, userdata, msg):
    mid = msg.mid
    payload = loads(msg.payload.decode("utf-8"))
    timestamp = msg.timestamp
    url = msg.topic
    nodule_id = url.split('/')[-2]
    payload['nodule_id'] = nodule_id
    payload['ts'] = timestamp
    # r = requests.post("http://10.0.88.108:9200/logs/test", data=dumps(payload))
    # sys.stdout.write(r.text)
    sys.stdout.write("ELK:\t{}, {}".format(url, payload))
    return

def send_to_db(client, userdata, msg):
    mid = msg.mid
    payload = loads(msg.payload.decode("utf-8"))
    timestamp = msg.timestamp
    url = msg.topic
    # sys.stdout.write("DB:\t", url, payload)
    topic, node_name = url.split('/')
    # sys.stdout.write(node_name, topic)
    if topic == 'presence':
        sys.stdout.write("{}.presence = {}".format(node_name, payload.get('presence')))
    elif topic == 'report':
        sys.stdout.write("{} = {}".format(node_name, payload))
    return

def send_to_file(client, userdata, msg):
    mid = msg.mid
    payload = loads(msg.payload.decode("utf-8"))
    timestamp = msg.timestamp
    url = msg.topic
    time = payload['timestamp'].split('.')[0]
    values = loads(payload['value'].replace("'", '"'))
    # temp = values['temp']
    log_line = '{}, {}\n'.format(time, values)
    with open('logfile.txt', 'a') as logfile:
        logfile.write(log_line)
    return

def add_timestamp(payload):
    ts = str(datetime.now())
    payload['rec_ts'] = ts
    return payload

router = {
    'elk': send_to_elk,
    'db': send_to_db,
    'file': send_to_file,
}

routes = {
    'presence': ['elk', 'db'],
    'sensors': ['elk'],#, 'file'],
    'logs': ['elk'],
    'errors': ['elk'],
    'report': ['elk', 'db'],
}

def msg_cb(client, userdata, msg):
    topic = msg.topic.split("/")[0]
    for route in routes.get(topic):
        # try:
        r_func = router.get(route)
        r_func(client, userdata, msg)
        # except:
        #     sys.stdout.write("Topic {} not recognized".format(topic))
        #     pass

config = load_config()
name = 'bridge'
channel = ChannelMgr(name)
elk_port = 9300
elk_host = 'dmip'
client = mqtt.Client(client_id=name)

def presence_msg(connected=True):
    return dumps({'presence': 'Connected' if connected else 'Disconnected', 'node': name})

client.will_set(channel.presence(), presence_msg(False), 0, False)
client.connect(config['MQTT_URL'], config['MQTT_PORT'], config['MQTT_KEEPALIVE'])

for topic in config['topics']:
    channel_name = topic + '/#'
    client.message_callback_add(channel_name, msg_cb)
    client.subscribe(channel_name)  # Add, modify, remove, trigger, etc

client.loop_start()
client.publish(channel.presence(), presence_msg(True))

while True:
    try:
        time.sleep(1)
    except KeyboardInterrupt:
        client.disconnect()
        break
exit(0)


# -*- coding: utf-8 -*-

'''
Main operations in EMBL2checklists
'''

#####################
# IMPORT OPERATIONS #
#####################
import ChecklistOps as ClOps
import PrerequisiteOps as PreOps
import globalVariables as GlobVars

import sys, os
from Bio import SeqIO

# Import package modules of EMBL2checklists irrespective of install status
try:
    import EMBL2checklists
except ImportError:
    try:
        package_topLevel = __file__.split(__info__)[0] + __info__
        if os.path.isdir(package_topLevel):
            sys.path.append(package_topLevel)
        if not os.path.isdir(package_topLevel):
            raise ValueError('Top level of package not set.')
    except ValueError:
        try:
            package_topLevel = os.path.dirname(os.path.split(inspect.getfile(EMBL2checklists))[0])  # Homes in on `__init__.py`
            if not os.path.isdir(package_topLevel):
                raise ValueError('Top level of package not set.')
        except ValueError:
            package_topLevel = os.path.dirname(os.path.dirname(__file__))
            if not os.path.isdir(package_topLevel):
                raise ValueError('Top level of package not set.')

###############
# AUTHOR INFO #
###############
__author__ = 'Michael Gruenstaeudl <m.gruenstaeudl@fu-berlin.de>'
__copyright__ = 'Copyright (C) 2016-2018 Michael Gruenstaeudl'
__info__ = 'EMBL2checklists'
__version__ = '2018.09.18.1700'

#############
# DEBUGGING #
#############
import pdb
# pdb.set_trace()

#############
# FUNCTIONS #
#############

def EMBL2checklists(path_to_embl, path_to_outfile, file_format="embl", checklist_type=None, env_sample="no"):
    ''' Primary code function; it executes all subfunctions in the correct order
    Args:
        path_to_embl (string)
        path_to_outfile (string)
        checklist_type (string)
        file_format (string)
        env_sample (string)
    Returns:
        bool: True if nothing went wrong
    Raises:
        Pass exceptions to GUI or CMD interface
    '''

# 1. OPEN OUTHANDLE
    outp_handle = []

# 2. PARSE DATA FROM EMBL-FILE
    try:
        seq_records = SeqIO.parse(open(path_to_embl, "r"), file_format)
    except:
        raise Exception('ERROR: The file ´%s´ could not be parsed.' % (path_to_embl))

# 3. CONVERSION TO CHECKLISTS
    for counter, seq_record in enumerate(seq_records): # Looping through records
        try:
            outdict = {}
    # 3.1. Extraction of marker abbreviations
            marker_abbrevs = ClOps.Parser().parse_marker_abbrevs(seq_record, GlobVars.standard_qualifiers)

    # 3.2. Check if minimal feature prerequisites present; if not, skip this seq_record
            try:
                PreOps.Checker().checkMinimalFeaturePrerequisites(checklist_type, marker_abbrevs)
            except Exception as e:
                GlobVars.warnings.append('WARNING: %s \nSelected checklist type: `%s`; Identified marker abbrevs: `%s` \nSkipping record `%s`...' % (e, checklist_type, marker_abbrevs, seq_record.id))
                continue # Close current loop iteration and continue with next iteration

    # 3.3. Conversion to checklist format
            outdict = ClOps.Parser().mandatoryQualifiers(seq_record, marker_abbrevs, counter, checklist_type, env_sample)

    # 3.4. Optional qualifiers will be added as long as outdict true
            if outdict:
                optional_qualifiers = ClOps.Parser().optionalQualifiers(seq_record, GlobVars.GlobalVariables().getQualifiers(checklist_type, 'o'))
                outdict.update(optional_qualifiers)

    # 3.5. Write parsed record information to handle
            outp_handle.append(outdict)

        except Exception as error:
            raise error

# 4. CLOSE OUTFILE (unless uncaught exceptions)
    outp_file = open(path_to_outfile,"w")
    ClOps.Writer().writer(checklist_type, outp_handle, outp_file)
    outp_file.close()

# 5. RETURN TRUE (unless uncaught exceptions)
    return True


import os
from tkinter import image_names
import cv2 as cv
import numpy as np

src_path = "/home/ur10/continual_data/motor_rider_data_merge/"
files = os.listdir(src_path)
sorted_files = sorted(files)

#TODO - get the image bounding box in the correct shape
# Get the Intersection between the motorcycle and the rider
# Store the points as input data

def read_annotation():
    total_motor_boxes = []
    total_rider_boxes = []
    for filename in sorted_files:
        motor_boxes = []
        rider_boxes = []
        if filename.endswith('.txt'):
            image_name = filename.replace('.txt', '.jpg')
            strings = []
            with open(src_path+filename) as file:
                if os.path.getsize(src_path+filename) <= 0:
                    print("No annotation")

                img  = cv.imread(src_path+image_name)
                height, width = img.shape[0], img.shape[1]
                m_box = []
                r_box = []

                for line in file:
                    clas, x, y, w, h = map(float, line.split(' '))

                    if clas == 0: # rider class
                        strings.append(line)
            with open(src_path+filename, 'w') as file:
                if len(strings) == 0:
                    new_line = '0 0.0 0.0 0.0 0.0'
                    strings.append(new_line)
                for line in strings:
                    new_line = '0' + line[1:]
                    print(new_line)
                    file.write(new_line)
                # input_boxes = get_motor_rider_intersection(m_box, r_box)
                # for pair_box in input_boxes:
                #     color = list(np.random.random(size=3) * 256)
                #     for box in pair_box:
                #         cv.rectangle(img, (box[0], box[1]), (box[2], box[3]), color)
                # cv.imshow('Imaeg',img)
                # cv.waitKey(0)
                # cv.destroyAllWindows()
    return motor_boxes, rider_boxes

def check_intersection(box1, box2):
    x1, y1, width1, height1 = box1[0], box1[1], abs(box1[2]- box1[0]), abs(box1[3] - box1[1])
    x2, y2, width2, height2 = box2[0], box2[1], abs(box2[2]- box2[0]), abs(box2[3] - box2[1])

    # Calculate the coordinates of the corners of the boxes
    top_left1 = (x1, y1)
    top_right1 = (x1 + width1, y1)
    bottom_left1 = (x1, y1 + height1)
    bottom_right1 = (x1 + width1, y1 + height1)

    top_left2 = (x2, y2)
    top_right2 = (x2 + width2, y2)
    bottom_left2 = (x2, y2 + height2)
    bottom_right2 = (x2 + width2, y2 + height2)

    # blank_image = np.ones((1400, 1200, 3), np.uint8) * 255
    # cv.rectangle(blank_image,top_left1, bottom_right1, (0,0,255))
    # cv.rectangle(blank_image, top_left2, bottom_right2, (0,0,255))
    # cv.imshow('Imaeg', blank_image)
    # cv.waitKey(0)
    # cv.destroyAllWindows()

    # if (bottom_left1[1] < top_left2[1]) or (bottom_right1[1] > top_left2[1]):
    #     return False
    # if (bottom_right1[0] < top_left2[0]) or (bottom_left1[0] > top_right2[0]):
    #     return False
    # if (bottom_left2[1] < top_left1[1]) or (bottom_right2[1] > top_left1[1]):
    #     return False
    # if (bottom_right2[0]  < top_left1[0]) or (bottom_left2[0] > top_right1[0]):
    #     return False
    #
    # return True

    # # Check for intersection
    if (top_left1[0] <= bottom_right2[0] and bottom_right1[0] >= top_left2[0] and
        top_left1[1] <= bottom_right2[1] and bottom_right1[1] >= top_left2[1]):
        return True  # Boxes intersect
    else:
        return False  # Boxes do not intersect

def get_motor_rider_intersection(motor_boxes, rider_boxes):
    input_boxes = []

    for m_box in motor_boxes:
        pair_boxes = [m_box]
        for r_box in rider_boxes:
            if check_intersection(m_box, r_box):
                if len(pair_boxes) < 5:
                    pair_boxes.append(r_box)
        input_boxes.append(pair_boxes)

    return input_boxes

def get_input_data(input_boxes, width, height):
    input_data = []

    for pair_boxes in input_boxes:
        pair_data = []
        for box in pair_boxes:
            x_c = (box[0] + box[2])/(2*width)
            y_c = (box[1] + box[3])/(2*height)
            w_m = abs(box[0]-box[1])/width
            h_m = abs(box[1]-box[3])/height

            pair_data.append([x_c,y_c, w_m, h_m])
        input_data.append(pair_data)
    return input_data

def main():
    motor_boxes, rider_boxes = read_annotation()
    # input_boxes = get_motor_rider_intersection(motor_boxes, rider_boxes)

if __name__ == "__main__":
    main()

		# count = count + 1
# print(len(empty_files))


import os
import shutil
import dropbox
import requests
import platform
from . import header
from .dropboxhasher import DropboxContentHasher

# OS info
OS_TYPE = platform.system().lower()
ARCH_TYPE = platform.architecture()[0].lower()
CLOUD_DIR = '/%s/%s' % (OS_TYPE, ARCH_TYPE)

def __checksum(path: str) -> str:
    algorithm = DropboxContentHasher()

    with open(path, 'rb') as file:
        for chunk in iter(lambda: file.read(4096), b''):
            algorithm.update(chunk)

    return algorithm.hexdigest()

def __get_files() -> list:
    client = dropbox.Dropbox(oauth2_access_token=header.ACCESS_TOKEN, timeout=None)
    result = []

    try:
        maximum_limit = 2000
        for file in client.files_list_folder(CLOUD_DIR, recursive=True, limit=maximum_limit).entries:
            if file.path_lower != CLOUD_DIR:
                result.append(file)
    except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError):
        pass
    finally:
        client.close()
        return result

def download():
    files = __get_files()
    client = dropbox.Dropbox(oauth2_access_token=header.ACCESS_TOKEN, timeout=None)
    os.makedirs(header.UPDATE_DIR, exist_ok=True)

    for file in files:
        if not isinstance(file, dropbox.files.FileMetadata):
            continue

        file_path = os.path.relpath(file.path_display, CLOUD_DIR)
        temp_path = os.path.join(header.UPDATE_DIR, file_path)
        app_path = os.path.join(header.APP_DIR, file_path)
        in_temp = os.path.exists(temp_path) and file.content_hash == __checksum(temp_path)
        in_app = os.path.exists(app_path) and file.content_hash == __checksum(app_path)

        if not in_temp and not in_app:
            os.makedirs(os.path.dirname(temp_path), exist_ok=True)
            client.files_download_to_file(download_path=temp_path, path=file.id)

    client.close()

def update() -> bool:
    valid = False
    if os.path.exists(header.UPDATE_DIR) and len(os.listdir(header.UPDATE_DIR)) > 0:
        shutil.copytree(header.UPDATE_DIR, header.APP_DIR, dirs_exist_ok=True)
        shutil.rmtree(header.UPDATE_DIR, ignore_errors=True)
        valid = True

    return valid


from fractions import Fraction

import cv2

from .stages.initialize_stage import InitializeStage, InitializeStageConfig
from .tools.video_stream import PiCameraConfig, PiCameraStream

camera_config = PiCameraConfig(
    framerate_range=(Fraction(10, 1), Fraction(30, 1)),
    resolution=(1280, 720),
    iso=800,
    exposure_mode="off",
    awb_mode="off",
    hflip=True,
    vflip=True,
)

def main() -> None:
    camera = None
    try:
        camera = PiCameraStream(camera_config)
        camera.start()

        detector = InitializeStage()
        detector_config = InitializeStageConfig(camera)
        detector.set_config(detector_config, None)
        detector.run()
    finally:
        cv2.destroyAllWindows()

        if camera is not None:
            camera.stop()

if __name__ == "__main__":
    main()


import os
import torch
import torch.nn as nn
# import curve
import platform
import dataset
from torch.utils.data import DataLoader,Dataset
import numpy as np
import argparse, time, random
import yaml
from yaml.loader import SafeLoader
from evaluation import *
import glob
import model
import net
from saver import Saver
from tqdm import tqdm
import matplotlib.pyplot as plt
from matplotlib.ticker import MultipleLocator
from sklearn.metrics import roc_curve, auc
from tensorboardX import SummaryWriter
from sklearn.metrics import cohen_kappa_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score, recall_score, f1_score
from sklearn.metrics import confusion_matrix
import gc
from sklearn import metrics
import h5py
from utils import *
import pandas as pd
from skimage import io

from apex import amp
"""
label 2016={ 0:'G2_O', 1:'G3_O', 2:'G2_A', 3:'G3_A', 4:'G2_OA', 5:'G3_OA', 6:'GBM'}
label 2021={ 0:'G2_O', 1:'G3_O', 2:'G2_A', 3:'G3_A', 4:'G4_A', 5:'GBM'}
"""

def train(opt):

    gpuID = opt['gpus']

    ############## Init #####################################

    device = torch.device('cuda:{}'.format(gpuID[0])) if gpuID else torch.device('cpu')

    Res_pretrain= net.Res50_pretrain().cuda(gpuID[0])
    # Res_pretrain.to(device)
    Res_pretrain = nn.DataParallel(Res_pretrain, device_ids=gpuID)
    Res_pretrain.eval()

    ###############  Datasets #######################

    trainDataset = dataset.Our_Dataset(phase='Train',opt=opt)
    valDataset = dataset.Our_Dataset(phase='Val', opt=opt)
    testDataset = dataset.Our_Dataset(phase='Test',opt=opt)
    trainLoader = DataLoader(trainDataset, batch_size=opt['batchSize'],
                             num_workers=opt['nThreads'], shuffle=True)
    valLoader = DataLoader(valDataset, batch_size=opt['Val_batchSize'],
                             num_workers=opt['nThreads'], shuffle=True)
    testLoader = DataLoader(testDataset, batch_size=opt['Test_batchSize'],
                            num_workers=opt['nThreads'], shuffle=True)

    train_bar = tqdm(trainLoader)
    for packs in train_bar:
        img = packs[0][0] #(N,3,224,224)
        imgPath = packs[2][0]
        patches_coor = packs[3][0] # list N,2
        if torch.cuda.is_available():
            img = img.cuda(gpuID[0])
        N=img.detach().cpu().numpy().shape[0]

        feature = Res_pretrain(img) # N 1024
        feature = torch.unsqueeze(feature, dim=0)
        feature_save = feature.detach().cpu().numpy()
        feature_save=np.float16(feature_save)
        print(feature_save.shape)
        if not os.path.exists(opt['dataDir'] +'Res50_feature_2000_fixdim0_512/'):
            os.makedirs(opt['dataDir'] +'Res50_feature_2000_fixdim0_512/')
        with h5py.File(opt['dataDir'] +'Res50_feature_2000_fixdim0_512/'+ imgPath+'.h5', 'w') as f:
            f['Res_feature'] = feature_save
            f['patches_coor'] = patches_coor

    train_bar = tqdm(valLoader)
    for packs in train_bar:
        img = packs[0][0]  # (N,3,224,224)
        imgPath = packs[2][0]
        patches_coor = packs[3][0]  # list N,2
        if torch.cuda.is_available():
            img = img.cuda(gpuID[0])
        N = img.detach().cpu().numpy().shape[0]

        feature = Res_pretrain(img)  # N 1024
        feature = torch.unsqueeze(feature, dim=0)
        feature_save = feature.detach().cpu().numpy()
        feature_save = np.float16(feature_save)
        print(feature_save.shape)
        if not os.path.exists(opt['dataDir'] + 'Res50_feature_2000_fixdim0_512/'):
            os.makedirs(opt['dataDir'] + 'Res50_feature_2000_fixdim0_512/')
        with h5py.File(opt['dataDir'] + 'Res50_feature_2000_fixdim0_512/' + imgPath + '.h5', 'w') as f:
            f['Res_feature'] = feature_save
            f['patches_coor'] = patches_coor
    #
    train_bar = tqdm(testLoader)
    for packs in train_bar:
        img = packs[0][0]  # (N,3,224,224)
        imgPath = packs[2][0]
        patches_coor = packs[3][0]  # list N,2
        if torch.cuda.is_available():
            img = img.cuda(gpuID[0])
        N = img.detach().cpu().numpy().shape[0]

        feature = Res_pretrain(img)  # N 1024
        feature = torch.unsqueeze(feature, dim=0)
        feature_save = feature.detach().cpu().numpy()
        feature_save = np.float16(feature_save)
        print(feature_save.shape)
        if not os.path.exists(opt['dataDir'] + 'Res50_feature_2000_fixdim0_512/'):
            os.makedirs(opt['dataDir'] + 'Res50_feature_2000_fixdim0_512/')
        with h5py.File(opt['dataDir'] + 'Res50_feature_2000_fixdim0_512/' + imgPath + '.h5', 'w') as f:
            f['Res_feature'] = feature_save
            f['patches_coor'] = patches_coor
        # features_WSI=[]
        # for i in range(N):
        #     feature=Res_pretrain(torch.unsqueeze(img[i],dim=0))
        #     feature=feature.detach().cpu().numpy()
        #     features_WSI.append(feature)
        # features_WSI=np.asarray(features_WSI) #(N,1024)
        # feature_save = np.expand_dims(features_WSI, axis=0)
        # feature_save = np.float16(feature_save)
        # print(feature_save.shape)
        # if not os.path.exists(opt['dataDir'] + 'Res50_feature_1200_iso_512/'):
        #     os.makedirs(opt['dataDir'] + 'Res50_feature_1200_iso_512/')
        # with h5py.File(opt['dataDir'] + 'Res50_feature_1200_iso_512/' + imgPath + '.h5', 'w') as f:
        #     f['Res_feature'] = feature_save
        #     f['patches_coor'] = patches_coor

        a=1

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('--opt', type=str, default='config/miccai.yml')
    args = parser.parse_args()
    with open(args.opt) as f:
        opt = yaml.load(f, Loader=SafeLoader)
    #
    # k1 = h5py.File(opt['dataDir'] + 'Res50_feature_1/TCGA-DH-5143-01Z-00-DX1.h5')['Res50_feature'][:][0,100,:]
    # k2 = h5py.File( './TCGA-DH-5143-01Z-00-DX1.h5')['Res50_feature'][:][0,100,:]

    # k3 = h5py.File(opt['dataDir'] + 'Res50_feature_1/TCGA-06-0876-01Z-00-DX1.h5')['Res50_feature'][:][0,0,100,:]
    # k4 = h5py.File(opt['dataDir'] + 'Res50_feature_2/TCGA-06-0876-01Z-00-DX1.h5')['Res50_feature'][:][0,100,:]
    #
    # k5 = h5py.File(opt['dataDir'] + 'Res50_feature_1/TCGA-CS-6290-01A-01-TS1.h5')['Res50_feature'][:][0,0,100,:]
    # k6 = h5py.File(opt['dataDir'] + 'Res50_feature_2/TCGA-CS-6290-01A-01-TS1.h5')['Res50_feature'][:][0,100,:]

    a = 1
    train(opt)

    a=1


import os
import logging
import multiprocessing as mp
from common import utils
from common import health_check
from election import leader_election

config = utils.initialize_config()
LOGGING_LEVEL = config['GENERAL']['logging_level']
utils.initialize_log(LOGGING_LEVEL)

health_check_process = mp.Process(target=health_check.echo_server, daemon=False)
health_check_process.start()

logging.basicConfig(level=logging.DEBUG)

ELECTION_ID = int(os.environ['ELECTION_ID'])
ELECTION_REPLICAS_AMOUNT = int(os.environ['ELECTION_REPLICAS_AMOUNT'])
SERVICES = os.environ['SERVICES'].split(',')
RETRIES = int(os.environ['RETRIES'])

def dummy():
	health_check.monitor(SERVICES, RETRIES)

le = leader_election.LeaderElection(ELECTION_REPLICAS_AMOUNT, ELECTION_ID, dummy)
le.run()

health_check_process.kill()
health_check_process.join()


import os
import numpy as np
import torch.nn as nn
import torch

def create_folder(path: str) -> None:
    """ Creates a folder if it does not exist

    Parameters:
    -----------
    path : str
        Path to the folder to be created

    Returns:
    --------
    None

    """

    if not os.path.exists(path):
        os.mkdir(path)

def mae(predicted: np.ndarray, actual: np.ndarray) -> np.ndarray:
    """Returns Mean Absolute Error

    Parameters:
    -----------
    predicted : np.ndarray
        Predicted values
    actual : np.ndarray
        Actual values

    Returns:
    --------
    absolute_error : np.ndarray
        Mean Absolute Error

    """

    absolute_error = np.abs(np.subtract(predicted, actual))

    if absolute_error.shape[0] == 1:
        return absolute_error
    else:
        return np.mean(absolute_error)

def my_KLDivLoss(prediction: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
    """Returns K-L Divergence loss
    Different from the default PyTorch nn.KLDivLoss in that
    a) the result is averaged by the 0th dimension (Batch size)
    b) the y distribution is added with a small value (1e-16) to prevent log(0) problem

    Parameters:
    -----------
    prediction : torch.Tensor
        Predicted values
    target : torch.Tensor
        Actual values

    Returns:
    --------
    loss : torch.Tensor
        K-L Divergence loss

    """

    prediction.cpu()
    target.cpu()

    loss_func = nn.KLDivLoss(reduction='sum')
    target += 1e-16
    target_shape = target.shape[0]
    loss = loss_func(prediction, target) / target_shape
    return loss


import json
from rest_framework.response import Response
from rest_framework import permissions
from rest_framework import viewsets
from rest_framework.decorators import action
from .models import Sheets, Series, Program
from .serializers import SheetSerializer, ProgramSerializer

# Create your views here.

class ProjectViewSet(viewsets.ModelViewSet):
    permission_classes = (permissions.AllowAny,)

    @action(methods=['get'], detail=False, url_path='status')
    def get_status(self, request):
        return Response({"status": "Online"})

    @action(methods=['get'], detail=False, url_path='getAllSeries')
    def get_series(self, request):
        qs = Series.objects.all()
        data = []
        for series in qs:
            data.append({'id': series.id, 'text': series.name})

        return Response({"series": data})

    @action(methods=['get'], detail=True, url_path='getAllPrograms')
    def get_programs(self, request, pk):
        qs = Sheets.objects.filter(series_id=pk)
        program_qs = Program.objects.all()
        data = []
        for sheets in qs:
            for program in program_qs:
                index = next((index for (index, d) in enumerate(data) if d["id"] == program.id), None)
                if sheets.id in program.sheets and index == None:
                    data.append({'id': program.id, 'text': program.name})

        return Response({"data": data})

    @action(methods=['get'], detail=True, url_path='getAllSheets')
    def get_sheets(self, request, pk):
        qs = Program.objects.get(id=pk)
        sheets = Sheets.objects.filter(id__in=qs.sheets)

        data = SheetSerializer(sheets, context={'request': request}, many=True). data

        return Response({"data": data})

    @action(methods=['get'], detail=True, url_path='getAllHeaders')
    def get_headers(self, request, pk):
        qs = Program.objects.get(id=pk)

        return Response({"data": qs.headers})

    @action(methods=['post'], detail=False, url_path='getTimeSeries')
    def get_time_series(self, request):
        data = request.data
        sheets_qs = Sheets.objects.filter(id__in=data['sheets'])
        time_series = []
        time_series_list = []
        for sheets in sheets_qs:
            for index, time in enumerate(sheets.Time_cycle):
                time_series_list.append(time)
                sensor = []
                for headers in data['headers']:
                    value = sheets.__getattribute__(headers)
                    if value[index] == '':
                        value[index] = 0
                    print(value)
                    sensor.append(float(value[index]))
                index = next((index for (index, d) in enumerate(time_series) if d["TIME"] == time), None)
                if not index:
                    obj = {
                        "TIME": time,
                        "SENSOR_DATA": sensor
                    }
                    time_series.append(obj)
                else:
                    time_series[index]['SENSOR_DATA'] += sensor

        return Response({"data": time_series, 'times': time_series_list})

    @action(methods=['post'], detail=False, url_path='signalTransform')
    def get_signal(self, request):
        data = request.data
        import numpy as np
        from scipy.fft import fft, ifft
        signal = []
        for time in data['time_series']:
            x = np.array(time['data'])
            y = fft(x)
            value = []
            for data in y:
                value.append(data.__float__())

            signal.append({
                'name': time['name'],
                'data': value
            })

        return Response({"data": signal})


from django import forms
from .models import UserProfile

# Class to display a form based on the UserProfile model
class UserProfileForm(forms.ModelForm):

    class Meta:
        model = UserProfile
        exclude = ('user',)

    gender = forms.ChoiceField(label='Gender', choices=[('Male', 'Male'), ('Female', 'Female'), ('Other', 'Other')], widget=forms.RadioSelect(), required=False)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        placeholders = {
            'date_of_birth': 'YYYY-MM-DD',
        }

        for field_name, field in self.fields.items():
            field.widget.attrs['class'] = 'border-dark rounded-0'

        self.fields['initials'].widget.attrs['autofocus'] = True
        for field in self.fields:
            if field == 'date_of_birth':
                placeholder = placeholders[field]
                self.fields[field].widget.attrs['placeholder'] = placeholder


import pytest
from app.model.application_model import ApplicationModel

STUDENTS = [
    (
        'Jessica',
        '2000',
        'f',
        '1',
        'christian',
        'no skill',
        'Marcos',
        '',
        'Monica',
        '',
    )
]

UPDATED_STUDENTS = [
    (
        1,
        'Carla Jessica',
        '2000',
        'f',
        '1',
        'christian',
        'no skill',
        'Marcos',
        '',
        'Monica',
        '',
    )
]

@pytest.fixture(scope='module')
def model_in_memory():
    model = ApplicationModel(':memory:')
    yield model
    model.close()

@pytest.mark.parametrize(
    'name, birthday, gender, classroom, religion, skill, '
    'father_name, father_ocupation, mother_name, mother_ocupation',
    STUDENTS,
)
def test_create_student_must_insert_student_register(
    model_in_memory: ApplicationModel,
    name,
    birthday,
    gender,
    classroom,
    religion,
    skill,
    father_name,
    father_ocupation,
    mother_name,
    mother_ocupation,
):
    primary_key = model_in_memory.create_student(
        name,
        birthday,
        gender,
        classroom,
        religion,
        skill,
        father_name,
        father_ocupation,
        mother_name,
        mother_ocupation,
    )

    assert isinstance(primary_key, int)
    register = model_in_memory.search_student_by_pk(primary_key)
    assert register['name'] == name
    assert register['birthday'] == birthday
    assert register['gender'] == gender
    assert register['classroom'] == classroom
    assert register['religion'] == religion
    assert register['skill'] == skill
    assert register['father_name'] == father_name
    assert register['father_ocupation'] == father_ocupation
    assert register['mother_name'] == mother_name
    assert register['mother_ocupation'] == mother_ocupation

def test_search_students_by_name_must_return_all_registers(
    model_in_memory: ApplicationModel,
):
    registers = model_in_memory.search_students_by_name('')
    assert len(registers) == len(STUDENTS)

def test_search_student_by_pk_must_return_single_register(
    model_in_memory: ApplicationModel,
):
    register = model_in_memory.search_student_by_pk(1)
    assert register is not None

@pytest.mark.parametrize(
    'primary_key, name, birthday, gender, classroom, religion, skill, '
    'father_name, father_ocupation, mother_name, mother_ocupation',
    UPDATED_STUDENTS,
)
def test_update_student_must_update_student_register(
    model_in_memory: ApplicationModel,
    primary_key,
    name,
    birthday,
    gender,
    classroom,
    religion,
    skill,
    father_name,
    father_ocupation,
    mother_name,
    mother_ocupation,
):

    rowcount = model_in_memory.update_student(
        primary_key,
        name,
        birthday,
        gender,
        classroom,
        religion,
        skill,
        father_name,
        father_ocupation,
        mother_name,
        mother_ocupation,
    )

    assert rowcount == 1
    register = model_in_memory.search_student_by_pk(primary_key)
    assert register['name'] == name
    assert register['birthday'] == birthday
    assert register['gender'] == gender
    assert register['classroom'] == classroom
    assert register['religion'] == religion
    assert register['skill'] == skill
    assert register['father_name'] == father_name
    assert register['father_ocupation'] == father_ocupation
    assert register['mother_name'] == mother_name
    assert register['mother_ocupation'] == mother_ocupation

@pytest.mark.parametrize(
    'primary_key', [index + 1 for index in range(len(STUDENTS))]
)
def test_delete_student_must_delete_register(
    model_in_memory: ApplicationModel, primary_key
):
    rowcount = model_in_memory.delete_student(primary_key)
    assert rowcount == 1
    assert model_in_memory.search_student_by_pk(primary_key) is None

def test_delete_student_must_return_rowcount_zero_if_pk_notexists(
    model_in_memory: ApplicationModel,
):
    rowcount = model_in_memory.delete_student(-999)
    assert rowcount == 0


"""
    user
    ~~~

    :author: dilless(Huangbo)
    :date: 2022/7/10
"""
import asyncio
import datetime
import hashlib
import random
import uuid
from typing import Optional

from fastapi import APIRouter, Path, Query, Depends
from sqlmodel import Session, select
from starlette.requests import Request

import crud
import models
from api import deps
from app import schemas
from core.config import settings
from db.mysql import engine
from utils.redis_ import aio_redis, get_login_code_key, get_login_user_obj_key, SignUtils

router = APIRouter()

@router.post('/code',
             response_model=schemas.GenericResponseModel)
async def send_code(
        *,
        phone: str = Query(),
):
    code = ''.join(random.sample(list(map(str, range(0, 10))), 6))
    await aio_redis.setex(get_login_code_key(phone), settings.LOGIN_CODE_EXPIRE, code)
    print('发送验证码成功，验证码：{}'.format(code))
    return schemas.GenericResponseModel()

@router.post('/login',
             response_model=schemas.GenericResponseModel)
async def login(
        *,
        login_model: schemas.UserLoginModel,
):
    cached_code = await aio_redis.get(get_login_code_key(login_model.phone))
    if login_model.code is None or cached_code != login_model.code:
        return schemas.GenericResponseModel(success=False, error_msg='验证码错误')

    user = crud.user_crud.get_user_by_phone(phone=login_model.phone)
    if user is None:
        user = crud.user_crud.create_user_with_phone(phone=login_model.phone)

    token = hashlib.md5(str(user.id).encode()).hexdigest()
    base_user = schemas.UserBaseModel.parse_obj(user)
    await aio_redis.hmset(get_login_user_obj_key(token), base_user.dict(exclude_none=True))
    await aio_redis.expire(get_login_user_obj_key(token), settings.LOGIN_USER_OBJ_EXPIRE)

    return schemas.GenericResponseModel(data=token)

@router.post('/logout',
             response_model=schemas.GenericResponseModel)
async def logout(
        *,
        user: schemas.UserBaseModel = Depends(deps.verify_user),
):
    pass

@router.get('/me',
            response_model=schemas.GenericResponseModel)
async def me(
        *,
        user: schemas.UserBaseModel = Depends(deps.verify_user),

):
    return schemas.GenericResponseModel(data=schemas.UserBaseModel.parse_obj(user))

@router.post('/sign',
             response_model=schemas.GenericResponseModel)
async def sign(
        *,
        user: schemas.UserBaseModel = Depends(deps.verify_user),
):
    await SignUtils.sign(datetime.datetime.now(), user.id)
    return schemas.GenericResponseModel()

@router.get('/sign/count',
            response_model=schemas.GenericResponseModel)
async def sign_count(
        *,
        user: schemas.UserBaseModel = Depends(deps.verify_user),
):
    count = await SignUtils.sign_count(datetime.datetime.now(), user.id)
    return schemas.GenericResponseModel(data=count)

@router.get('/{user_id}',
            response_model=schemas.GenericResponseModel)
async def get_user(
        *,
        user_id: int = Path(),
):
    user = crud.user_crud.get_by_id(pk=user_id)
    return schemas.GenericResponseModel(data=schemas.UserBaseModel.parse_obj(user))

@router.get('/info/{user_id}',
            response_model=schemas.GenericResponseModel)
async def info(
        *,
        user_id: int = Path()
):
    with Session(engine) as sess:
        statement = select(models.User).where(models.UserInfo.user_id == user_id)
        result = sess.exec(statement).first()
    return schemas.GenericResponseModel(data=result)

if __name__ == '__main__':
    async def do_job(u, lock):
        code = await send_code(phone=u.phone)
        login_obj = schemas.UserLoginModel(
            phone=u.phone,
            code=code,
        )
        login_resp = await login(login_model=login_obj)
        token = login_resp.data
        async with lock:
            with open('../../../token.txt', 'a', encoding='utf-8') as f:
                f.write(token + '\n')

    async def main():
        with Session(engine) as sess:
            users = sess.exec(select(models.User)).all()[:1000]
        lock = asyncio.Lock()
        for u in users:
            await do_job(u, lock)

    asyncio.run(main())


# 인스타그램 에서 해시태그 수집

# http://www.instagram.com/randyolson
# SPA - singgle page application 단일페이지로 웹 어플리케이션 생성
# SPA 특성상 서버로부터 데이터를 일겅와서
# script 태그를 이용해서 _shareData 변수를 만들고
# 이 변수에 데이터를 저장해 둠
# 이미지 클릭시 자바스크립트를 이용해서 UI를 생성후 출력

# request + beatifulsoup
import pprint

from bs4 import BeautifulSoup
import requests

import json
import re

url = 'http://www.instagram.com/randyolson'
ua = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36'}

res = requests.get(url, headers= ua)
html =BeautifulSoup(res.text,'lxml')

# 본문에서 script 태그에 정의된 _sharedData 변수를 찾아냄
body = html.find('body')
print(body)
print('\r\n')
script = body.find('script')
print(script)
print('\r\n')

# script 태그내 문자열만 추출함
raw_json_data = script.text.strip() # 문자열추출, 공백제거
raw_json_data = raw_json_data.replace('window._sharedData = ','').replace(';','')
print(raw_json_data)
print('\r\n')
print('----------------------------------------------------------------------------')

# 전처리된 json 데이터 확인 - 예쁘게 출력
# pprint.pprint(raw_json_data, indent=2)
# print('\r\n')

# json 형식으로 데이터를 메모리에 불러온 후
# 필요한 데이터 추출

#화면에 출력된 게시물 관련 데이터는 총 12개
json_data = json.loads(raw_json_data)

# for i in range(0,12):
#     post_list = json_data['entry_data']['ProfilePage'][0]
#     post_list = post_list['graphql']['user']
#     post_list = post_list['edge_owner_to_timeline_media']
#     post_list = post_list['edges'][i]['node']
#     post_list = post_list['edge_media_to_caption']['edges']
#     tag_list = post_list[0]['node']['text']
#
# # pprint.pprint(tag_list, indent=1)
#
# # 게시물 본문을 #으로 구분지어서 분리한 뒤 리스트에 저장
# tags = tag_list.split('#')
#
# for j in range(1,len(tags)):
#     print(tags[j])

# 정규 표현식으로 해쉬 태그 찾기
# #으로 시작하는 단어를 모두 찾아서 리스트에 저장
tags = re.findall(r'#\w+',raw_json_data)
print(len(tags))

# 리스트에 저장된 해쉬태그를 반복문으로 출력
for i in range(0,len(tags)):
    print(tags[i])

# requests는 브라우저 스크롤 기능이 없음
# 인스타그램 첫페이지에 대해 해쉬태그 추출하고
# 두번째 페이지 부터는 수집불가 - selenium 이용


# -*- coding: utf-8 -*-

"""
Turkish sentence and word tokenizers
"""

__all__ = ['SentenceTokenizer', 'WordTokenizer']
__version__ = '0.0.0.1'

import os
import pathlib
from enum import Enum
from typing import Optional, Tuple, Union

import regex as re

from trtokenizer.errors import NonPrefixFileNotExistError

class SentenceTokenizer:
    class PrefixType(Enum):
        DEFAULT: int
        NUMERIC_ONLY: int

        DEFAULT = 1
        NUMERIC_ONLY = 2

    __slots__ = (
        # Dictionary of non-breaking prefixes; keys are string prefixes, values are PrefixType enums
        '__non_breaking_prefixes',
        'non_breaking_prefix_file',
        'pre_compiled_regexes'
    )

    def __repr__(self):
        if self.non_breaking_prefix_file:
            return f'Sentence tokenizer integrated with "{self.non_breaking_prefix_file}" look-up table'
        else:
            return f'Sentence tokenizer not integrated with look-up table'

    def __init__(self, non_breaking_prefix_file: Optional[str] = None):
        """Sentence tokenizer

        Parameters
        ----------
        non_breaking_prefix_file : str, optional
            path to non-breaking prefix file.

        Raises
        ------
        NonBreakingPrefixFileError
            when non breaking file for given language not exist in abbrev_sentence_tokenizer folder

        """

        self.non_breaking_prefix_file: Optional[str] = non_breaking_prefix_file
        self.__non_breaking_prefixes: dict
        self.pre_compiled_regexes: dict
        line: str
        prefix_type: int
        item: int

        if self.non_breaking_prefix_file is None:
            self.non_breaking_prefix_file = str(
                pathlib.Path(__file__).parent.resolve() / "tr_non_suffixes"
            )

        if not os.path.isfile(self.non_breaking_prefix_file):
            raise NonPrefixFileNotExistError(self.non_breaking_prefix_file)

        self.__non_breaking_prefixes = {}
        with open(self.non_breaking_prefix_file, mode='r', encoding='utf-8') as fp_prefix_file:
            for line in fp_prefix_file.readlines():

                if '#NUMERIC_ONLY#' in line:
                    prefix_type = SentenceTokenizer.PrefixType.NUMERIC_ONLY
                else:
                    prefix_type = SentenceTokenizer.PrefixType.DEFAULT

                # Remove comments
                line = re.sub(pattern=r'#.*', repl='', string=line, flags=re.DOTALL | re.UNICODE)

                line = line.strip()

                if not line:
                    continue

                self.__non_breaking_prefixes[line] = prefix_type

        prefix_type = SentenceTokenizer.PrefixType.DEFAULT
        for item in range(1, 10000):
            self.__non_breaking_prefixes[str(item)] = prefix_type

        # boost regex performance
        self.pre_compiled_regexes = {
            1: re.compile(
                r'([?!]) *([\'"([\u00bf\u00A1\p{Initial_Punctuation}]*[\p{Uppercase_Letter}\p{Other_Letter}])',
                re.U
            ),
            2: re.compile(  # + changed to *
                r'(\.[\.]+) *([\'"([\u00bf\u00A1\p{Initial_Punctuation}]*[\p{Uppercase_Letter}\p{Other_Letter}])',
                re.U
            ),
            3: re.compile(
                r'([?!\.][\ ]*[\'")\]\p{Final_Punctuation}]+) +([\'"([\u00bf\u00A1\p{Initial_Punctuation}]*[\ ]*'
                r'[\p{Uppercase_Letter}\p{Other_Letter}])',
                re.U
            ),
            4: re.compile(
                r'([?!\.]) +([\'"[\u00bf\u00A1\p{Initial_Punctuation}]+[\ ]*[\p{Uppercase_Letter}\p{Other_Letter}])',
                re.U
            ),
            5: re.compile(r' +', re.U),
            6: re.compile(r'([\w\.\-]*)([\'\"\)\]\%\p{Final_Punctuation}]*)(\.+)$', re.U),
            7: re.compile(r'[\p{Uppercase_Letter}\p{Lowercase_Letter}\p{Other_Letter}\-]+', re.U),
            8: re.compile(r'(\.)[\p{Uppercase_Letter}\p{Other_Letter}\-]+(\.+)$', re.U),
            9: re.compile(r'^([ ]*[\'"([\u00bf\u00A1\p{Initial_Punctuation}]*[ ]*[\p{Uppercase_Letter}'
                          r'\p{Other_Letter}0-9])',
                          re.U),
            10: re.compile('^[0-9]+', re.U),
            11: re.compile(r' +'),
            12: re.compile(r'\n '),
            13: re.compile(r' \n ')
        }

    def tokenize(self, text: Optional[str], mode: Optional[str] = None) -> Tuple:
        """Sentence tokenize

        Parameters
        ----------
        text: str
            Text to be split into individual sentences
        mode: None, str
            if given as table, sentence tokenizer is performed only splitting text using enters (\n)
            other wise considers Turkish abbreviations and grammar rules

        Returns
        -------

        """

        words: list
        i: int
        w: list
        forward_word: str

        if text is None:
            return ()

        if not text:
            return ()

        if mode not in {'table', 'only_index'}:

            # Add sentence breaks as needed:

            # Non-period end of sentence markers (?!) followed by sentence starters
            text = self.pre_compiled_regexes[1].sub(repl='\\1\n\\2', string=text)

            # Multi-dots followed by sentence starters
            text = self.pre_compiled_regexes[2].sub(repl='\\1\n\\2', string=text)

            # Add breaks for sentences that end with some sort of punctuation inside a quote or parenthetical and are
            # followed by a possible sentence starter punctuation and upper case
            text = self.pre_compiled_regexes[3].sub(repl='\\1\n\\2', string=text)

            # Add breaks for sentences that end with some sort of punctuation are followed by a
            # sentence starter punctuation and upper case
            text = self.pre_compiled_regexes[4].sub(repl='\\1\n\\2', string=text)

            # Special punctuation cases are covered. Check all remaining periods
            words = self.pre_compiled_regexes[5].split(string=text)

            text = ''
            for i in range(0, len(words) - 1):
                match = self.pre_compiled_regexes[6].search(string=words[i])

                if match:
                    w = words[i].split('.')
                    if self.pre_compiled_regexes[7].search(string=w[-1]):
                        forward_word = w[-1]
                    else:
                        forward_word = words[i + 1]
                    prefix = match.group(1)
                    starting_punctuation = match.group(2)

                    def is_prefix_honorific(prefix_: str, starting_punct_: str) -> bool:
                        """Check if \\1 is a known honorific and \\2 is empty."""
                        if prefix_:
                            if prefix_ in self.__non_breaking_prefixes:
                                if self.__non_breaking_prefixes[prefix_] == SentenceTokenizer.PrefixType.DEFAULT:
                                    if not starting_punct_:
                                        return True
                        return False

                    if is_prefix_honorific(prefix_=prefix, starting_punct_=starting_punctuation):
                        # Not breaking
                        pass

                    elif self.pre_compiled_regexes[8].search(words[i]):
                        # Not breaking - upper case acronym
                        pass

                    elif self.pre_compiled_regexes[9].search(string=forward_word):
                        def is_numeric(prefix_: str, starting_punct_: str, next_word: str):
                            """The next word has a bunch of initial quotes, maybe a space, then either upper case or a
                            number."""
                            if prefix_:
                                if prefix_ in self.__non_breaking_prefixes:
                                    if self.__non_breaking_prefixes[prefix_] == \
                                            SentenceTokenizer.PrefixType.NUMERIC_ONLY:
                                        if not starting_punct_:
                                            if self.pre_compiled_regexes[10].search(string=next_word):
                                                return True
                            return False

                        if not is_numeric(prefix_=prefix, starting_punct_=starting_punctuation,
                                          next_word=forward_word):
                            if words[i + 1] == forward_word:
                                words[i] = ''.join([w[0], '.\n'])
                            else:
                                words[i] = ''.join(['.'.join(w[:-1]), '.\n', forward_word])
                        # We always add a return for these unless we have a numeric non-breaker and a number start
                text = ''.join([text, words[i], " "])

            # We stopped one token from the end to allow for easy look-ahead. Append it now.
            text = ''.join([text, words[-1]])
            text = self.pre_compiled_regexes[11].sub(repl=' ', string=text)
            text = self.pre_compiled_regexes[12].sub(repl='\n', string=text)
            text = self.pre_compiled_regexes[13].sub(repl='\n', string=text)
            text = text.strip()

        return tuple(text.split('\n'))

class WordTokenizer:
    __slots__ = 'pre_compiled_regexes'

    def __init__(self):
        suffixes: str
        numbers: str
        any_word: str
        punctuations: str

        suffixes = r"[a-zğçşöüı]{3,}' ?[a-zğçşöüı]+"
        numbers = r"%\d{2,}[.,:/\d-]+"
        any_word = r"[a-zğçşöüı_+%\.()@&`’/\\\d-]+"
        punctuations = r"[a-zğçşöüı]*[,!?;:]"
        self.pre_compiled_regexes = re.compile(
            "|".join(
                [suffixes,
                 numbers,
                 any_word,
                 punctuations
                 ]
            ), re.I
        )

    def tokenize(self, sentence: str) -> Tuple:
        words: Union[list, tuple]
        dots: str

        try:
            words = self.pre_compiled_regexes.findall(sentence)
        except (re.error, TypeError):
            return ()
        else:
            # If last word ends with dot, it should be another word
            words = tuple(words)
            if words:
                end_dots = re.search(r'\b(\.+)$', words[-1])
                if end_dots:
                    dots = end_dots.group(1)
                    words = words[:-1] + (words[-1][:-len(dots)],) + (dots,)
            return words