##################
#
# TESTS FOR THE FUNCTIONS THAT COMPUTE MOMENTS
# OF THE PROJECTED NORMAL DISTRIBUTION
#
##################

import pytest
import torch
import numpy as np
import qr_library as qr
from test_functions import *

# Parameters of simulation
nX = 5
nDim = 20
nSamples = 500000

####################################
####### ISOTROPIC NOISE CASE #######
#######    EXACT FORMULAS    #######
####################################

# Parameters of distribution
mu = make_mu_sin(nX=nX, nDim=nDim)
sigma = 0.5
covariance = make_covariance(nDim=nDim, sigmaScale=sigma, covType='isotropic')
B = torch.eye(nDim)

# Get theoretical estimates
meanT = qr.projected_normal_mean_iso(mu=mu, sigma=sigma)
smT = qr.projected_normal_sm_iso(mu=mu, sigma=sigma)
covT = qr.secondM_2_cov(secondM=smT, mean=meanT)

# Get empirical estimates
meanE, covE, smE = empirical_moments_projected_gaussian(mu, covariance,
                                                        nSamples=nSamples, B=B)

# Get errors
meanErr = torch.max(torch.abs(meanE - meanT)/(meanE+0.02))
covErr = torch.max(torch.abs(covE - covT)/(covE+0.02))
smErr = torch.max(torch.abs(smE - smT)/(smE+0.02))

# Plot covariances
maxN = 5
cols = np.min([nX, maxN])
for i in range(cols):
    maxVal = np.max([torch.max(covE[i, :, :]), torch.max(covT[i, :, :])])
    minVal = np.min([torch.min(covE[i, :, :]), torch.min(covT[i, :, :])])
    # Set min and max values to the color scale
    plt.subplot(4, cols, i+1)
    plt.imshow(covE[i, :, :], vmin=minVal, vmax=maxVal)
    plt.yticks(ticks=[])
    plt.xticks(ticks=[])
    if i == 0:
        plt.ylabel('Empirical')
    plt.subplot(4, cols, i+1+cols)
    plt.imshow(covT[i, :, :], vmin=minVal, vmax=maxVal)
    plt.yticks(ticks=[])
    plt.xticks(ticks=[])
    if i == 0:
        plt.ylabel('Theoretical')
    plt.subplot(4, cols, i+1+2*cols)
    plt.imshow(covE[i, :, :] - covT[i, :, :], vmin=minVal, vmax=maxVal)
    plt.yticks(ticks=[])
    plt.xticks(ticks=[])
    if i == 0:
        plt.ylabel('Difference')
    plt.subplot(4, cols, i+1+3*cols)
    plt.plot(meanE[i, :], label='Empirical')
    plt.plot(meanT[i, :], label='Theoretical')
    plt.legend()
    plt.yticks(ticks=[])
    plt.xticks(ticks=[])
plt.show()

####################################
####### INDEPENDENT NOISE CASE #####
#######  TAYLOR APPROXIMATION ######
####################################

nX = 1
nDim = 10
nSamples = 1000000

# Parameters of distribution
meanType = 'sin'
covType = 'diagonal'
sigma = 1
muMult = 4
approxMethod = 'taylor'

# Instantiate parameters
if meanType == 'sin':
    mu = make_mu_sin(nX=nX, nDim=nDim) * muMult
    mu = torch.abs(mu)
elif meanType == 'ones':
    mu = torch.ones((nX,nDim)) * muMult
elif meanType == 'sparse':
    mu = torch.rand((nX,nDim)) + 0.1
    # Select random indices
    ind = np.random.randint(0, nDim, size=5)
    # Set values to 2
    mu[:,ind] = 2
    mu = mu * muMult

if covType != 'poisson':
    print('not pois')
    covariance = make_covariance(nDim=nDim, sigmaScale=sigma, covType=covType)
    variances = torch.diag(covariance)
elif covType == 'poisson':
    print('pois')
    variances = mu.clone().squeeze() * sigma**2
    variances = torch.abs(variances)
    covariance = torch.diag(variances)
B = torch.eye(nDim)

# Get empirical moments
meanE, covE, smE = empirical_moments_projected_gaussian(mu, covariance,
                                                        nSamples=nSamples, B=B)

# Get theoretical moments
if approxMethod == 'taylor':
    meanT = qr.projected_normal_mean_taylor(mu=mu, variances=variances, order=2)
elif approxMethod == '0th':
    meanT = qr.projected_normal_mean_taylor(mu=mu, variances=variances, order=0)
elif approxMethod == 'empirical':
    meanT = meanE
smT = qr.projected_normal_sm(mu=mu, covariance=covariance, B=B)
covT = qr.secondM_2_cov(smT, meanT)

# Plot covariances
maxN = 5
cols = np.min([nX, maxN])
for i in range(cols):
    maxVal = np.max([torch.max(covE[i, :, :]), torch.max(covT[i, :, :])])
    minVal = np.min([torch.min(covE[i, :, :]), torch.min(covT[i, :, :])])
    # Set min and max values to the color scale
    plt.subplot(4, cols, i+1)
    plt.imshow(covE[i, :, :], vmin=minVal, vmax=maxVal)
    plt.yticks(ticks=[])
    plt.xticks(ticks=[])
    if i == 0:
        plt.ylabel('Empirical')
    plt.subplot(4, cols, i+1+cols)
    plt.imshow(covT[i, :, :], vmin=minVal, vmax=maxVal)
    plt.yticks(ticks=[])
    plt.xticks(ticks=[])
    if i == 0:
        plt.ylabel('Theoretical')
    plt.subplot(4, cols, i+1+2*cols)
    plt.imshow(covE[i, :, :] - covT[i, :, :], vmin=minVal, vmax=maxVal)
    plt.yticks(ticks=[])
    plt.xticks(ticks=[])
    if i == 0:
        plt.ylabel('Difference')
    plt.subplot(4, cols, i+1+3*cols)
    plt.plot(meanE[i, :], label='Empirical')
    plt.plot(meanT[i, :], label='Theoretical')
    plt.legend()
    plt.yticks(ticks=[])
    plt.xticks(ticks=[])
plt.show()

# Plot second moments
#maxN = 5
#cols = np.min([nX, maxN])
#for i in range(cols):
#    maxVal = np.max([torch.max(smE[i, :, :]), torch.max(smT[i, :, :])])
#    minVal = np.min([torch.min(smE[i, :, :]), torch.min(smT[i, :, :])])
#    # Set min and max values to the color scale
#    plt.subplot(4, cols, i+1)
#    plt.imshow(smE[i, :, :], vmin=minVal, vmax=maxVal)
#    plt.yticks(ticks=[])
#    plt.xticks(ticks=[])
#    if i == 0:
#        plt.ylabel('Empirical')
#    plt.subplot(4, cols, i+1+cols)
#    plt.imshow(smT[i, :, :], vmin=minVal, vmax=maxVal)
#    plt.yticks(ticks=[])
#    plt.xticks(ticks=[])
#    if i == 0:
#        plt.ylabel('Theoretical')
#    plt.subplot(4, cols, i+1+2*cols)
#    plt.imshow(smE[i, :, :] - smT[i, :, :], vmin=minVal, vmax=maxVal)
#    plt.yticks(ticks=[])
#    plt.xticks(ticks=[])
#    if i == 0:
#        plt.ylabel('Difference')
#    plt.subplot(4, cols, i+1+3*cols)
#    plt.plot(meanE[i, :], label='Empirical')
#    plt.plot(meanT[i, :], label='Theoretical')
#    plt.legend()
#    plt.yticks(ticks=[])
#    plt.xticks(ticks=[])
#plt.show()

####################################
#######  TAYLOR APPROXIMATION ######
#######     MATHEMATICS       ######
####################################

## TEST THAT THE METHOD FOR COMPUTING MOMENTS WORKS
meanX2 = qr.quadratic_form_mean(mu=mu, covariance=covariance, M=B)
meanXi2 = mu**2 + variances
meanV = meanX2 - meanXi2
varX2 = qr.quadratic_form_var(mu=mu, covariance=covariance, M=B)
varSubtract = 2 * variances**2 + 4 * mu**2 * variances
varV = varX2 - varSubtract

meanV2 = torch.zeros(nDim)
varV2 = torch.zeros(nDim)
for i in range(nDim):
    muRemainder = torch.cat((mu[0,:i], mu[0,i+1:]))
    muRemainder = muRemainder.unsqueeze(0)
    varRemainder = torch.cat((variances[:i], variances[i+1:]))
    covRemainder = torch.diag(varRemainder)
    B2 = torch.eye(nDim-1)
    meanV2[i] = qr.quadratic_form_mean(mu=muRemainder, covariance=covRemainder, M=B2)
    varV2[i] = qr.quadratic_form_var(mu=muRemainder, covariance=covRemainder, M=B2)

## TEST THAT THE METHOD FOR COMPUTING DERIVATIVES WORKS
def f(x, y):
    " f = x/sqrt(x^2+y) "
    return x/(torch.sqrt(x**2+y))

# Function to compute the second derivative of f with respect to x
# using the autograd package
def d2f_dx2_autograd(x, y):
    # First derivative with respect to x
    df_dx = torch.autograd.grad(f(x, y), x, create_graph=True)[0]
    # Second derivative with respect to x (the gradient of df_dx with respect to x)
    d2f_dx2 = torch.autograd.grad(df_dx, x)[0]
    return d2f_dx2

# Function to compute the second derivative of f with respect to y
# using the autograd package
def d2f_dy2_autograd(x, y):
    # First derivative with respect to y
    df_dy = torch.autograd.grad(f(x, y), y, create_graph=True)[0]
    # Second derivative with respect to y (the gradient of df_dy with respect to y)
    d2f_dy2 = torch.autograd.grad(df_dy, y)[0]
    return d2f_dy2

du2_autograd = torch.zeros(nDim)
dv2_autograd = torch.zeros(nDim)
for i in range(nDim):
    x = mu[0,i].clone().detach().requires_grad_(True)
    y = meanV[0,i].clone().detach().requires_grad_(True)
    # Compute the derivative of f with respect to x
    du2_autograd[i] = d2f_dx2_autograd(x, y)
    # Compute the derivative of f with respect to y
    dv2_autograd[i] = d2f_dy2_autograd(x, y)

# Function to compute the second derivative of f with respect to x
du2_fun = qr.proj_normal_dx2(u=mu, v=meanV)
dv2_fun = qr.proj_normal_dv2(u=mu, v=meanV)

print(f'du2_autograd = {du2_autograd.numpy()}')
print(f'du2_fun = {du2_fun.numpy()}')
print(f'dv2_autograd = {dv2_autograd.numpy()}')
print(f'dv2_fun = {dv2_fun.numpy()}')


import argparse

from pathlib import Path

import numpy as np
import pandas as pd
import os
import xgboost as xgb
from xgboost import XGBClassifier

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MaxAbsScaler
from sklearn.ensemble import AdaBoostClassifier

from sklearn.model_selection import RandomizedSearchCV
from sklearn.preprocessing import StandardScaler
from imblearn.over_sampling import RandomOverSampler
from sklearn.preprocessing import OneHotEncoder
from sklearn.ensemble import GradientBoostingClassifier
from keras.callbacks import EarlyStopping

from sklearn import metrics
from sklearn.metrics import classification_report
from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve
from collections import Counter
from sklearn.preprocessing import LabelEncoder

from keras.layers import Input, Embedding, Flatten, Dense, BatchNormalization, ReLU, Dropout, Concatenate
from keras.models import Model
from keras.models import Sequential
from keras.optimizers import Adam, Adadelta

import matplotlib.pyplot as plt
from collections import Counter
import lightgbm as lgbm
from sklearn.metrics import f1_score, accuracy_score, precision_score, recall_score, roc_auc_score, ConfusionMatrixDisplay, confusion_matrix
import mlflow
import mlflow.sklearn

def parse_args():
    '''Parse input arguments'''

    parser = argparse.ArgumentParser()
    parser.add_argument("--input_data", type=str, help="Path to data")
    parser.add_argument("--model_output", type=str, help="Path to model")

    # for lighgbm
    parser.add_argument('--num_leaves', type=int, default = 100,
                        help='Maximum number of leaves in one tree, should not be high, defaul is 31')

    parser.add_argument('--objective', type=str, default = 'binary',
                        help='loss function')

    parser.add_argument('--num_boost_round', type=int, default=100,
                        help='number of boosting iterations')

    parser.add_argument('--learning_rate_lgbm', type=float, default=0.1,
                        help='learning_rate used to do the new prediction')

    parser.add_argument('--max_depth_lgbm', type=int, default = 100,
                        help='max_depth')

    parser.add_argument('--min_data_in_leaf', type=int, default=50,
                        help='Minimum number of samples required in a terminal leaf')

    parser.add_argument('--min_gain_to_split', type=float, default = 0.0001,
                        help='random_state')

    parser.add_argument('--max_bin', type=int, default = 255,
                        help='small number of bins may reduce training accuracy but may increase general power ')

    parser.add_argument('--categorical_feature', type=str, default =['origine', 'destination','voiture', 'duplex', 'place', 'classe_voiture', 'lib_article',
    'lib_canal_dist','type_point_vente', 'lib_canal_av', 'lib_motif_av','lib_nature_av',
    'cd_poste_av', 'lib_transporteur', 'list_type_paiement','date_prestation_bin', 'date_naissance_bin'],
                        help='specify categorical features')

    parser.add_argument('--num_class', type=int, default = 2,
                        help='random_state')

    #parser.add_argument('--early_stopping_round', type=int, default = 50,
                       # help='early_stopping')

    parser.add_argument('--data_sample_strategy', type=str, default = 'goss',
                        help='lightgbm samples data after each weak learner, either bagging (random sampling with replacement, either goss, wich is one of the most features of lightgbm that makes it powerful, by default implementation of lightgbm, it is bagging, that is why i have to specify that i want goss method !')

    parser.add_argument('--lambda_l2', type=float, default = 0.1,
                        help='l2 regularisation')

    parser.add_argument('--top_rate', type=float, default = 0.8,
                        help='the retain ratio of large gradient data')

    parser.add_argument('--other_rate', type=float, default = 0.2,
                        help='the retain ratio of small gradient data')

    # for Gradient Boosting
    parser.add_argument('--n_estimators_gb', type=int, default = 1,
                        help='Number of trees')
    parser.add_argument('--learning_rate_gb', type=float, default = 0.01,
                        help='learning_rate')
    parser.add_argument('--max_depth_gb', type=int, default=10,
                        help=' Maximum number of levels in tree')
    parser.add_argument('--max_features', type=float, default=1.0,
                        help='Number of features to consider at every split')
    parser.add_argument('--subsample_gb', type=float, default=0.7,
                        help='Number of observations taken by bootstrap to every estimator')
    parser.add_argument('--min_samples_leaf', type=int, default = 300,
                        help='Minimum number of samples required at each leaf node')
    parser.add_argument('--min_samples_split', type=int, default=300,
                        help='Minimum number of samples required to split a node')
    parser.add_argument('--min_impurity_decrease', type=float, default = 0.01,
                        help='Minimum of impurity decrease')
    parser.add_argument('--random_state', type=int, default = 42,
                        help='random_state')
    parser.add_argument('--ccp_alpha', type=float, default = 0.5,
                        help='ccp_alpha')

    # for xgboost
    parser.add_argument("--n_estimators_xgb", type=int,  default = 10, help="nombre de weak learners")
    parser.add_argument('--eta', type=float, default = 0.3,
                        help='learning_rate')
    parser.add_argument('--gamma', type=int, default = 0,
                        help='valeur à enlever du Gain')
    parser.add_argument('--max_depth_xgb', type=int, default=6,
                        help=' Maximum number of levels in tree')
    parser.add_argument('--min_child_weight', type=int, default=1,
                        help='Cover to be satisfied by the residuals in the terminal leaf') ## 1 for classification doen't mean 1 residual in the leaf !
    parser.add_argument('--subsample_xgb', type=float, default=1,
                        help='Number of observations taken by each weak learner')
    parser.add_argument('--reg_lambda', type=int, default = 1,
                        help='l2 Regularisation hyperparameter')
    parser.add_argument('--alpha', type=int, default=0,
                        help='l1 Regularisation hyperparameter')
    parser.add_argument('--num_parallel_tree', type=int, default = 1,
                        help='Nombres des arbres pour chaque weak learner, when >1, each weak leaner is like a random forest, when 1, it is like a decision tree')
    parser.add_argument('--tree_method', type=str, default = 'hist',
                        help='random_state')
    parser.add_argument('--importance_type', type=str, default = 'gain',
                        help='feature_importance')
    parser.add_argument('--enable_categorical', type=bool, default = True,
                        help='handling categorical variables')

    # for adaboost :
    #parser.add_argument("--base_estimator", type=object, default = DecisionTreeClassifier())

    #parser.add_argument('--n_estimators', type=int, default = 10000,
                        #help='Number of weak learners')

    # for neural network
    parser.add_argument('--learning_rate_MLP', type=float, default=0.10, help='percentage of dropout')
    parser.add_argument('--activation', type=str, default='relu', help='activation function')
    parser.add_argument('--optimizer', type=str, default='Adam', help='optimizer')
    parser.add_argument('--loss', type=str, default='binary_crossentropy', help='loss')
    parser.add_argument('--rate', type=float, default=0.1, help='Dropout layer')
    parser.add_argument('--batch_size', type=int, default=64, help='batch_size')
    parser.add_argument('--epochs', type=int, default=5, help='epochs')
    args = parser.parse_args()

    return args

def main(args):
    '''Read train dataset, train model, save trained model'''

    mlflow.sklearn.autolog()
    mlflow.log_param("lightgbm", "Lightgbm")
    mlflow.log_param('num_leaves', int(args.num_leaves))
    mlflow.log_param('objective', str(args.objective))
    mlflow.log_param('num_boost_round', str(args.num_boost_round))
    mlflow.log_param('learning_rate_lgbm', float(args.learning_rate_lgbm))
    #mlflow.log_param('num_threads', int(args.num_threads))
    mlflow.log_param('max_depth_lgbm', int(args.max_depth_lgbm))
    mlflow.log_param('min_data_in_leaf', int(args.min_data_in_leaf))
    mlflow.log_param('min_gain_to_split', float(args.min_gain_to_split))
    mlflow.log_param('max_bin', int(args.max_bin))
    mlflow.log_param('categorical_feature', str(args.categorical_feature))
    mlflow.log_param('num_class', int(args.num_class))
    #mlflow.log_param('early_stopping_round', int(args.early_stopping_round))
    mlflow.log_param('data_sample_strategy', str(args.data_sample_strategy))
    mlflow.log_param('lambda_l2', float(args.lambda_l2))
    mlflow.log_param('top_rate', float(args.top_rate))
    mlflow.log_param('other_rate', float(args.other_rate))

    # adaboost
    #mlflow.log_param("adaboost_model", "AdaBoostClassifier")
    #mlflow.log_param('n_estimators', int(args.n_estimators))

    # NN
    mlflow.log_param("MLP_model", "MLP")
    mlflow.log_param('learning_rate_MLP', float(args.learning_rate_MLP))
    mlflow.log_param('rate', float(args.rate))
    mlflow.log_param('activation', str(args.activation))
    mlflow.log_param('optimizer', str(args.activation))
    mlflow.log_param('loss', str(args.loss))
    mlflow.log_param('epochs', int(args.epochs))

    # GRADIENT BOOSITNG
    mlflow.log_param('n_estimators_gb', int(args.n_estimators_gb))
    mlflow.log_param('max_depth_gb', int(args.max_depth_gb))
    mlflow.log_param('max_features', float(args.max_features))
    mlflow.log_param('subsample_gb', float(args.subsample_gb))
    mlflow.log_param('min_samples_leaf', int(args.min_samples_leaf))
    mlflow.log_param('min_samples_split', int(args.min_samples_split))
    mlflow.log_param('min_impurity_decrease', float(args.min_impurity_decrease))
    mlflow.log_param('random_state', int(args.random_state))
    mlflow.log_param('ccp_alpha', float(args.ccp_alpha))
    mlflow.log_param('learning_rate_gb', float(args.learning_rate_gb))

    # XGBOOST
    mlflow.log_param('eta', float(args.eta))
    mlflow.log_param('n_estimators_xgb', float(args.n_estimators_xgb))
    mlflow.log_param('gamma', int(args.gamma))
    mlflow.log_param('max_depth_xgb', int(args.max_depth_xgb))
    mlflow.log_param('min_child_weight', int(args.min_child_weight))
    mlflow.log_param('subsample_xgb', float(args.subsample_xgb))
    mlflow.log_param('reg_lambda', int(args.reg_lambda))
    mlflow.log_param('alpha', int(args.alpha))
    mlflow.log_param('num_parallel_tree', int(args.num_parallel_tree))
    mlflow.log_param('importance_type', str(args.importance_type))
    mlflow.log_param('tree_method', str(args.tree_method))
    #mlflow.log_param('nthread', float(args.nthread))
    mlflow.log_param('enable_categorical', bool(args.enable_categorical))

    print("reading data")
    df = pd.read_csv(Path(args.input_data) / "data.csv", sep=";")
    Numeric_Var = ["List of numerical variables in your data frame"]
    Cat_Var = ["List of categorical variables in your data frame"]

    print("encoding variables")
    label_encoder = LabelEncoder()
    for column in Cat_Var:
        df[column] = label_encoder.fit_transform(df[column])

    df[Cat_Var] = df[Cat_Var].astype('category')

    print("normalisation des variables continues")
    #scaler = StandardScaler()
    scaler = MaxAbsScaler()
    df[Numeric_Var] = scaler.fit_transform(df[Numeric_Var])

    print("splitting")
    X = df.drop("fraud", axis=1)
    Y = df["fraud"]
    X_train, X_test, Y_train, Y_test = train_test_split(X,Y,test_size=0.2,random_state=15,shuffle=True,stratify=Y)

    X_train, X_val, Y_train, Y_val = train_test_split(X_train,Y_train,test_size=0.1,random_state=15,shuffle=True,stratify=Y_train)
    print("Y_train:\n",  Y_train.value_counts(),
    "\nY_test:\n", Y_test.value_counts()
    ,"\nY_valid:\n", Y_val.value_counts())

    print("SAMPLING")
    sampling_strategy = {0: 12044636, 1:4687331}
    ros = RandomOverSampler(random_state=0, sampling_strategy=sampling_strategy)
    X_train_resampled, Y_train_resampled = ros.fit_resample(X_train, Y_train )

    print("Class distribution after under-sampling:")
    print("X_train_resampled shape:", X_train_resampled.shape)
    print("Y_train_resampled shape:", Y_train_resampled.shape)
    print("Y_train_Count:", Y_train_resampled.value_counts())
    print(sorted(Counter(Y_train_resampled).items()))

    # defining models
    categorical_indices = [X_train.columns.get_loc(cat_feature) for cat_feature in Cat_Var]
    lgbm_model = lgbm.LGBMClassifier(num_leaves= args.num_leaves,
                                    objective = args.objective,
                                    max_depth = args.max_depth_lgbm,
                                    num_boost_round  = args.num_boost_round ,
                                    learning_rate = args.learning_rate_lgbm,
                                    #num_threads = args.num_threads,
                                    min_data_in_leaf = args.min_data_in_leaf,
                                    max_bin = args.max_bin,
                                    categorical_feature = categorical_indices,
                                    #early_stopping_round = args.early_stopping_round,
                                    data_sample_strategy = args.data_sample_strategy,
                                    lambda_l2 = args.lambda_l2,
                                    top_rate = args.top_rate,
                                    other_rate = args.other_rate,
                                    )

    gradient_boosting_model = GradientBoostingClassifier(n_estimators= args.n_estimators_gb,
                                    learning_rate = args.learning_rate_gb,
                                    max_depth = args.max_depth_gb,
                                    max_features = args.max_features,
                                    subsample = args.subsample_gb,
                                    min_samples_leaf = args.min_samples_leaf,
                                    min_samples_split = args.min_samples_split,
                                    ccp_alpha = args.ccp_alpha,
                                    min_impurity_decrease = args.min_impurity_decrease,
                                    random_state = args.random_state)

    xgboost_model = xgb.XGBClassifier(eta= args.eta,
                                    gamma = args.gamma,
                                    max_depth = args.max_depth_xgb,
                                    min_child_weight = args.min_child_weight,
                                    subsample = args.subsample_xgb,
                                    reg_lambda = args.reg_lambda,
                                    num_parallel_tree = args.num_parallel_tree,
                                    alpha = args.alpha,
                                    tree_method = args.tree_method,
                                    n_estimators = args.n_estimators_xgb,
                                    importance_type = args.importance_type,
                                    #nthread = args.nthread,
                                    enable_categorical = args.enable_categorical)

    #adaboost_model = AdaBoostClassifier(n_estimators= args.n_estimators,
                                                #base_estimator = base_classifier,
                                                #learning_rate = args.learning_rate,)

    print("training models on the sampled data")
    lgbm_model = lgbm_model.fit(X_train_resampled, Y_train_resampled)
    xgboost_model = xgboost_model.fit(X_train_resampled, Y_train_resampled)
    gradient_boosting_model = gradient_boosting_model.fit(X_train_resampled, Y_train_resampled)

    print("predicting values on validation data")
    y_lgbm = lgbm_model.predict(X_val)
    y_xgboost = xgboost_model.predict(X_val)
    y_gradientboosting  = gradient_boosting_model.predict(X_val)

    print("training the meta model")
    X_val_meta = np.column_stack((y_lgbm, y_xgboost, y_gradientboosting))

    model_MLP = Sequential()
    model_MLP.add(Dense(256,input_shape=(3,))) # input shape doit etre egal au nombre de base models
    model_MLP.add(BatchNormalization())
    model_MLP.add(Dropout(rate=args.rate))
    model_MLP.add(ReLU())
    model_MLP.add(Dense(128))
    model_MLP.add(BatchNormalization())
    model_MLP.add(Dropout(rate=args.rate))
    model_MLP.add(ReLU())
    model_MLP.add(Dense(64))
    model_MLP.add(BatchNormalization())
    model_MLP.add(Dropout(rate=args.rate))
    model_MLP.add(ReLU())
    model_MLP.add(Dense(32))
    model_MLP.add(BatchNormalization())
    model_MLP.add(Dropout(rate=args.rate))
    model_MLP.add(ReLU())
    model_MLP.add(Dense(1, activation='sigmoid'))
    model_MLP.summary()
    #opt = Adam(lr=args.lr)
    model_MLP.compile(optimizer=Adadelta(learning_rate=args.learning_rate_MLP), loss=args.loss, metrics=['accuracy'])
    es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience = 3)

    hist_MLP = model_MLP.fit(X_val_meta, Y_val, batch_size=args.batch_size,
              epochs=args.epochs,
              shuffle=True,
              validation_split=0.1,
              callbacks = [es])

    print("predicting on new data wich is X_test")
    y_lgbm_test = lgbm_model.predict(X_test)
    y_gradientboosting_test = gradient_boosting_model.predict(X_test)
    y_xgboost_test = xgboost_model.predict(X_test)

    print("Combining the predictions of the base models into a single feature matrix")
    X_new_meta = np.column_stack((y_lgbm_test, y_gradientboosting_test,y_xgboost_test))

    print("prediction using the meta-model on X_meta_test")
    y_test_pred = model_MLP.predict(X_new_meta)

    y_test_pred_labels = (y_test_pred > 0.5).astype(int)

    # Visualize results
    cm = confusion_matrix(Y_test, y_test_pred_labels)
    cm_display = ConfusionMatrixDisplay(cm).plot()
    cm_display.figure_.savefig('confusion_matrix-MLP.png')
    mlflow.log_artifact("confusion_matrix-MLP.png")

    accuracyscore = metrics.accuracy_score(Y_test,y_test_pred_labels)*100
    f1score = metrics.f1_score(Y_test,y_test_pred_labels)*100
    recallscore = metrics.recall_score(Y_test,y_test_pred_labels)*100
    specificityscore = metrics.recall_score(Y_test, y_test_pred_labels, pos_label=0)*100
    precisionscore = metrics.precision_score(Y_test,y_test_pred_labels)*100

    mlflow.log_metric("accuracy_score", accuracyscore)
    mlflow.log_metric("f1_score", f1score)
    mlflow.log_metric("precision_score", precisionscore)
    mlflow.log_metric("recall_score", recallscore)
    mlflow.log_metric("specificity_score", specificityscore)
    print ("Accuracy=", accuracyscore)
    print ("f1.score=", f1score)
    print ("precision.score=", precisionscore)
    print ("recall_score=", recallscore)
    print("Specificity=", specificityscore)

    ######## COURBE DE ROC
    fpr_d, tpr_d, threshold = metrics.roc_curve(Y_test, y_test_pred)
    roc_auc = metrics.auc(fpr_d, tpr_d)
    print("AUC MLP=", roc_auc)
    auc_d = roc_auc_score(Y_test, y_test_pred)
    plt.title('Courbe de ROC - MLP')
    plt.plot(fpr_d, tpr_d, 'b', label = 'AUC = %0.2f' % auc_d)
    plt.legend(loc = 'lower right')
    plt.plot([0, 1], [0, 1],'r--')
    plt.xlim([0, 1])
    plt.ylim([0, 1])
    plt.ylabel('True Positive Rate - Sensitivity')
    plt.xlabel('False Positive Rate - (1-Specificity)')
    plt.savefig('roc_curve.png')
    mlflow.log_artifact("roc_curve.png")

    hist_MLP.history.keys()
    plt.figure(2)
    plt.plot(hist_MLP.history['loss'], label='training set',marker='o', linestyle='solid',linewidth=1, markersize=6)
    plt.plot(hist_MLP.history['val_loss'], label='validation set',marker='o', linestyle='solid',linewidth=1, markersize=6)
    plt.title("MLP-model loss")
    plt.xlabel('Nombre d\'epochs')
    plt.ylabel('Total Loss')
    plt.legend(bbox_to_anchor=( 1.35, 1.))
    plt.savefig('Loss.png')
    mlflow.log_artifact("Loss.png")

    # Visualize results
    cm = confusion_matrix(Y_test, y_lgbm_test)
    plt.figure(3)
    cm_display = ConfusionMatrixDisplay(cm).plot()
    cm_display.figure_.savefig('confusion_matrix-lgbm.png')
    mlflow.log_artifact("confusion_matrix-lgbm.png")

    cm = confusion_matrix(Y_test, y_gradientboosting_test)
    plt.figure(4)
    cm_display = ConfusionMatrixDisplay(cm).plot()
    cm_display.figure_.savefig('confusion_matrix-gradientboosting.png')
    mlflow.log_artifact("confusion_matrix-gradientboosting.png")

    cm = confusion_matrix(Y_test, y_xgboost_test)
    plt.figure(5)
    cm_display = ConfusionMatrixDisplay(cm).plot()
    cm_display.figure_.savefig('confusion_matrix-xgboost.png')
    mlflow.log_artifact("confusion_matrix-xgboost.png")

    plt.figure(figsize=(8, 6))
    plt.figure(6)
    plt.plot(hist_MLP.history['loss'], label='Train')
    plt.plot(hist_MLP.history['val_loss'], label='Validation')
    plt.xlabel('Époque')
    plt.ylabel('Perte')
    plt.legend()
    plt.title('Loss_Curve')
    plt.savefig('loss_curve.png')
    mlflow.log_artifact("loss_curve.png")

if __name__ == "__main__":

    mlflow.start_run()

    # ---------- Parse Arguments ----------- #
    # -------------------------------------- #

    args = parse_args()
    lines = [
        f"input_data: {args.input_data}",
        f"model_output: {args.model_output}"
        f"num_leaves: {args.num_leaves}"
        f"objective: {args.objective}"
        f"max_depth_lgbm: {args.max_depth_lgbm}"
        f"num_boost_round : {args.num_boost_round }"
        f"learning_rate_lgbm: {args.learning_rate_lgbm}"
        #f"num_threads: {args.num_threads}"
        f"min_data_in_leaf: {args.min_data_in_leaf}"
        f"max_bin: {args.max_bin}"
        f"categorical_feature: {args.categorical_feature}"
        f"num_class: {args.num_class}"
        #f"early_stopping_round: {args.early_stopping_round}"
        f"data_sample_strategy: {args.data_sample_strategy}"
        f"lambda_l2: {args.lambda_l2}"
        f"top_rate: {args.top_rate}"
        f"other_rate: {args.other_rate}"
        #f"base_estimator: {args.base_estimator}"

        # MLP
        f"learning_rate_MLP: {args.learning_rate_MLP}"
        f"rate: {args.rate}"
        f"activation: {args.activation}"
        f"optimizer: {args.optimizer}"
        f"loss: {args.loss}"

        # GRADIENT BOOSTING
        f"n_estimators_gb: {args.n_estimators_gb}"
        f"learning_rate_gb: {args.learning_rate_gb}"
        f"max_features: {args.max_features}"
        f"max_depth_gb: {args.max_depth_gb}"
        f"subsample_gb: {args.subsample_gb}"
        f"min_samples_leaf: {args.min_samples_leaf}"
        f"min_samples_split: {args.min_samples_split}"
        f"random_state: {args.random_state}"
        f"ccp_alpha: {args.ccp_alpha}"
        f"min_impurity_decrease: {args.min_impurity_decrease}"

        # XGBOOST
        f"enable_categorical: {args.enable_categorical}"
        f"n_estimators_xgb: {args.n_estimators_xgb}"
        f"eta: {args.eta}"
        f"gamma: {args.gamma}"
        f"max_depth_xgb: {args.max_depth_xgb}"
        f"min_child_weight: {args.min_child_weight}"
        f"subsample_xgb: {args.subsample_xgb}"
        f"max_depth_xgb: {args.max_depth_xgb}"
        f"subsample_xgb: {args.subsample_xgb}"
        f"reg_lambda: {args.reg_lambda}"
        f"num_parallel_tree: {args.num_parallel_tree}"
        f"alpha: {args.alpha}"
        f"tree_method: {args.tree_method}"
        #f"nthread: {args.nthread}"
        f"importance_type: {args.importance_type}"
    ]

    for line in lines:
        print(line)

    main(args)

    mlflow.end_run()


import logging

# A nosec comment is appended to the following line in order to disable the B404 check.
# In this file the input of the module subprocess is trusted.
import subprocess  # nosec B404
from datetime import datetime
from typing import List, Union

from common.schedulers.slurm_commands import DEFAULT_SCONTROL_COMMAND_TIMEOUT, SCONTROL
from common.utils import (
    SlurmCommandError,
    SlurmCommandErrorHandler,
    check_command_output,
    run_command,
    validate_subprocess_argument,
)
from retrying import retry
from slurm_plugin.slurm_resources import SlurmReservation

logger = logging.getLogger(__name__)

SCONTROL_SHOW_RESERVATION_OUTPUT_AWK_PARSER = (
    'awk \'BEGIN{{RS="\\n\\n" ; ORS="######\\n";}} {{print}}\' | '
    + "grep -oP '^(ReservationName=\\S+)|(?<!Next)(State=\\S+)|(Users=\\S+)|(Nodes=\\S+)|(######)'"
)

def _create_or_update_reservation(
    base_command: str,
    name: str,
    nodes: str = None,
    partition: str = None,
    user: str = None,
    start_time: Union[datetime, str] = None,  # 'now' is accepted as a value
    duration: Union[int, str] = None,  # 'infinite' is accepted as a value
    number_of_nodes: int = None,
    flags: str = None,
    command_timeout=DEFAULT_SCONTROL_COMMAND_TIMEOUT,
    raise_on_error=True,
):
    """
    Create or update slurm reservation, adding all the parameters.

    Official documentation is https://slurm.schedmd.com/reservations.html
    """
    cmd = _add_param(base_command, "ReservationName", name)
    cmd = _add_param(cmd, "nodes", nodes)
    cmd = _add_param(cmd, "partition", partition)
    cmd = _add_param(cmd, "user", user)
    if isinstance(start_time, datetime):
        # Convert start time to format accepted by slurm command
        cmd = _add_param(cmd, "starttime", start_time.strftime("%Y-%m-%dT%H:%M:%S"))
    elif start_time:
        cmd = _add_param(cmd, "starttime", start_time)
    cmd = _add_param(cmd, "duration", duration)
    cmd = _add_param(cmd, "nodecnt", number_of_nodes)
    cmd = _add_param(cmd, "flags", flags)

    run_command(cmd, raise_on_error=raise_on_error, timeout=command_timeout, shell=True)  # nosec B604

@retry(
    stop_max_attempt_number=2,
    wait_fixed=1000,
    retry_on_exception=lambda exception: isinstance(exception, SlurmCommandError),
)
@SlurmCommandErrorHandler.handle_slurm_command_error
def create_slurm_reservation(
    name: str,
    nodes: str = "ALL",
    partition: str = None,
    user: str = "slurm",
    start_time: Union[datetime, str] = None,  # 'now' is accepted as a value
    duration: Union[int, str] = None,  # 'infinite' is accepted as a value
    number_of_nodes: int = None,
    flags: str = "maint",
    command_timeout: int = DEFAULT_SCONTROL_COMMAND_TIMEOUT,
    raise_on_error: bool = True,
):
    """
    Create slurm reservation with scontrol call.

    The command to create a reservation is something like the following:
    scontrol create reservation starttime=2009-02-06T16:00:00 duration=120 user=slurm flags=maint nodes=ALL
    scontrol create reservation user=root partition=queue1 starttime=noon duration=60 nodecnt=10

    We're using slurm as default user because it is the default Slurm administrator user in ParallelCluster,
    this is the user running slurmctld daemon.
    "maint" flag permits to overlap existing reservations.
    Official documentation is https://slurm.schedmd.com/reservations.html
    """
    cmd = f"{SCONTROL} create reservation"

    logger.debug("Creating Slurm reservation with command: %s", cmd)
    _create_or_update_reservation(
        cmd, name, nodes, partition, user, start_time, duration, number_of_nodes, flags, command_timeout, raise_on_error
    )

@retry(
    stop_max_attempt_number=2,
    wait_fixed=1000,
    retry_on_exception=lambda exception: isinstance(exception, SlurmCommandError),
)
@SlurmCommandErrorHandler.handle_slurm_command_error
def update_slurm_reservation(
    name: str,
    nodes: str = None,
    partition: str = None,
    user: str = None,
    start_time: Union[datetime, str] = None,  # 'now' is accepted as a value
    duration: Union[int, str] = None,  # 'infinite' is accepted as a value
    number_of_nodes: int = None,
    flags: str = None,
    command_timeout: int = DEFAULT_SCONTROL_COMMAND_TIMEOUT,
    raise_on_error: bool = True,
):
    """
    Update slurm reservation with scontrol call.

    The command to update a reservation is something like the following:
    scontrol update ReservationName=root_3 duration=150 users=admin

    Official documentation is https://slurm.schedmd.com/reservations.html
    """
    cmd = f"{SCONTROL} update"

    logger.debug("Updating Slurm reservation with command: %s", cmd)
    _create_or_update_reservation(
        cmd, name, nodes, partition, user, start_time, duration, number_of_nodes, flags, command_timeout, raise_on_error
    )

@retry(
    stop_max_attempt_number=2,
    wait_fixed=1000,
    retry_on_exception=lambda exception: isinstance(exception, SlurmCommandError),
)
@SlurmCommandErrorHandler.handle_slurm_command_error
def delete_slurm_reservation(
    name: str,
    command_timeout: int = DEFAULT_SCONTROL_COMMAND_TIMEOUT,
    raise_on_error: bool = True,
):
    """
    Delete slurm reservation with scontrol call.

    The command to delete a reservation is something like the following:
    scontrol delete ReservationName=root_6

    Official documentation is https://slurm.schedmd.com/reservations.html
    """
    cmd = f"{SCONTROL} delete"
    cmd = _add_param(cmd, "ReservationName", name)

    logger.debug("Deleting Slurm reservation with command: %s", cmd)
    run_command(cmd, raise_on_error=raise_on_error, timeout=command_timeout, shell=True)  # nosec B604

def _add_param(cmd, param_name, value):
    """If the given value is not None, validate it and concatenate ' param_name=value' to cmd."""
    if value:
        validate_subprocess_argument(value)
        cmd += f" {param_name}={value}"
    return cmd

@retry(
    stop_max_attempt_number=2,
    wait_fixed=1000,
    retry_on_exception=lambda exception: isinstance(exception, SlurmCommandError),
)
@SlurmCommandErrorHandler.handle_slurm_command_error
def is_slurm_reservation(
    name: str,
    command_timeout: int = DEFAULT_SCONTROL_COMMAND_TIMEOUT,
    raise_on_error: bool = True,
):
    """
    Check if slurm reservation exists, by retrieving information with scontrol call.

    Return True if reservation exists, False otherwise.
    Raise a CalledProcessError if the command fails for other reasons.

    $ scontrol show ReservationName=root_5
    Reservation root_5 not found
    $ echo $?
    1

    $ scontrol show ReservationName=root_6
    ReservationName=root_6 StartTime=2023-10-13T15:57:03 EndTime=2024-10-12T15:57:03 Duration=365-00:00:00
    Nodes=q1-st-cr2-1,q2-dy-cr4-[1-5] NodeCnt=6 CoreCnt=481 Features=(null) PartitionName=(null) Flags=MAINT,SPEC_NODES
    TRES=cpu=481
    Users=root Groups=(null) Accounts=(null) Licenses=(null) State=ACTIVE BurstBuffer=(null) Watts=n/a
    MaxStartDelay=(null)
    $ echo $?
    0

    Official documentation is https://slurm.schedmd.com/reservations.html
    """
    cmd = f"{SCONTROL} show"
    cmd = _add_param(cmd, "ReservationName", name)

    try:
        logger.debug("Retrieving Slurm reservation with command: %s", cmd)
        output = check_command_output(
            cmd, raise_on_error=raise_on_error, timeout=command_timeout, shell=True, log_error=False
        )  # nosec B604
        reservation_exists = f"ReservationName={name}" in output

    except subprocess.CalledProcessError as e:
        expected_output = f"Reservation {name} not found"
        error = f" Error is: {e.stderr.rstrip()}." if e.stderr else ""
        output = f" Output is: {e.stdout.rstrip()}." if e.stdout else ""
        if expected_output in error or expected_output in output:
            logger.debug(f"Slurm reservation {name} not found.")
            reservation_exists = False
        else:
            msg = f"Failed when retrieving Slurm reservation info with command {cmd}.{error}{output} {e}"
            logger.error(msg)
            raise SlurmCommandError(msg)

    return reservation_exists

@retry(
    stop_max_attempt_number=2,
    wait_fixed=1000,
    retry_on_exception=lambda exception: isinstance(exception, SlurmCommandError),
)
@SlurmCommandErrorHandler.handle_slurm_command_error
def get_slurm_reservations_info(
    command_timeout=DEFAULT_SCONTROL_COMMAND_TIMEOUT, raise_on_error: bool = True
) -> List[SlurmReservation]:
    """
    List existing slurm reservations with scontrol call.

    The output of the command is something like the following:
    $ scontrol show reservations
    ReservationName=root_7 StartTime=2023-10-25T09:46:49 EndTime=2024-10-24T09:46:49 Duration=365-00:00:00
    Nodes=queuep4d-dy-crp4d-[1-5] NodeCnt=5 CoreCnt=480 Features=(null) PartitionName=(null) Flags=MAINT,SPEC_NODES
    TRES=cpu=480
    Users=root Groups=(null) Accounts=(null) Licenses=(null) State=ACTIVE BurstBuffer=(null) Watts=n/a
    MaxStartDelay=(null)

    Official documentation is https://slurm.schedmd.com/reservations.html
    """
    # awk is used to replace the \n\n record separator with '######\n'
    show_reservations_command = f"{SCONTROL} show reservations | {SCONTROL_SHOW_RESERVATION_OUTPUT_AWK_PARSER}"
    slurm_reservations_info = check_command_output(
        show_reservations_command, raise_on_error=raise_on_error, timeout=command_timeout, shell=True
    )  # nosec B604

    return _parse_reservations_info(slurm_reservations_info)

def _parse_reservations_info(slurm_reservations_info: str) -> List[SlurmReservation]:
    """Parse slurm reservations info into SlurmReservation objects."""
    # $ /opt/slurm/bin/scontrol show reservations awk 'BEGIN{{RS="\n\n" ; ORS="######\n";}} {{print}}' |
    # grep -oP '^(ReservationName=\S+)|(?<!Next)(State=\S+)|(Users=\S+)|(Nodes=\S+)|(######)'
    # ReservationName=root_8
    # Nodes=queuep4d-dy-crp4d-[1-5]
    # Users=root
    # State=ACTIVE
    # ######
    # ReservationName=root_9
    # Nodes=queue1-st-crt2micro-1
    # Users=root
    # State=ACTIVE
    # ######
    map_slurm_key_to_arg = {"ReservationName": "name", "Nodes": "nodes", "Users": "users", "State": "state"}

    reservation_info = slurm_reservations_info.split("######\n")
    slurm_reservations = []
    for reservation in reservation_info:
        lines = reservation.splitlines()
        kwargs = {}
        for line in lines:
            key, value = line.split("=")
            kwargs[map_slurm_key_to_arg[key]] = value
        if lines:
            reservation = SlurmReservation(**kwargs)
            slurm_reservations.append(reservation)

    return slurm_reservations


# coding=utf-8
'''
Selenium 3 版本需要下载第3方webdriver 'geckodriver'，并放到firefox资料夹中
之后再将该路径添加到环境变数path中
参考网址:https://www.zhihu.com/question/49568096
'''
from selenium import webdriver
from time import sleep

driver = webdriver.Firefox()
driver.get("https://www.google.com")

driver.find_element_by_id("lst-ib").send_keys("Selenium2")
driver.find_element_by_xpath(".//*[@id='tsf']/div[2]/div[3]/center/input[1]").click()
sleep(2)

driver.quit()


# -*- mode: python ; coding: utf-8 -*-

block_cipher = None
from os.path import *
import sys
import win32com.client
import zipfile
import PyInstaller.config
os.path.expanduser

# https://stackoverflow.com/a/42056050
def zipdir(path, ziph):
    length = len(path)

    # ziph is zipfile handle
    for root, dirs, files in os.walk(path):
        folder = root[length:] # path without "parent"
        for file in files:
            ziph.write(os.path.join(root, file), os.path.join(folder, file))

# pyinstaller build
PyInstaller.config.CONF['distpath'] = "./dist/potatofy_win"
a = Analysis(['potatofy.py'],
             pathex=['C:\\Users\\chanc\\Documents\\Projects\\potatofy'],
             binaries=[],
             datas=[('icons/Questrade.png', 'icons')],
             hiddenimports=['pkg_resources.py2_warn'],
             hookspath=[],
             runtime_hooks=[],
             excludes=[],
             win_no_prefer_redirects=False,
             win_private_assemblies=False,
             cipher=block_cipher,
             noarchive=False)
pyz = PYZ(a.pure, a.zipped_data,
             cipher=block_cipher)
exe = EXE(pyz,
          a.scripts,
          [],
          exclude_binaries=True,
          name='potatofy',
          debug=False,
          bootloader_ignore_signals=False,
          strip=False,
          upx=True,
          console=False )
coll = COLLECT(exe,
               a.binaries,
               a.zipfiles,
               a.datas,
               strip=False,
               upx=True,
               upx_exclude=[],
               name='potatofy')

# post-build script
shell = win32com.client.Dispatch("WScript.Shell")
shortcut = shell.CreateShortCut("./dist/potatofy_win/potatofy.lnk")
shortcut.Targetpath = abspath(abspath(".\dist\potatofy_win\potatofy\potatofy.exe"))
shortcut.save()

print("Creating zip")
os.chdir("./dist/")
zipf = zipfile.ZipFile('./potatofy.zip', 'w', zipfile.ZIP_DEFLATED)
zipdir('./potatofy_win/', zipf)
zipf.close()
print("Finished zip")


# https://www.acmicpc.net/problem/7576
# 토마토

import sys
from collections import deque

queue = deque()
visit = list()

dxs = [1, -1, 0, 0]
dys = [0, 0, 1, -1]

ripping = False
ripe_day = 0

size_M, size_N = list(map(int, sys.stdin.readline().strip().split()))
tomato_box = list()
for _ in range(size_N):
    tomato_box.append(list(map(int, sys.stdin.readline().strip().split())))

for i in range(size_N):
    for j in range(size_M):
        if tomato_box[i][j] == 1: queue.append([i, j])

while queue:
    node = queue.popleft()
    tomato_x, tomato_y = node

    for dx, dy in zip(dxs, dys):
        new_tomato_x, new_tomato_y = tomato_x+dx, tomato_y+dy
        if new_tomato_x>=0 and new_tomato_x<size_N \
            and new_tomato_y>=0 and new_tomato_y<size_M \
            and tomato_box[new_tomato_x][new_tomato_y] == 0:
            tomato_box[new_tomato_x][new_tomato_y] = tomato_box[tomato_x][tomato_y]+1
            queue.append([new_tomato_x, new_tomato_y])
            ripping = True

        if ripping:
            ripe_day = tomato_box[tomato_x][tomato_y]
            ripping = False

full_tomato = True
for i in range(size_N):
    for j in range(size_M):
        if tomato_box[i][j] == 0 :
            full_tomato = False
            break

if not full_tomato : print(-1)
else : print(ripe_day)


import base
import utils
import os
import sys
import info

class subinfo(info.infoclass):
    def setTargets( self ):
        self.svnTargets['gitHEAD'] = '[git]kde:pykde4'
        self.defaultTarget = 'gitHEAD'
        self.patchToApply['gitHEAD'] = ('pykde-20110318.patch', 1)

    def setDependencies( self ):
        self.hardDependencies['kde/kdelibs'] = 'default'
        self.hardDependencies['testing/sip'] = 'default'
        self.hardDependencies['testing/pyqt'] = 'default'

from Package.CMakePackageBase import *

class Package(CMakePackageBase):
    def __init__( self ):
        self.subinfo = subinfo()
        CMakePackageBase.__init__( self )
        localPythonPath = os.path.join(self.rootdir, 'emerge', 'python')
        haveLocalPython = os.path.exists(localPythonPath)
        if haveLocalPython:
            self.subinfo.options.merge.destinationPath = "emerge/python"
        else:
            # PyQt4 does not install itself in C:\Python27\Lib\site-packages
            # so how is python supposed to be able to import it?
            pyqtDir = os.path.join(self.rootdir,'lib','site-packages')
            if pyqtDir not in os.getenv("PYTHONPATH").split(';'):
                newPath = ";".join([os.getenv("PYTHONPATH"), pyqtDir])
                os.environ["PYTHONPATH"] = newPath
                utils.debug('added %s to PYTHONPATH' % pyqtDir, 2)

if __name__ == '__main__':
    Package().execute()


from collections import deque

while True:
    w, h = map(int,input().split())
    if w == 0 and h == 0:
        break
    cnt = 0
    arr = [list(map(int,input().split())) for _ in range(h)]
    # print(arr)
    dr = [1, 1, 0, -1, -1, -1, 0, 1]
    dc = [0, 1, 1, 1, 0, -1, -1, -1]

    queue = deque()
    for i in range(h):
        for j in range(w):
            if arr[i][j] == 1:
                queue.append([i,j])
                arr[i][j] = -1

                while queue:
                    r, c = queue.popleft()
                    for k in range(8):
                        nr = r + dr[k]
                        nc = c + dc[k]

                        if 0<= nr < h and 0<= nc < w and arr[nr][nc] == 1:
                            queue.append([nr, nc])
                            arr[nr][nc] = -1

                cnt += 1
    print(cnt)


from urllib import request
from django.shortcuts import get_object_or_404, render, reverse, redirect
from django.contrib.auth.decorators import login_required
from .models import Suggestion
from .forms import SuggestionForm
from django.contrib import messages

def view_suggestions(request):
    """
    View all submitted suggestions
    """
    suggestions = Suggestion.objects.all().order_by('-number_of_upvotes')

    context = {
        'suggestions': suggestions,
    }

    return render(request, 'suggestions/view_suggestions.html', context)

@login_required
def add_suggestion(request):
    """
    Add a suggestion form if request = GET
    Submits the form if request = POST
    """
    if request.method == "POST":
        suggestion_form = SuggestionForm(request.POST)
        if suggestion_form.is_valid():
            suggestion = suggestion_form.save(commit=False)
            suggestion.creator = request.user
            suggestion.save()
            messages.success(request, 'Successfully added suggestion!')
            return redirect(reverse('view_suggestions'))
    else:
        suggestion_form = SuggestionForm()

        context = {
            'suggestion_form': suggestion_form,
        }

        return render(request, 'suggestions/add_suggestion.html', context)

@login_required
def edit_suggestion(request, suggestion_id):
    """
    Load the suggestion in the editor to allow the creator to make changes
    """
    if request.method == "POST":
        suggestion = get_object_or_404(Suggestion, pk=suggestion_id)
        suggestion_form = SuggestionForm(request.POST, instance=suggestion)
        if suggestion_form.is_valid():
            suggestion = suggestion_form.save()
            messages.success(request, 'Suggestion succesfully updated!')
        else:
            messages.warning(request, 'Error while saving, please try again')

        return redirect(reverse('view_suggestions'))
    else:
        suggestion = get_object_or_404(Suggestion, pk=suggestion_id)
        suggestion_form = SuggestionForm(instance=suggestion)
        context = {
            'suggestion_form': suggestion_form,
            'suggestion': suggestion,
        }

        return render(request, 'suggestions/edit_suggestion.html', context)

@login_required
def delete_suggestion(request, suggestion_id):
    """
    Allows the creator to delete the suggestion
    """
    suggestion = get_object_or_404(Suggestion, pk=suggestion_id)
    suggestion.delete()
    messages.warning(request, 'Suggestion succesfully deleted')

    return redirect(reverse('view_suggestions'))

@login_required
def upvote_suggestion(request, suggestion_id):
    """
    Upvotes a suggestion if a user is logged in
    and hasn't voted for this suggestion before
    """
    suggestion = get_object_or_404(Suggestion, pk=suggestion_id)
    user = request.user
    # check if user is in list of upvoters
    upvoters = suggestion.upvoters_as_list()
    if str(user) not in upvoters:
        suggestion.upvoters = str(suggestion.upvoters) + f';{user}'
        suggestion.number_of_upvotes += 1
        messages.success(request, f'Upvoted: {suggestion.common_name}')
        suggestion.save()
    else:
        messages.warning(request, 'You have already voted for this plant')

    return redirect(reverse('view_suggestions'))


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# author: a2htray
# create date: 2023/4/18

"""
PAT 乙级 1082
"""

from functools import cmp_to_key

def cmp(a, b):
    if a[1] > b[1]:
        return 1
    else:
        return -1

if __name__ == '__main__':
    n = int(input())

    players = []
    for _ in range(n):
        idstr, x, y = input().split()

        players.append([idstr, int(x) ** 2 + int(y) ** 2])

    players.sort(key=cmp_to_key(cmp))

    print(players[0][0], players[-1][0])


import os
from flask import Flask, g
from resources.users import users
from resources.dogs import dogs
import models
from flask_cors import CORS
from flask_login import LoginManager

DEBUG=True
PORT=8000

app = Flask(__name__)

### LoginManager Configuration ###
## SECRET/KEY FOR SESSIONS ##
app.secret_key = 'Milo is the best dog in the universe! This is not a secret.'
## INSTANTIATE LoginManager ##
login_manager = LoginManager()
## CONNECT APP WITH THE LOGIN MANAGER ##
login_manager.init_app(app)

@login_manager.user_loader
def load_user(user_id):
	try:
		print('loading the following user')
		user = models.User.get_by_id(user_id)
		return user

	except models.DoesNotExist:
		return None

@login_manager.unauthorized_handler
def unauthorized():

	# response
	return jsonify(
		data={
			'error': 'User not logged in'
		},
		message="You must have an account to access this resource",
		status=401
	), 401

### CORS -- CROSS ORIGIN RESOURCE SHARING ###
CORS(users, origins=['http://localhost:3000', 'https://perritos-camperos.herokuapp.com'], supports_credentials=True)
CORS(dogs, origins=['http://localhost:3000', 'https://perritos-camperos.herokuapp.com'], supports_credentials=True)

### "CONTROLLERS" ###
app.register_blueprint(users, url_prefix='/api/v1/users')
app.register_blueprint(dogs, url_prefix='/api/v1/dogs')

### PSTGRESQL ###
@app.before_request
def before_request():
	"""Connect to the database before each request"""
	print('you should see this before each request')
	g.db = models.DATABASE
	g.db.connect()

@app.after_request
def after_request(response):
	"""Close the database connection after each request"""
	print('you should see this after each request')
	g.db.close()
	return response

@app.route('/')
def hello():
	return 'Hello, World!'

if 'ON_HEROKU' in os.environ:
	print('\non heroku!')
	models.initialize()

if __name__ == '__main__':
	models.initialize()
	app.run(debug=DEBUG, port=PORT)


import json

from django.conf.urls import url

from tastypie.authentication import MultiAuthentication, SessionAuthentication
from tastypie.resources import Resource

from repos.models import Repository
from visualizations.models import Visualization, VisualizationSerializer

from .authentication import ApiTokenAuthentication
from .authorization import VizAuthorization

class VizResource( Resource ):

    class Meta:
        resource_name = 'viz'

        detail_allowed_methods = [ 'delete', 'get', 'post' ]

        # Ensure we have an API token before returning any data.
        # TODO: Make sure this API token concept works with public/private
        # data.
        authentication = MultiAuthentication( ApiTokenAuthentication(),
                                              SessionAuthentication() )

        authorization = VizAuthorization()

    def prepend_urls( self ):
        base_url = '^(?P<resource_name>%s)/(?P<pk>\w+)' % ( self._meta.resource_name )

        return [
            url( r'%s/(?P<viz_id>\d+)/' % ( base_url ), self.wrap_view( 'delete_detail' ), name='api_delete_detail' )
        ]

    def dehydrate( self, bundle ):
        serializer = VisualizationSerializer()
        bundle.data = serializer.serialize( [bundle.obj] )
        return bundle

    def delete_detail( self, request, **kwargs ):

        # Grab the repo we're creating a visualization for
        repo_id = kwargs.get( 'pk' )
        viz_id  = kwargs.get( 'viz_id' )

        try:
            basic_bundle = self.build_bundle( request=request )
            basic_bundle.obj = Visualization.objects.get( id=viz_id, repo__mongo_id=repo_id )

            if not self.authorized_delete_detail( basic_bundle.obj, basic_bundle ):
                return HttpUnauthorized()

            basic_bundle.obj.delete()
            response_data = { 'success': True }
            return self.create_response( request, response_data )

        except ValueError as e:
            return HttpBadRequest( str( e ) )

        return HttpUnauthorized()

    def get_detail( self, request, **kwargs ):

        # Grab the repo we're creating a visualization for
        repo_id = kwargs[ 'pk' ]

        try:
            basic_bundle = self.build_bundle( request=request )
            basic_bundle.obj = Visualization.objects.get(repo__mongo_id=repo_id)

            if not self.authorized_read_detail( basic_bundle.obj, basic_bundle ):
                return HttpUnauthorized()

            basic_bundle = self.dehydrate( basic_bundle )

            data = {
                'objects': basic_bundle.data }

            return self.create_response( request, data )
        except ValueError as e:
            return HttpBadRequest( str( e ) )

    def post_detail( self, request, **kwargs ):

        # Grab the repo we're creating a visualization for
        repo_id = kwargs[ 'pk' ]

        try:
            basic_bundle = self.build_bundle( request=request )

            repo = Repository.objects.get( mongo_id=repo_id )
            if not self.authorized_create_detail( repo, basic_bundle ):
                return HttpUnauthorized()

            params = json.loads( request.body )

            new_viz = Visualization( name=params.get( 'name' ),
                                     repo=repo,
                                     x_axis=params.get( 'x_axis' ),
                                     y_axis=params.get( 'y_axis' ) )
            new_viz.save()
            response_data = { 'success': True, 'id': str( new_viz.id ) }
            return self.create_response( request, response_data )

        except ValueError as e:
            return HttpBadRequest( str( e ) )


"""
In charge of indexing and preprocessing road parts.
"""

from dataclasses import dataclass, field
from hashlib import sha384

from panda3d import core
from direct.stdpy.file import *
from direct.stdpy import pickle

from .common import PR_DEFAULT_DENSITY
from . import preprocess as pp
from . import util

def chk_timestamp(key, base_type):
    h = sha384(f'{key}-{base_type}'.encode()).hexdigest()
    lvl_fname = core.Filename(base.main_dir, f'assets/cache/{h}')
    bam_fname = core.Filename(base.main_dir, f'assets/models/{key}.bam')
    timestamp = 0
    need_reload = True
    if lvl_fname.exists():
        with open(lvl_fname.to_os_specific(), 'r') as f:
            try:
                timestamp = int(f.read())
            except ValueError:
                pass
    else:
        lvl_fname.make_dir()
        need_reload = True
    bts = bam_fname.get_timestamp()
    if bts == timestamp:
        need_reload = False
    with open(lvl_fname.to_os_specific(), 'w') as f:
        f.write(str(bts))
    return need_reload, lvl_fname

def get_bounds(need_reload, key, np):
    h = sha384(f'{key}-road-{np.name}'.encode()).hexdigest()
    fname = core.Filename(base.main_dir, f'assets/cache/{h}')
    if need_reload:
        bounds = pp.get_model_bounds(np)
        with open(fname.to_os_specific(), 'wb') as f:
            pickle.dump(bounds, f)
    else:
        with open(fname.to_os_specific(), 'rb') as f:
            bounds = pickle.load(f)
    return bounds

@dataclass
class Part:
    level_type:str
    part_type:str
    model:core.NodePath
    bounds:pp.Bounds
    density:float = -1.0

class PartMgr:
    def __init__(self, models, keys=('test', )):
        self._parts = {}
        self._build_lib(models, keys)

    def _build_lib(self, models, keys):
        for k in keys:
            if k == 'parts':
                self._scan_road_parts(models)
            elif k == 'props':
                self._scan_props(models)
            else:
                print(f'Unknown key "{k}"')

    def _compute_ground(self, key):
        ground = float('inf')
        for part in self._parts[key]['road']:
            ground = min(ground, part.bounds.depth)
        self._parts[key]['ground'] = ground

    def _scan_road_parts(self, models):
        for key, model in models.items():
            need_reload, lvl_fname = chk_timestamp(key, 'road')
            if key not in self._parts:
                self._parts[key] = {}
            if 'road' not in self._parts[key]:
                self._parts[key]['road'] = []
            for np in model.find_all_matches('**/road_part*'):
                np.detach_node()
                util.set_faux_lights(np)
                np.clear_transform()
                bounds = get_bounds(need_reload, key, np)
                part = Part(key, np.name, np, bounds)
                if 'transition' in np.name:
                    self._parts[key]['transition'] = part
                else:
                    self._parts[key]['road'].append(part)
                np.flatten_strong()
            self._compute_ground(key)

    def _scan_props(self, models):
        for key, model in models.items():
            need_reload, lvl_fname = chk_timestamp(key, 'props')
            if key not in self._parts:
                self._parts[key] = {}
            if 'props' not in self._parts[key]:
                self._parts[key]['props'] = []
            for np in model.find_all_matches('**/props*'):
                np.detach_node()
                np.clear_transform()
                for child in np.get_children():
                    child.detach_node()
                    child.clear_transform()
                    d = child.get_net_tag('density')
                    d = float(d) if d else PR_DEFAULT_DENSITY
                    bounds = get_bounds(need_reload, key, child)
                    part = Part(key, child.name, child, bounds, d)

                    lights = []
                    for n in child.find_all_matches('**/*light*'):
                        n.detach_node()
                        lights.append(n)
                    child.flatten_strong()
                    for n in lights:
                        n.reparent_to(child)
                    util.set_faux_lights(child)
                    if part.bounds.width > 0:
                        self._parts[key]['props'].append(part)
                    else:
                        print(f'Could not scan Prop, apparently width is 0: {part}')

    def get_road_part(self, part_type, variant):
        return self._parts[part_type]['road'][variant]

    def get_road_transition(self, part_type):
        return self._parts[part_type]['transition']

    def get_prop(self, part_type, variant):
        return self._parts[part_type]['props'][variant]

    def get_prop_by_density(self, part_type, density_min):
        return [i for i in self._parts[part_type]['props'] if i.density >= density_min]

    def get_prop_by_prefix(self, part_type, prefix):
        return [i for i in self._parts[part_type]['props'] if i.part_type.startswith(f'{prefix}')]

    def get_part_types(self):
        return list(self._parts.keys())

    def num_roads(self, part_type):
        return len(self._parts[part_type]['road'])

    def num_props(self, part_type):
        return len(self._parts[part_type]['props'])

    def ground(self, part_type):
        return self._parts[part_type]['ground']

    def store(self, fpath):
        pass

    def load(self, fpath):
        pass

    def __getitem__(self, item):
        return self._parts[item[0]][item[1]]


import torch
from torch import nn
import torchvision.models as models
from transformers import *
# from transformers.pytorch_transformers import *
# from pytorch_transformers import *
import torch.nn.functional as F
from .frozen_batch_norm import FrozenBatchNorm2d

LANG_MODELS = {
          'bert':    (BertModel,       BertTokenizer,       'bert-base-uncased'),
          'bert-large':  (BertModel,       BertTokenizer,       'bert-large-uncased'),
          'gpt':     (OpenAIGPTModel,  OpenAIGPTTokenizer,  'openai-gpt'),
          'gpt2':    (GPT2Model,       GPT2Tokenizer,       'gpt2'),
        #   'ctrl':    (CTRLModel,       CTRLTokenizer,       'ctrl'),
        #   'xl':      (TransfoXLModel,  TransfoXLTokenizer,  'transfo-xl-wt103'),
        #   'xlnet':   (XLNetModel,      XLNetTokenizer,      'xlnet-base-cased'),
        #   'xlm':     (XLMModel,        XLMTokenizer,        'xlm-mlm-enfr-1024'),
        #   'distil':  (DistilBertModel, DistilBertTokenizer, 'distilbert-base-cased'),
        #   'roberta': (RobertaModel,    RobertaTokenizer,    'roberta-base'),
        #   'xlm-roberta': (XLMRobertaModel, XLMRobertaTokenizer, 'xlm-roberta-base'),
}

# sample for use: classifier = MLP( 2048, [1024,1024,3000])
#  x = classifier(x)
class MLP(nn.Module):

    def __init__(self,
            input_dim,
            dimensions,
            activation='relu',
            dropout=0.2):
        super(MLP, self).__init__()
        self.input_dim = input_dim
        self.dimensions = dimensions
        self.activation = activation
        self.dropout = dropout
        # Modules
        self.linears = nn.ModuleList([nn.Linear(input_dim, dimensions[0])])
        for din, dout in zip(dimensions[:-1], dimensions[1:]):
            self.linears.append(nn.Linear(din, dout))

    def forward(self, x):
        for i,lin in enumerate(self.linears):
            x = lin(x)
            if (i < len(self.linears)-1):
                x = F.__dict__[self.activation](x)
                if self.dropout > 0:
                    x = F.dropout(x, self.dropout, training=self.training)
        return x

def get_visn_arch(arch):
    try:
        return getattr(models, arch)
    except AttributeError as e:
        print(e)
        print("There is no arch %s in torchvision." % arch)

class Identity(nn.Module):
    r"""Identity activation layer."""

    def forward(self,  # type: ignore
                input: torch.Tensor) -> torch.Tensor:
        return input

# Modified by Jokie
class VisnModel(nn.Module):
    def __init__(self, dim, arch='resnet50', pretrained=True, finetuning=False):
        """
        :param dim: dimension of the output
        :param arch: backbone architecture,
        :param pretrained: load feature with pre-trained vector
        :param finetuning: finetune the model
        """
        super().__init__()
        self.finetuning = finetuning

        # Setup Backbone
        resnet = get_visn_arch(arch)(pretrained=pretrained)
        backbone_dim = resnet.fc.in_features
        if not self.finetuning:
            for param in resnet.parameters():
                param.requires_grad = False
        resnet.fc = nn.Identity()
        # resnet.fc = Identity()
        self.backbone = resnet

        # Surgery on the Networks
        # 1. Frozen Batch Norm
        #    Note that BatchNorm modules have been in-place replaced!
        #    This piece of code is copied from Detectron2, and it was copied from mask-rcnn?
        self.backbone = FrozenBatchNorm2d.convert_frozen_batchnorm(
            self.backbone)
        # print(self.backbone)
        # 2. Frozen the first two (blocks of) layers
        for module in [self.backbone.conv1,
                       self.backbone.layer1]:
            for param in module.parameters():
                param.requires_grad = False

        print(f"Visn Model: {arch}, Finetune: {finetuning}, Pre-trained: {pretrained}")
        print(f"Visn Model: backbone dim {backbone_dim} --> output dim {dim}")

        # Setup follow-up layers
        # self.mlp = nn.Sequential(
        #     nn.Linear(backbone_dim, 256),
        #     nn.ReLU(),
        #     nn.Dropout(0.3),
        #     nn.Linear(256, dim),
        # )

        self.mlp = MLP(backbone_dim, [1024,1024, dim])

    def forward(self, img):
        """
        :param img: a tensor of shape [batch_size, H, W, C]
        :return: a tensor of [batch_size, d]
        """
        if not self.finetuning:
            with torch.no_grad():
                x = self.backbone(img)
                x = x.detach()
        else:
            x = self.backbone(img)
        x = self.mlp(x)         # [b, dim]
        x = x / x.norm(2, dim=-1, keepdim=True)
        return x

# Modified by Jokie
class Sent_LangModel(nn.Module):
    def __init__(self, dim, arch='BERT', layers=(-1,), pretrained=True, finetuning=False):
        """
        :param dim: dimension of the output
        :param arch: backbone architecture,
        :param aggregate: one of 'last4',
        :param pretrained: load feature with pre-trained vector
        :param finetuning: finetune the model
        """
        super().__init__()
        self.finetuning = finetuning

        # Setup Backbone
        Model, Tokenizer, weight = LANG_MODELS[arch]
        bert = Model.from_pretrained(
            weight,
            output_hidden_states=True
        )
        if not pretrained:
            bert.init_weights()

        if not self.finetuning:
            for param in bert.parameters():
                param.requires_grad = False
        backbone_dim = bert.config.hidden_size
        self.backbone = bert
        self.layers = sorted(layers)

        print(f"Language Model: {arch} with weight {weight}; Fine-tuning: {finetuning}, Pre-trained: {pretrained}.")
        print(f"Language Model: using layers {self.layers}, result in backbone dim {backbone_dim * len(self.layers)} "
              f"--> output dim {dim}.")

        # Setup follow-up layers
        # self.mlp = nn.Sequential(
        #     nn.Linear(backbone_dim * len(self.layers), dim),
        #     nn.ReLU(),
        #     nn.Dropout(0.3),
        #     nn.Linear(dim, dim),
        # )

        self.mlp = MLP(backbone_dim * len(self.layers), [1024,1024, dim])

    def forward(self, input_ids, attention_mask, token_type_ids=None):
        """
        :param input_ids: [batch_size, max_len]
        :param attention_mask: [batch_size, max_len]
        :param token_type_ids: [batch_size, max_len]
        :return: [batch_size, max_len, dim]
        """
        if not self.finetuning:
            with torch.no_grad():
                x = self.backbone(
                    input_ids,
                    attention_mask=attention_mask,
                    token_type_ids=token_type_ids,
                )
        else:
            x = self.backbone(
                input_ids,
                attention_mask=attention_mask,
                token_type_ids=token_type_ids,
            )

        # sequence_output, pooled_output, (hidden_states), (attentions) --> seq_output
        if type(self.backbone) is XLNetModel:
            output, hidden_states = x[:2]
        else:
            output, pooled_output, hidden_states = x[:3]

        # gather the layers
        if type(self.backbone) is XLNetModel:
            x = torch.cat(list(hidden_states[layer].permute(1, 0, 2) for layer in self.layers), -1)
        else:
            x = torch.cat(list(hidden_states[layer] for layer in self.layers), -1)

        if not self.finetuning:
            x = x.detach()

        # if sentence model
        # [batch_size, backbone_dim] -->
        # [batch_size, output_dim]
        x = x[:,0,:]
        x = self.mlp(x)
        x = x / x.norm(2, dim=-1, keepdim=True)
        return x

class LangModel(nn.Module):
    def __init__(self, dim, arch='BERT', layers=(-1,), pretrained=True, finetuning=False):
        """
        :param dim: dimension of the output
        :param arch: backbone architecture,
        :param aggregate: one of 'last4',
        :param pretrained: load feature with pre-trained vector
        :param finetuning: finetune the model
        """
        super().__init__()
        self.finetuning = finetuning

        # Setup Backbone
        Model, Tokenizer, weight = LANG_MODELS[arch]
        bert = Model.from_pretrained(
            weight,
            output_hidden_states=True
        )
        if not pretrained:
            bert.init_weights()

        if not self.finetuning:
            for param in bert.parameters():
                param.requires_grad = False
        backbone_dim = bert.config.hidden_size
        self.backbone = bert
        self.layers = sorted(layers)

        print(f"Language Model: {arch} with weight {weight}; Fine-tuning: {finetuning}, Pre-trained: {pretrained}.")
        print(f"Language Model: using layers {self.layers}, result in backbone dim {backbone_dim * len(self.layers)} "
              f"--> output dim {dim}.")

        # Setup follow-up layers
        self.mlp = nn.Sequential(
            nn.Linear(backbone_dim * len(self.layers), 256),
            nn.ReLU(),
            nn.Dropout(0.3),
            nn.Linear(256, dim),
        )

    def forward(self, input_ids, attention_mask, token_type_ids=None):
        """
        :param input_ids: [batch_size, max_len]
        :param attention_mask: [batch_size, max_len]
        :param token_type_ids: [batch_size, max_len]
        :return: [batch_size, max_len, dim]
        """
        if not self.finetuning:
            with torch.no_grad():
                x = self.backbone(
                    input_ids,
                    attention_mask=attention_mask,
                    token_type_ids=token_type_ids,
                )
        else:
            x = self.backbone(
                input_ids,
                attention_mask=attention_mask,
                token_type_ids=token_type_ids,
            )

        # sequence_output, pooled_output, (hidden_states), (attentions) --> seq_output
        if type(self.backbone) is XLNetModel:
            output, hidden_states = x[:2]
        else:
            output, pooled_output, hidden_states = x[:3]

        # gather the layers
        if type(self.backbone) is XLNetModel:
            x = torch.cat(list(hidden_states[layer].permute(1, 0, 2) for layer in self.layers), -1)
        else:
            x = torch.cat(list(hidden_states[layer] for layer in self.layers), -1)

        if not self.finetuning:
            x = x.detach()

        # [batch_size, max_len, backbone_dim] -->
        # [batch_size, max_len, output_dim]
        x = self.mlp(x)
        x = x / x.norm(2, dim=-1, keepdim=True)

        # if sentence model
        # [batch_size, max_len, backbone_dim] -->
        # [batch_size, max_len, output_dim]
        # x = x[:,0,:]
        # x = self.mlp(x)
        # x = x / x.norm(2, dim=-1, keepdim=True)
        return x

class JointModel(nn.Module):
    def __init__(self, lang_model, visn_model):
        super().__init__()
        self.lang_model = lang_model
        self.visn_model = visn_model

    def forward(self, lang_input, visn_input):
        lang_output = self.lang_model(*lang_input)
        visn_output = self.visn_model(*visn_input)
        return lang_output, visn_output


from datetime import datetime, timedelta
import os
from airflow import DAG
from airflow.operators.dummy_operator import DummyOperator
from airflow.operators import (StageToRedshiftOperator, LoadFactOperator,
                                LoadDimensionOperator, DataQualityOperator, CleanCsvOperator, PostgresOperator)
from helpers import SqlQueries

# AWS_KEY= os.environ.get('AWS_KEY')
# AWS_S ECRET = os.environ.get('AWS_SECRET')

default_args = {
    'owner': 'udacity',
    'start_date': datetime(2020, 3, 28),
    'depends_on_past': False,
    'retries': 0,
    'retry_delay': timedelta(minutes=5),
    'catchup': False,
    'email_on_retry': False
}

dag = DAG('udac_capstone_project',
          default_args=default_args,
          description='Load and transform data in Redshift with Airflow',
          start_date=datetime.now(),
          schedule_interval='0 * * * *'
        )

start_operator = DummyOperator(task_id='Begin_execution',  dag=dag)

clean_data_csv = CleanCsvOperator(
    task_id="clean_data_csv",
    dag=dag,
    aws_credentials_id='aws_credentials',
    s3_bucket='audacity-capstone-luke',
    s3_key='covid_19_data.csv',
    region='ap-southeast-2',
    datecols=[]
)

clean_case_csv = CleanCsvOperator(
    task_id="clean_case_csv",
    dag=dag,
    aws_credentials_id='aws_credentials',
    s3_bucket='audacity-capstone-luke',
    s3_key='COVID19_line_list_data.csv',
    region='ap-southeast-2',
    datecols=['reporting date', 'symptom_onset', 'hosp_visit_date', 'exposure_start', 'exposure_end']
)

create_tables = PostgresOperator(
    task_id="create_tables",
    dag=dag,
    postgres_conn_id="redshift",
    sql="create_tables.sql"
)

stage_cases_to_redshift = StageToRedshiftOperator(
    task_id='Stage_cases',
    dag=dag,
    redshift_conn_id='redshift',
    aws_credentials_id='aws_credentials',
    table='staging_cases',
    s3_bucket='audacity-capstone-luke',
    s3_key='COVID19_line_list_data.csv',
    region='ap-southeast-2'
)

stage_data_to_redshift = StageToRedshiftOperator(
    task_id='Stage_data',
    dag=dag,
    redshift_conn_id='redshift',
    aws_credentials_id='aws_credentials',
    table='staging_data',
    s3_bucket='audacity-capstone-luke',
    s3_key='covid_19_data.csv',
    region='ap-southeast-2'
)

load_cases_table = LoadFactOperator(
    task_id='Load_cases_fact_table',
    dag=dag,
    redshift_conn_id='redshift',
    table='fact_cases',
    select_sql=SqlQueries.fact_cases_table_insert
)

load_patients_table = LoadFactOperator(
    task_id='Load_patients_fact_table',
    dag=dag,
    redshift_conn_id='redshift',
    table='patientinfo',
    select_sql=SqlQueries.patient_info_table_insert
)

load_data_table = LoadFactOperator(
    task_id='Load_data_fact_table',
    dag=dag,
    redshift_conn_id='redshift',
    table='fact_data',
    select_sql=SqlQueries.fact_data_table_insert
)

load_location_dimension_table = LoadDimensionOperator(
    task_id='Load_location_dim_table',
    dag=dag,
    redshift_conn_id='redshift',
    table='dim_locations',
    cols='(location, country)',
    select_sql=SqlQueries.dim_location_table_insert
)

load_source_dimension_table = LoadDimensionOperator(
    task_id='Load_source_dim_table',
    dag=dag,
    redshift_conn_id='redshift',
    table='dim_sources',
    cols='(sourcename, link)',
    select_sql=SqlQueries.dim_sources_table_insert
)

load_time_dimension_table = LoadDimensionOperator(
    task_id='Load_time_dim_table',
    dag=dag,
    redshift_conn_id='redshift',
    table='dim_times',
    cols='',
    select_sql=SqlQueries.time_table_insert
)

run_quality_checks = DataQualityOperator(
    task_id='Run_data_quality_checks',
    dag=dag,
    redshift_conn_id='redshift',
    tables=["fact_cases", "fact_data", "patientinfo", "dim_locations", "dim_times", "dim_sources"]
)

end_operator = DummyOperator(task_id='Stop_execution',  dag=dag)

start_operator >> create_tables
create_tables >> clean_case_csv >> stage_cases_to_redshift
create_tables >> clean_data_csv >> stage_data_to_redshift
stage_cases_to_redshift >> load_location_dimension_table
stage_data_to_redshift >> load_location_dimension_table
stage_cases_to_redshift >> load_source_dimension_table
load_location_dimension_table >> load_patients_table
load_source_dimension_table >> load_cases_table
load_location_dimension_table >> load_data_table
load_data_table >> load_time_dimension_table
load_cases_table >> load_time_dimension_table
load_patients_table >> load_time_dimension_table
load_time_dimension_table >> run_quality_checks
run_quality_checks >> end_operator


from typing import Any, List, Optional

from wire.core.container import Container
from wire.core.settings import BaseAppSettings

def prometheus_metrics_provider(
    container: Container[BaseAppSettings],
) -> Optional[List[Any]]:
    """Add prometheus metrics to your application."""
    if not container.settings.telemetry.metrics_enabled:
        return None

    from prometheus_fastapi_instrumentator import Instrumentator

    instrumentor = Instrumentator(
        excluded_handlers=[
            path.strip() for path in container.settings.telemetry.ignore_path.split(",")
        ]
    )
    instrumentor.instrument(container.app).expose(
        container.app,
        endpoint=container.settings.telemetry.metrics_path,
        include_in_schema=True,
        tags=["Telemetry"],
    )

    return [instrumentor]


from flask import Flask
from flask_restful import Resource, Api, reqparse, abort

app = Flask(__name__)
api = Api(app)

info = {}

parser = reqparse.RequestParser()
parser.add_argument("phone", type=int, help="Number is required", required=True)
parser.add_argument("age", type=int, help="Age must be an integer")

class HelloWorld(Resource):
    def get(self):
        return {"Hello": "World"}

def not_found(name):
    if name not in info:
        abort(404, message='Cannot find friend')

def dupe_entry(name):
    if name in info:
        abort(409, message='friend already exists')

class Basics(Resource):
    # def get(self, name):
    #     return {"Hello": name}
    def get(self, name):
        not_found(name)
        return info[name], 200

    def post(self, name):
        dupe_entry(name)
        args = parser.parse_args()
        info[name] = args
        return info[name], 201

    def put(self, name):
        args = parser.parse_args()
        info[name] = args
        return info[name], 201

    def delete(self,name):
        del info[name]
        return '', 204

api.add_resource(HelloWorld, '/hi')
api.add_resource(Basics, '/basic/<string:name>')

if __name__ == '__main__':
    app.run(debug=True)


import serial
import time

def serialWriteInt(ser, value):
	'''
	Send value to serial device and delays for buffering of target device.
	Parameters:
		ser - Serial. Target device object.
		value - Interger. Number to be sent.
	'''
	ser.write(str(value).encode('utf-8'))
	print('Serial write: ' + str(value))
	time.sleep(0.002 * len(str(value)))

def main():
	ser = serial.Serial('/dev/ttyACM0', 9600)
	x = 0
	while True:
		# Rising edge
		while x<255:
			x+=5
			serialWriteInt(ser, x)

		# Falling edge
		while x>0:
			x-=5
			serialWriteInt(ser, x)

if __name__ == '__main__':
	main()


"""
# Title : PySpark Script - Create Parquet
# Description : This script is used to read a CSV file, clean it and persist it to Parquet file
# Usage : spark-submit write-parquet.py
"""

# First import the neccesary modules
from pyspark.sql import SparkSession
from pyspark.sql.types import StructType, StructField, StringType, TimestampType, DecimalType
import sys,logging
from datetime import datetime

# Logging configuration
formatter = logging.Formatter('[%(asctime)s] %(levelname)s @ line %(lineno)d: %(message)s')
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.INFO)
handler.setFormatter(formatter)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.addHandler(handler)

# Schema Configuration
# Here we define a schema for the CSV File read
# If a record is not compatible with the CSV, a null value is read

schema = StructType([
    StructField("trasaction_timestamp", TimestampType(), True),
    StructField("transaction_amount", DecimalType(4, 4), True),
    StructField("transaction_channel", StringType(), True)])

# current time variable to be used for logging purpose
dt_string = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")

AppName = "Spark-Project"

def main():

    # Start spark code
    spark = SparkSession.builder.appName(AppName+"_"+str(dt_string)).getOrCreate()
    spark.sparkContext.setLogLevel("ERROR")
    logger.info("Starting spark application")

    # Read CSV File into a Spark Dataframe
    logger.info("Reading CSV File")
    df = spark.read.load('hdfs://localhost:9000/files/data.csv', format="csv", sep=",", header="false", schema=schema)

    # Preview Spark Dataframe (for the project's purposes we use df.show(200) to see all records)
    logger.info("Previewing CSV File Data")
    print(df.show(10))

    # Clean and Transform the Spark Dataframe
    # When string (and generally a data type) is transformed to timestamp (to another data type) and doesn't match timestamp (this data type) format null is returned

    # Define the wanted values for the transaction_channel column
    # Assumption: Only Call and Email Values valid
    type_list = ["Call", "Email"]

    # Keep only the rows that have these values
    df = df.filter(df["transaction_channel"].isin(type_list))

    # Drop the rows with null
    df = df.na.drop()

    logger.info("Ending spark application")

    # Persist to parquet file, overwrite if already exists, and save it to HDFS
    df.write.mode("overwrite").parquet("/files/test")

    spark.stop()
    return None

# Starting point for PySpark
if __name__ == '__main__':
    main()
    sys.exit()