import numpy as np
#import ExoMinerArch as ExoMiner

import torch.nn as nn
import FeatureEngineering as FE
import GatherData as gd
import torch.optim as optim
import pandas as pd
import matplotlib.pyplot as plt
import shutil
import glob

#import MyDataSet as D
#import RawDataSet as R
from torch.utils.data import DataLoader
import torch

from torchmetrics.classification import BinaryPrecision, BinaryRecall,BinaryAccuracy,BinaryStatScores,Precision , BinaryAUROC,BinaryF1Score
import ExoMinerRawVariation as ExoMinerRaw
def GenRandomTensor(size):
    return torch.rand(size)
def RandTestModel(seed=None):
    if (seed == None):
        pass
    else:
        torch.manual_seed(seed)
    f1 = GenRandomTensor((1, 1, 301))  # feature 1 input data
    f2 = GenRandomTensor((1, 1, 31))  # feature 2 input data
    f2_2 = GenRandomTensor((1, 1))  # feature 2 transit depth
    f3 = GenRandomTensor((1, 1, 301))  # feature 3 input data
    f4 = GenRandomTensor((1, 1, 31))  # feature 4 input data
    f4_2 = GenRandomTensor((1, 5))  # feature 4 centroid scalars
    f5_odd = GenRandomTensor((1, 1, 31))  # feature 5 odd input data
    f5_even = GenRandomTensor((1, 1, 31))  # feature 5 even input data
    f6 = GenRandomTensor((1, 1, 31))  # feature 6 input data
    f6_2 = GenRandomTensor((1, 4))  # feature 6 secondary scalars
    f7 = GenRandomTensor((1, 6))  # feature 7 stellar parameters
    f8 = GenRandomTensor((1, 6))  # feature 7 DV Diagnostic tests
    model = ExoMiner.ExoMiner()
    out = model.forward(f1, f2, f2_2, f3, f4, f4_2, f5_odd, f5_even, f6, f6_2, f7, f8)
    return out
def GenerateFeatures(ID, mish, Plot):
    Data = FE.Features(mish=mish, max_number_of_obs=3, Chosen_ID=ID, Plot=Plot)
    return Data
def Test():
    out = RandTestModel(1)
    print(out)
    return None
def GetRandomID(DataObject, label='CAND'):
    ValidCriteria = DataObject.IdData['WrapperDispo'][DataObject.IdData['WrapperDispo'] == label]
    idx = np.random.choice(ValidCriteria.index.values)
    return idx

def ReshapeEnsureNotNan(Feature, shape,nan=0):
    Feature[Feature==0] = nan
    Feature[Feature < 0]= nan
    return torch.tensor(np.nan_to_num(Feature,nan = nan)).reshape(shape).float()
def ReshapeEnsureNotNan2D(Feature,shape,nan=0):
    f1 = ReshapeEnsureNotNan(Feature[0], (shape[0],1,shape[2]),nan=nan)
    f2 = ReshapeEnsureNotNan(Feature[1], (shape[0], 1,shape[2]), nan=nan)
    return torch.cat((f1, f2), 1)
def UnPackData(mission,fp='FeatureData/'):
    files = glob.glob('./'+fp+mission+'/'+mission+r'Features_*.npy')
    idxs = ['load' not in file for file in files]
    f = []
    for i in range (0,len(idxs)):
        if(idxs[i]):
            f.append(files[i])
        else:
            pass
    files = f
    files.sort()
    data_return = {}
    for file in files:
        shutil.copyfile(file , file[:-4]+'_load.npy')
        data = np.load(file[:-4]+'_load.npy', allow_pickle=True).item()
        for id in data.keys():
            data_return[id] = data[id]
    return data_return

def DispoFunc(dispo):
    if(dispo=='CAND' or dispo=='UNK'):
        return .5
    elif(dispo =='CONF'):
        return 1
    else:
        return 0

def ReduceSetForBatchSize(data,batch_size,set_type):
    #r = len(data) % batch_size
    #data = data.sample(n=int(len(data)-r),random_state = 1)
    #print('Removing {} ids from {} to use batch size of {}'.format(r,set_type,batch_size))
    #data = data.sample(n=int(len(data) - 0), random_state=1)
    return data

def OverSample(CPs ,FPs):
    # oversample
    len_cp =CPs.__len__()
    len_fp = FPs.__len__()
    if(len_cp <=len_fp):
        df_small=CPs
        df_large=FPs
        len_small = df_small.__len__()
        len_large = df_large.__len__()
        ret =1
    else:
        df_small  = FPs
        df_large  = CPs
        len_small = df_small.__len__()
        len_large = df_large.__len__()
        ret = 2
    len_delta = int(len_large - len_small)
    df_small_temp = df_small.sample(n=len_delta , random_state=1 , replace=True)
    df_small_new = pd.concat([df_small,df_small_temp]).sample(frac=1, random_state=1, replace=False)
    assert df_small_new.index.nunique() ==df_small.index.nunique()
    assert df_small_new.__len__() == df_large.__len__()
    if(ret==1):
        # small = CP
        return df_small_new ,df_large
    else:
        # large = CP
        return df_large,df_small_new

def UnderSample(CPs,FPs):
    num_conf = np.floor(CPs.__len__())
    num_false = np.floor(FPs.__len__())
    num_conf = int(num_conf)
    num_false = int(num_false)
    n = min(num_conf, num_false)
    CPs = CPs.sample(n=n, random_state=1)
    FPs = FPs.sample(n=n, random_state=1)
    return CPs,FPs

def ClassImbalance(CPs,FPs,type_sample='Oversample'):

    if(type_sample!='None'):
        if(type_sample=='Oversample'):
            CPs,FPs = OverSample(FPs, CPs)
        elif(type_sample=='Undersample'):
            CPs, FPs = UnderSample(FPs, CPs)
    else:
        pass
    return  CPs,FPs

def GetTrainingAndValidationSets(CPs,FPs,training_size):
    CPs_for_training = CPs.sample(frac=training_size, random_state=1, replace=False)
    FPs_for_training = FPs.sample(frac=training_size, random_state=1, replace=False)
    #

    indexs_in_CP_train = CPs_for_training.index
    indexs_in_FP_train = FPs_for_training.index
    #
    CP_bool_array = ~CPs.index.isin(indexs_in_CP_train)
    FP_bool_array = ~FPs.index.isin(indexs_in_FP_train)
    #
    CPs_for_validation = CPs[CP_bool_array]
    FPs_for_validation = FPs[FP_bool_array]
    return CPs_for_training ,FPs_for_training ,CPs_for_validation ,FPs_for_validation

def GrabDatasets(data,DataObject,training_size,batch_size,Combo = True,type_sample='Oversample',Raw=False):
    df = pd.DataFrame(data).T
    Ids = DataObject.IdData['WrapperId']
    Dispo = DataObject.IdData['WrapperDispo']
    Dispo = pd.DataFrame(Dispo.values, columns=['Disposition'])
    Ids = pd.DataFrame(Ids.values, columns=['ID'])
    df2 = pd.concat([Dispo, Ids], axis=1)
    df2 = df2.set_index('ID')
    df = pd.concat([df, df2], axis=1)
    targets = pd.DataFrame([DispoFunc(x) for x in df.Disposition.values], columns=['Target'], index=df.index)
    df = pd.concat([df, targets], axis=1)
    known_set = df[df['Target'] != .5]
    unk_set = df[df['Target']==.5]
    CPs = known_set[known_set['Target']==1]
    FPs = known_set[known_set['Target']==0]
    CPs_for_training ,FPs_for_training ,CPs_for_validation ,FPs_for_validation =GetTrainingAndValidationSets(CPs, FPs, training_size)
    CPs_for_training, FPs_for_training = ClassImbalance(CPs_for_training, FPs_for_training,type_sample=type_sample)
    train_data = pd.concat([CPs_for_training,FPs_for_training]).sample(frac=1,random_state=1)
    #train_data = ReduceSetForBatchSize(train_data,batch_size,set_type='Training data')
    valid_data = pd.concat([CPs_for_validation, FPs_for_validation]).sample(frac=1, random_state=1)
    #valid_data = ReduceSetForBatchSize(valid_data, batch_size,set_type='Validation data')
    #unk_set = ReduceSetForBatchSize(unk_set, batch_size, set_type='Candidate data')
    if(Raw==False):
        TrainDataset = D.MyDataset(train_data)
        ValidDataset = D.MyDataset(valid_data)
        CandDataset = D.MyDataset(unk_set)
    elif(Raw==True):
        TrainDataset = R.RawDataset(train_data)
        ValidDataset = R.RawDataset(valid_data)
        CandDataset = R.RawDataset(unk_set)
    if(type_sample!='None'):
        weight=torch.tensor([1])
    else:

        weight =  torch.tensor(
            (len(known_set['Target']) - known_set['Target'].sum())
            /len(known_set['Target'])
        )
    if(Combo):
        return train_data ,valid_data ,unk_set ,weight
    else:
        return TrainDataset ,ValidDataset ,CandDataset ,weight

def train_one_epoch(epoch_index, training_loader,optimizer,model,criterion,mission):
    running_loss = 0.
    last_loss = 0.
    batch_size = training_loader.batch_size

    # Here, we use enumerate(training_loader) instead of
    # iter(training_loader) so that we can track the batch
    # index and do some intra-epoch reporting
    mission_flag = 'Raw' in mission  # if true then running new model
    for i, data in enumerate(training_loader):
        # Every data instance is an input + label pair
        inputs, labels,_ = data

        # Zero your gradients for every batch!
        optimizer.zero_grad()

        # Make predictions for this batch

        if(mission_flag):
            p = np.random.permutation(len(_))
            inputs = [inputs[0][p], inputs[1][p], inputs[2][p], inputs[3][p], inputs[4][p], inputs[5][p],
                            inputs[6][p],inputs[7][p]]
            labels = labels[p]
            outputs = model(
                flux     = ReshapeEnsureNotNan(inputs[0], shape=(batch_size, 1, 6000)),              #flux
                centroid = ReshapeEnsureNotNan2D([inputs[1],inputs[2]], shape=(batch_size, 2, 6001)) , #centroid data
                f2_2 = ReshapeEnsureNotNan(inputs[3], shape=(batch_size, 1)),                        #transit depth
                f4_2 = ReshapeEnsureNotNan(inputs[4], shape=(batch_size, 5)),                        #centroid scalars
                f6_2 = ReshapeEnsureNotNan(inputs[5], shape=(batch_size, 4)),                        #secondary Scalars
                f7   = ReshapeEnsureNotNan(inputs[6], shape=(batch_size, 6)),                        #Stellar parameters
                f8   = ReshapeEnsureNotNan(inputs[7], shape=(batch_size, 6))                        #DV Diagnosti                                                            #model Vector
            )
        else:
            p = np.random.permutation(len(_))
            inputs = [inputs[0][p], inputs[1][p], inputs[2][p], inputs[3][p], inputs[4][p], inputs[5][p],
                      inputs[6][p], inputs[7][p],inputs[8][p],inputs[9][p],inputs[10][p],inputs[11][p]]
            labels = labels[p]
            #run ExoMiner
            outputs=model(f1=ReshapeEnsureNotNan(inputs[0], shape=(batch_size, 1, 301)),
                  f2=ReshapeEnsureNotNan(inputs[1], shape=(batch_size, 1, 31)),
                  f2_2=ReshapeEnsureNotNan(inputs[2], shape=(batch_size, 1)),
                  f3=ReshapeEnsureNotNan(inputs[3], shape=(batch_size, 1, 301)),
                  f4=ReshapeEnsureNotNan(inputs[4], shape=(batch_size, 1, 31)),
                  f4_2=ReshapeEnsureNotNan(inputs[5], shape=(batch_size, 5)),
                  f5_odd=ReshapeEnsureNotNan(inputs[6], shape=(batch_size, 1, 31)),
                  f5_even=ReshapeEnsureNotNan(inputs[7], shape=(batch_size, 1, 31)),
                  f6=ReshapeEnsureNotNan(inputs[8], shape=(batch_size, 1, 31)),
                  f6_2=ReshapeEnsureNotNan(inputs[9], shape=(batch_size, 4)),
                  f7=ReshapeEnsureNotNan(inputs[10], shape=(batch_size, 6)),
                  f8=ReshapeEnsureNotNan(inputs[11], shape=(batch_size, 6)))

        # Compute the loss and its gradients
        labels = [torch.tensor([l]) for l in labels]
        loss = criterion(outputs, torch.tensor([labels]).reshape(outputs.shape).float())
        loss.backward()

        # Adjust learning weights
        optimizer.step()

        # Gather data and report
        running_loss += loss.item()

    return running_loss/len(training_loader.sampler)

def TrainAndValidate(NumEpochs,training_loader,validation_loader,optimizer,model,criterion,mission):
    epoch_number = 0
    batch_size = training_loader.batch_size
    TrainingLoss, ValidationLoss = [],[]
    writer = SummaryWriter()
    mission_flag = 'Raw' in mission # if true then running new model
    if(mission_flag):
        best_vloss =.61
    else:
        best_vloss =.47
    for epoch in range(NumEpochs):
        #print('EPOCH {}:'.format(epoch_number + 1))

        # Make sure gradient tracking is on, and do a pass over the data
        model.train(True)
        training_loss = train_one_epoch(epoch_number,training_loader,optimizer,model,criterion,mission)

        # We don't need gradients on to do reporting
        model.train(False)

        running_vloss = 0.0
        vdata_arr = []
        vlabels_arr = []
        model.eval()
        with torch.no_grad():
            for i, vdata in enumerate(validation_loader):
                vinputs, vlabels,_ = vdata
                if(mission_flag):
                    #run our model

                    voutputs = model(
                        flux=ReshapeEnsureNotNan(vinputs[0], shape=(batch_size, 1, 6000)),  # flux
                        centroid=ReshapeEnsureNotNan2D([vinputs[1], vinputs[2]], shape=(batch_size, 2, 6001)),
                        # centroid data
                        f2_2=ReshapeEnsureNotNan(vinputs[3], shape=(batch_size, 1)),  # transit depth
                        f4_2=ReshapeEnsureNotNan(vinputs[4], shape=(batch_size, 5)),  # centroid scalars
                        f6_2=ReshapeEnsureNotNan(vinputs[5], shape=(batch_size, 4)),  # secondary Scalars
                        f7=ReshapeEnsureNotNan(vinputs[6], shape=(batch_size, 6)),  # Stellar parameters
                        f8=ReshapeEnsureNotNan(vinputs[7], shape=(batch_size, 6))  # DV Diagnostics
                          # model Vector
                    )
                else:
                    #run exominer
                    voutputs = model(f1=ReshapeEnsureNotNan(vinputs[0], shape=(batch_size, 1, 301)),
                          f2=ReshapeEnsureNotNan(vinputs[1], shape=(batch_size, 1, 31)),
                          f2_2=ReshapeEnsureNotNan(vinputs[2], shape=(batch_size, 1)),
                          f3=ReshapeEnsureNotNan(vinputs[3], shape=(batch_size, 1, 301)),
                          f4=ReshapeEnsureNotNan(vinputs[4], shape=(batch_size, 1, 31)),
                          f4_2=ReshapeEnsureNotNan(vinputs[5], shape=(batch_size, 5)),
                          f5_odd=ReshapeEnsureNotNan(vinputs[6], shape=(batch_size, 1, 31)),
                          f5_even=ReshapeEnsureNotNan(vinputs[7], shape=(batch_size, 1, 31)),
                          f6=ReshapeEnsureNotNan(vinputs[8], shape=(batch_size, 1, 31)),
                          f6_2=ReshapeEnsureNotNan(vinputs[9], shape=(batch_size, 4)),
                          f7=ReshapeEnsureNotNan(vinputs[10], shape=(batch_size, 6)),
                          f8=ReshapeEnsureNotNan(vinputs[11], shape=(batch_size, 6)))

                vlabels = [torch.tensor([l]) for l in vlabels]
                vloss = criterion(voutputs, torch.tensor([vlabels]).reshape(voutputs.shape).float())
                running_vloss += vloss
                vdata_arr.append([voutputs.detach().numpy()])
                vlabels_arr.append([vlabels])
            vdata_arr= torch.as_tensor(vdata_arr).reshape(torch.as_tensor(vdata_arr).shape[0:3])
            vlabels_arr = torch.as_tensor(vlabels_arr).reshape(vdata_arr.shape)
            # plot epochs used for testing
            plot_epochs = [10,20,50,75,100]
            validation_loss = running_vloss / len(validation_loader.sampler)
            print('Epoch {}: LOSS train {} valid {}'.format(epoch_number + 1, training_loss, validation_loss))
            TrainingLoss.append(training_loss)
            ValidationLoss.append(validation_loss.detach().item())
            writer.add_scalars(f'Loss/LossPlots_{batch_size}_{optimizer.param_groups[0]["lr"]}_{NumEpochs}', {
                'train': training_loss,
                'validation': validation_loss }
                               ,epoch_number)

                # p_val_dict = {}
                # a_val_dict = {}
                # r_val_dict = {}
                # fpr_val_dict ={}
                # for t in np.arange(0,1,.01):
                #     percision = BinaryPrecision(threshold=t)
                #     recall =BinaryRecall(threshold=t)
                #     accuracy =BinaryAccuracy(threshold=t)
                #     stats=  BinaryStatScores(threshold=t)
                #     TP, FP, TN, FN, _ = stats(vdata_arr, vlabels_arr)
                #     fpr_val_dict[t] = FP / (FP + TN)
                #     p_val_dict[t]  = percision(vdata_arr, vlabels_arr)
                #     r_val_dict[t]  = recall(vdata_arr, vlabels_arr)
                #     a_val_dict[t]  = accuracy(vdata_arr, vlabels_arr)

                # PlotAccuracyThreshold(mission, a_val_dict, epoch+1, optimizer.param_groups[0]['lr'], batch_size)
                # PlotFprRecal(mission, fpr_val_dict, r_val_dict,epoch+1,optimizer.param_groups[0]['lr'],batch_size)
                # PlotPercisionRecal(mission,p_val_dict,r_val_dict,epoch+1,optimizer.param_groups[0]['lr'],batch_size)
                # SaveStats(mission, a_val_dict, p_val_dict, r_val_dict, fpr_val_dict,epoch+1,optimizer.param_groups[0]['lr'],batch_size)
                # LossCurve(mission, TrainingLoss, ValidationLoss, epoch+1, optimizer.param_groups[0]['lr'], batch_size)

            # Track best performance, and save the model's state
            if validation_loss < best_vloss:
                best_vloss = validation_loss
                model_path = 'ModelCheckpoints/'+mission+'/_bf_model_{}_batch_size_{}_lr_{}'.format(epoch_number,batch_size,optimizer.param_groups[0]['lr'])
                torch.save(model.state_dict(), model_path)
                plot_data_set = validation_loader.dataset
                Plot_loader = DataLoader(plot_data_set, batch_size=1)
                GenPercisionPlots(model, Plot_loader, mission, optimizer, epoch_number + 1, fp='Predictions/')
                LossCurve(mission, TrainingLoss, ValidationLoss, epoch_number + 1, optimizer.param_groups[0]['lr'], batch_size)
            epoch_number += 1
    LossCurve(mission , TrainingLoss, ValidationLoss,NumEpochs,optimizer.param_groups[0]['lr'],batch_size)
    writer.flush()

    return model

def PlotPercisionRecal(mission,Percision,Recall,EpochNum,lr,batch_size,fp = 'Figures/' ):
    p_temp = []
    r_temp = []
    for ci in Percision.keys():
        p_temp.append(Percision[ci].item())
        r_temp.append(Recall[ci].item())
    plt.plot(r_temp,p_temp)
    v = Recall[.5]
    v2 = Percision[.5]
    plt.scatter(Recall[.5],Percision[.5],label=f'R(t=.5)={v:.2f},P(t=.5)={v2:.2f}',c='r')
    plt.xlabel('Recall')
    plt.ylabel('Percision')
    plt.title('Precision vs Recall Plot')
    plt.legend()
    plt.savefig(fp +mission + f'/Epoch_{EpochNum}_lr_{lr}_batchsize_{batch_size}_RecallVsPercision.png')
    plt.close()
    return None

def PlotFprRecal(mission,Fpr,Recall,EpochNum,lr,batch_size,fp = 'Figures/'):
    fpr_temp = []
    r_temp = []
    for ci in Fpr.keys():
        fpr_temp.append(Fpr[ci].item() )
        r_temp.append(Recall[ci].item())
    plt.plot(fpr_temp,r_temp,label='Recall')

    x = [0,1]
    y= [0,1]
    plt.plot(x,y, c='g', ls='--', label='No Skill')
    v = Fpr[.5]
    v2 = Recall[.5]
    plt.scatter( Fpr[.5],Recall[.5], label=f'FPR(t=.5)={v:.2f},R(t=.5)={v2:.2f}', c='r')
    plt.ylabel('Recall')
    plt.xlabel('False Positive Rate')
    plt.legend()
    plt.savefig(fp +mission + f'/Epoch_{EpochNum}_lr_{lr}_batchsize_{batch_size}_RecallVsFPR.png')
    plt.close()
    return None

def PlotAccuracyThreshold(mission,Acc,EpochNum,lr,batch_size,fp = 'Figures/'):
    acc_temp = []
    for ci in Acc.keys():
        acc_temp.append(Acc[ci].item() )
    plt.plot(Acc.keys(),acc_temp,label = 'Accuracy')
    v = Acc[.5]
    plt.scatter(.5,Acc[.5],label=f't=.5, Accuracy={v:.2f}',c='r')
    plt.xlabel('Threshold')
    plt.ylabel('Accuracy')
    plt.title(mission + f' Accuracy vs Threshold lr={lr},batch_size={batch_size}')
    plt.legend()
    plt.savefig(fp +mission + f'/Epoch_{EpochNum}_lr_{lr}_batchsize_{batch_size}_AccVsThreshold.png')
    plt.close()
    return None

def PlotAccuracyEpoch(mission,Acc,Epochs,t,lr,batch_size,fp = 'Figures/'):
    plt.plot(Epochs,Acc,label='Accuracy')
    plt.xlabel('Epochs')
    plt.ylabel('Accuracy at {}'.format(t))
    plt.legend()
    plt.savefig(fp +mission + f'/All_Epochs_Accuracy_Threshold_{t}_lr_{lr}_batchsize_{batch_size}.png')
    plt.close()
    return None

def LossCurve(mission,TrainingLoss,ValidationLoss,EpochNum,lr,batch_size,fp = 'Figures/'):
    plt.plot(TrainingLoss, label='Training Loss')
    plt.plot(ValidationLoss, label='Validation Loss')
    plt.legend()
    plt.savefig(fp +mission+ f'/Epoch_{EpochNum}_lr_{lr}_batchsize_{batch_size}_ExoMinerTrainAndValidLossPlot.png')
    plt.close()
    return None

def SaveStats(mission ,Accuracy,Percision,Recall,FPR,F1,AUROC, EpochNum,lr,batch_size,fp='Stats/'):
    file_str = fp + mission + f'/Epoch_{EpochNum}_lr_{lr}_batchsize_{batch_size}_ExoMinerResults.txt'
    File = open(file_str, 'w')
    for t in Accuracy.keys():
        ExoMinerdata = {'Threshold': t,'Accuracy': Accuracy[t].item(), 'Percision': Percision[t].item(), 'Recall': Recall[t].item(),
                        'False Positive Rate':FPR[t].item() , 'F1 Score': F1[t].item()}

        File.write(str(ExoMinerdata))
        File.write('\n')
    File.write(f'AUROC Val:{AUROC}')
    File.close()
    return None

def SavePatK(mission,PatK ,EpochNum,lr,batch_size,fp='Stats/PatK/'):
    file_str = fp + mission + f'/Epoch_{EpochNum}_lr_{lr}_batchsize_{batch_size}_PatKResults.txt'
    File = open(file_str, 'w')
    for key in PatK .keys():
        PatKdata = {'k= '+str(key) :  'val='+str(PatK[key])}
        File.write(str(PatKdata))
        File.write('\n')
    File.close()
    return None

def GenPercisionPlots(model,Plot_loader,mission,optimizer, EpochNum ,fp='Predictions/'):
    batch_size = Plot_loader.batch_size
    outputs_arr = []
    targets = []
    mission_flag = 'Raw' in mission
    with torch.no_grad():
        for i, cdata in enumerate(Plot_loader):
            vinputs, clabels,ids = cdata
            if (mission_flag):
                # run our model

                coutputs = model(
                    flux=ReshapeEnsureNotNan(vinputs[0], shape=(batch_size, 1, 6000)),  # flux
                    centroid=ReshapeEnsureNotNan2D([vinputs[1], vinputs[2]], shape=(batch_size, 2, 6001)),
                    # centroid data
                    f2_2=ReshapeEnsureNotNan(vinputs[3], shape=(batch_size, 1)),  # transit depth
                    f4_2=ReshapeEnsureNotNan(vinputs[4], shape=(batch_size, 5)),  # centroid scalars
                    f6_2=ReshapeEnsureNotNan(vinputs[5], shape=(batch_size, 4)),  # secondary Scalars
                    f7=ReshapeEnsureNotNan(vinputs[6], shape=(batch_size, 6)),  # Stellar parameters
                    f8=ReshapeEnsureNotNan(vinputs[7], shape=(batch_size, 6))  # DV Diagnostics
                    # model Vector
                )
            else:
                # run exominer
                coutputs = model(f1=ReshapeEnsureNotNan(vinputs[0], shape=(batch_size, 1, 301)),
                                 f2=ReshapeEnsureNotNan(vinputs[1], shape=(batch_size, 1, 31)),
                                 f2_2=ReshapeEnsureNotNan(vinputs[2], shape=(batch_size, 1)),
                                 f3=ReshapeEnsureNotNan(vinputs[3], shape=(batch_size, 1, 301)),
                                 f4=ReshapeEnsureNotNan(vinputs[4], shape=(batch_size, 1, 31)),
                                 f4_2=ReshapeEnsureNotNan(vinputs[5], shape=(batch_size, 5)),
                                 f5_odd=ReshapeEnsureNotNan(vinputs[6], shape=(batch_size, 1, 31)),
                                 f5_even=ReshapeEnsureNotNan(vinputs[7], shape=(batch_size, 1, 31)),
                                 f6=ReshapeEnsureNotNan(vinputs[8], shape=(batch_size, 1, 31)),
                                 f6_2=ReshapeEnsureNotNan(vinputs[9], shape=(batch_size, 4)),
                                 f7=ReshapeEnsureNotNan(vinputs[10], shape=(batch_size, 6)),
                                 f8=ReshapeEnsureNotNan(vinputs[11], shape=(batch_size, 6)))
            out = coutputs.detach().numpy()
            target = clabels.detach().numpy()
            out = np.reshape(out, batch_size)
            target = np.reshape(target , batch_size)
            outputs_arr.append(out[0])
            targets.append(target[0])
    p_val_dict = {}
    a_val_dict = {}
    r_val_dict = {}
    fpr_val_dict = {}
    f1_score_dict ={}
    Patk = {50:0,100:0,200:0,1000:0,2200:0}

    for t in np.arange(0, 1, .01):
        percision = BinaryPrecision(threshold=t)
        recall = BinaryRecall(threshold=t)
        accuracy = BinaryAccuracy(threshold=t)
        stats = BinaryStatScores(threshold=t)
        f1 = BinaryF1Score(threshold=t)
        TP, FP, TN, FN, _ = stats(torch.tensor(outputs_arr), torch.tensor(targets))

        fpr_val_dict[t] = FP / (FP + TN)
        f1_score_dict[t] = f1(torch.tensor(outputs_arr), torch.tensor(targets))
        p_val_dict[t] = percision(torch.tensor(outputs_arr), torch.tensor(targets))
        r_val_dict[t] = recall(torch.tensor(outputs_arr), torch.tensor(targets))
        a_val_dict[t] = accuracy(torch.tensor(outputs_arr), torch.tensor(targets))

        if (t == .5):
            for k in Patk.keys():
                PatK = Precision(task ='binary' , threshold=.5, top_k=k)
                Patk[k] = PatK(torch.tensor(outputs_arr), torch.tensor(targets))
            SavePatK(mission, Patk, EpochNum, optimizer.param_groups[0]['lr'], batch_size)
    auroc = BinaryAUROC(thresholds=None)
    auroc_val = auroc(torch.tensor(outputs_arr), torch.tensor(targets))

    SaveStats(mission, a_val_dict, p_val_dict, r_val_dict, fpr_val_dict,f1_score_dict,auroc_val, EpochNum, optimizer.param_groups[0]['lr'], batch_size, fp='Stats/')
    PlotPrecisionThreshold(mission, p_val_dict, EpochNum , optimizer.param_groups[0]['lr'], batch_size)
    PlotF1Score(mission, f1_score_dict, EpochNum, optimizer.param_groups[0]['lr'], batch_size)
    PlotRecallThreshold(mission, r_val_dict, EpochNum, optimizer.param_groups[0]['lr'], batch_size)
    PlotAccuracyThreshold(mission, a_val_dict, EpochNum , optimizer.param_groups[0]['lr'], batch_size)
    PlotFprRecal(mission, fpr_val_dict, r_val_dict, EpochNum, optimizer.param_groups[0]['lr'], batch_size)
    PlotPercisionRecal(mission=mission, Percision=p_val_dict, Recall=r_val_dict, EpochNum=EpochNum, lr=optimizer.param_groups[0]['lr'], batch_size=batch_size)
    return None

def PlotF1Score(mission,f1_dict,EpochNum,lr,batch_size,fp = 'Figures/'):
    f1_temp = []
    for ci in f1_dict.keys():
        f1_temp.append(f1_dict[ci].item())
    plt.plot(f1_dict.keys(), f1_temp, label='F1 Score')
    v = f1_dict[.5]
    plt.scatter(.5,f1_dict[.5],label=f't=.5, F1={v:.2f}',c='r')
    plt.xlabel('Threshold')
    plt.ylabel('Precision')
    plt.title(mission + f' Precision vs Threshold lr={lr},batch_size={batch_size}')
    plt.legend()
    plt.savefig(fp + mission + f'/Epoch_{EpochNum}_lr_{lr}_batchsize_{batch_size}_F1ScoreVsThreshold.png')
    plt.close()

def PlotPrecisionThreshold(mission,Precision,EpochNum,lr,batch_size,fp = 'Figures/'):
    Precision_temp = []
    for ci in Precision.keys():
        Precision_temp.append(Precision[ci].item() )

    plt.plot(Precision.keys(),Precision_temp,label = 'Precision')
    v =Precision[.5]
    plt.scatter(.5,Precision[.5],label=f't=.5, Precision={v:.2f}',c='r')
    plt.xlabel('Threshold')
    plt.ylabel('Precision')
    plt.title(mission + f' Precision vs Threshold lr={lr},batch_size={batch_size}')
    plt.legend()
    plt.savefig(fp +mission + f'/Epoch_{EpochNum}_lr_{lr}_batchsize_{batch_size}_PrecisionVsThreshold.png')
    plt.close()
    return None

def PlotRecallThreshold(mission,Recall,EpochNum,lr,batch_size,fp = 'Figures/'):
    recall_temp = []
    for ci in Recall.keys():
        recall_temp.append(Recall[ci].item() )
    plt.plot(Recall.keys(),recall_temp,label = 'Recall')
    v = Recall[.5]
    plt.scatter(.5,Recall[.5],label=f't=.5, Recall={v:.2f}',c='r')
    plt.xlabel('Threshold')
    plt.ylabel('Recall')
    plt.title(mission + f' Recall vs Threshold lr={lr},batch_size={batch_size}')
    plt.legend()
    plt.savefig(fp +mission + f'/Epoch_{EpochNum}_lr_{lr}_batchsize_{batch_size}_RecallVsThreshold.png')
    plt.close()
    return None

def UnpackRawData(mission, fp='RawData/'):
    files = glob.glob('./' + fp + mission + r'/dataset_*.npy')
    idxs = ['load' not in file for file in files]
    f = []
    for i in range(0, len(idxs)):
        if (idxs[i]):
            f.append(files[i])
        else:
            pass
    files = f
    files.sort()
    data_return = {}
    for file in files:
        shutil.copyfile(file, file[:-4] + '_load.npy')
        shutil.copyfile('./RawData/'+mission+'/'+mission+'Scalars.npy', './RawData/'+mission+'/'+mission+'Scalars' + '_load.npy')
        data = np.load(file[:-4]+'_load.npy', allow_pickle=True).item()
        data2= np.load('./RawData/'+mission+'/'+mission+'Scalars' + '_load.npy',allow_pickle=True).item()
        for id in data.keys():
            data_return[id] = data[id]
            temp = data2[id]
            d = {}
            for key in list(temp.keys()):
                d[key] = temp[key]
            m = [1 if mission=='Kepler'  else 0] # mission feature to determine Kepler or TESS
            centroid_col = data_return[id]['Centroid Col']
            centroid_row = data_return[id]['Centroid Row']
            centroid_col = np.append(centroid_col,m)
            centroid_row = np.append(centroid_row,m)
            data_return[id]['Centroid Col'] =centroid_col
            data_return[id]['Centroid Row'] =centroid_row
            data_return[id].update(d)
    return data_return

def RunModelOnCandidates(mission,CandLoader,model,threshold =.5,fp='Predictions/'):
    batch_size = CandLoader.batch_size
    df = pd.DataFrame()
    mission_flag = 'Raw' in mission
    with torch.no_grad():
        for i, cdata in enumerate(CandLoader):
            vinputs, clabels,ids = cdata
            if (mission_flag):
                # run our model

                coutputs = model(
                    flux=ReshapeEnsureNotNan(vinputs[0], shape=(batch_size, 1, 6000)),  # flux
                    centroid=ReshapeEnsureNotNan2D([vinputs[1], vinputs[2]], shape=(batch_size, 2, 6001)),
                    # centroid data
                    f2_2=ReshapeEnsureNotNan(vinputs[3], shape=(batch_size, 1)),  # transit depth
                    f4_2=ReshapeEnsureNotNan(vinputs[4], shape=(batch_size, 5)),  # centroid scalars
                    f6_2=ReshapeEnsureNotNan(vinputs[5], shape=(batch_size, 4)),  # secondary Scalars
                    f7=ReshapeEnsureNotNan(vinputs[6], shape=(batch_size, 6)),  # Stellar parameters
                    f8=ReshapeEnsureNotNan(vinputs[7], shape=(batch_size, 6))  # DV Diagnostics
                    # model Vector
                )
            else:
                # run exominer
                coutputs = model(f1=ReshapeEnsureNotNan(vinputs[0], shape=(batch_size, 1, 301)),
                                 f2=ReshapeEnsureNotNan(vinputs[1], shape=(batch_size, 1, 31)),
                                 f2_2=ReshapeEnsureNotNan(vinputs[2], shape=(batch_size, 1)),
                                 f3=ReshapeEnsureNotNan(vinputs[3], shape=(batch_size, 1, 301)),
                                 f4=ReshapeEnsureNotNan(vinputs[4], shape=(batch_size, 1, 31)),
                                 f4_2=ReshapeEnsureNotNan(vinputs[5], shape=(batch_size, 5)),
                                 f5_odd=ReshapeEnsureNotNan(vinputs[6], shape=(batch_size, 1, 31)),
                                 f5_even=ReshapeEnsureNotNan(vinputs[7], shape=(batch_size, 1, 31)),
                                 f6=ReshapeEnsureNotNan(vinputs[8], shape=(batch_size, 1, 31)),
                                 f6_2=ReshapeEnsureNotNan(vinputs[9], shape=(batch_size, 4)),
                                 f7=ReshapeEnsureNotNan(vinputs[10], shape=(batch_size, 6)),
                                 f8=ReshapeEnsureNotNan(vinputs[11], shape=(batch_size, 6)))
            out = coutputs.detach().numpy()
            out = np.reshape(out, batch_size)
            ids = np.array(ids)
            dispo = ['Conf' if o>=threshold else 'unk' for o in out]
            mission_arr=[mission]*len(ids)
            df_temp = pd.DataFrame([mission_arr,out,dispo],index=['Mission','Score','Disposition'],columns=ids)
            df = pd.concat([df,df_temp.T])
    df.to_csv(fp+mission + '/'+'Predictions_threshold_{}_.csv'.format(threshold))
    return None

def LoadBestModel(batch_size,lr,mission,fp = 'ModelCheckpoints/'):
    files = glob.glob('./' + fp  + mission + r'/model_*_batch_size_{}_lr_{}*'.format(batch_size,lr))
    files.sort()
    checkpoint = torch.load(files[-1])
    return checkpoint

def LoadBestFitExoMiner(mission='Combo',fp = 'ModelCheckpoints/'):
    files = glob.glob('./' + fp  + mission + r'/*bf*')
    files.sort()
    checkpoint = torch.load(files[-1])
    return checkpoint

def UseExoMinerWeights(ExoMinerRawModel,ExoMiner):
    ExoMinerRawModel.f1_conv1.weight.data = ExoMiner['f1_conv1.weight']
    ExoMinerRawModel.f1_conv1.weight.requires_grad_(True)
    ExoMinerRawModel.f1_conv1.bias.data =ExoMiner['f1_conv1.bias']

    ###########
    ExoMinerRawModel.f1_conv2.weight.data =ExoMiner['f1_conv2.weight']
    ExoMinerRawModel.f1_conv2.weight.requires_grad_(True)
    ExoMinerRawModel.f1_conv2.bias.data =ExoMiner['f1_conv2.bias']

    ###########
    ExoMinerRawModel.f1_conv3.weight.data =ExoMiner['f1_conv3.weight']
    ExoMinerRawModel.f1_conv3.weight.requires_grad_(True)
    ExoMinerRawModel.f1_conv3.bias.data =ExoMiner['f1_conv3.bias']

    ###########
    ExoMinerRawModel.f1_lin1.weight.data =ExoMiner['f1_lin1.weight']
    ExoMinerRawModel.f1_lin1.weight.requires_grad_(True)
    ExoMinerRawModel.f1_lin1.bias.data =ExoMiner['f1_lin1.bias']

    #####
    ExoMinerRawModel.f2_conv1.weight.data = ExoMiner['f2_conv1.weight']
    ExoMinerRawModel.f2_conv1.weight.requires_grad_(True)
    ExoMinerRawModel.f2_conv1.bias.data =ExoMiner['f2_conv1.bias']

    ###########
    ExoMinerRawModel.f2_conv2.weight.data =ExoMiner['f2_conv2.weight']
    ExoMinerRawModel.f2_conv2.weight.requires_grad_(True)
    ExoMinerRawModel.f2_conv2.bias.data =ExoMiner['f2_conv2.bias']

    ###########

    ###########
    ExoMinerRawModel.f2_lin1.weight.data =ExoMiner['f2_lin1.weight']
    ExoMinerRawModel.f2_lin1.weight.requires_grad_(True)
    ExoMinerRawModel.f2_lin1.bias.data =ExoMiner['f2_lin1.bias']

    #####
    ExoMinerRawModel.f3_conv1.weight.data = ExoMiner['f3_conv1.weight']
    ExoMinerRawModel.f3_conv1.weight.requires_grad_(True)
    ExoMinerRawModel.f3_conv1.bias.data = ExoMiner['f3_conv1.bias']

    ###########
    ExoMinerRawModel.f3_conv2.weight.data = ExoMiner['f3_conv2.weight']
    ExoMinerRawModel.f3_conv2.weight.requires_grad_(True)
    ExoMinerRawModel.f3_conv2.bias.data = ExoMiner['f3_conv2.bias']

    ###########
    ExoMinerRawModel.f3_conv3.weight.data = ExoMiner['f3_conv3.weight']
    ExoMinerRawModel.f3_conv3.weight.requires_grad_(True)
    ExoMinerRawModel.f3_conv3.bias.data = ExoMiner['f3_conv3.bias']

    ###########
    ExoMinerRawModel.f3_lin1.weight.data = ExoMiner['f3_lin1.weight']
    ExoMinerRawModel.f3_lin1.weight.requires_grad_(True)
    ExoMinerRawModel.f3_lin1.bias.data = ExoMiner['f3_lin1.bias']

    #####
    ExoMinerRawModel.f4_conv1.weight.data = ExoMiner['f4_conv1.weight']
    ExoMinerRawModel.f4_conv1.weight.requires_grad_(True)
    ExoMinerRawModel.f4_conv1.bias.data = ExoMiner['f4_conv1.bias']

    ExoMinerRawModel.f4_conv2.weight.data = ExoMiner['f4_conv2.weight']
    ExoMinerRawModel.f4_conv2.weight.requires_grad_(True)
    ExoMinerRawModel.f4_conv2.bias.data = ExoMiner['f4_conv2.bias']

    ###########
    ExoMinerRawModel.f4_lin1.weight.data = ExoMiner['f4_lin1.weight']
    ExoMinerRawModel.f4_lin1.weight.requires_grad_(True)
    ExoMinerRawModel.f4_lin1.bias.data = ExoMiner['f4_lin1.bias']

    #####
    ExoMinerRawModel.f5_conv1.weight.data = ExoMiner['f5_conv1.weight']
    ExoMinerRawModel.f5_conv1.weight.requires_grad_(True)
    ExoMinerRawModel.f5_conv1.bias.data = ExoMiner['f5_conv1.bias']

    ###########
    ExoMinerRawModel.f5_conv2.weight.data = ExoMiner['f5_conv2.weight']
    ExoMinerRawModel.f5_conv2.weight.requires_grad_(True)
    ExoMinerRawModel.f5_conv2.bias.data = ExoMiner['f5_conv2.bias']

    ###########
    ExoMinerRawModel.f5_lin1.weight.data = ExoMiner['f5_lin1.weight']
    ExoMinerRawModel.f5_lin1.weight.requires_grad_(True)
    ExoMinerRawModel.f5_lin1.bias.data = ExoMiner['f5_lin1.bias']

    #####
    ExoMinerRawModel.f6_conv1.weight.data = ExoMiner['f6_conv1.weight']
    ExoMinerRawModel.f6_conv1.weight.requires_grad_(True)
    ExoMinerRawModel.f6_conv1.bias.data = ExoMiner['f6_conv1.bias']

    ###########
    ExoMinerRawModel.f6_conv2.weight.data = ExoMiner['f6_conv2.weight']
    ExoMinerRawModel.f6_conv2.weight.requires_grad_(True)
    ExoMinerRawModel.f6_conv2.bias.data = ExoMiner['f6_conv2.bias']

    ###########
    ExoMinerRawModel.f6_lin1.weight.data = ExoMiner['f6_lin1.weight']
    ExoMinerRawModel.f6_lin1.weight.requires_grad_(True)
    ExoMinerRawModel.f6_lin1.bias.data = ExoMiner['f6_lin1.bias']

    ####
    ExoMinerRawModel.FinalLin1.weight.data = ExoMiner['FinalLin1.weight']
    ExoMinerRawModel.FinalLin2.weight.data = ExoMiner['FinalLin2.weight']
    ExoMinerRawModel.FinalLin3.weight.data = ExoMiner['FinalLin3.weight']
    ExoMinerRawModel.FinalLin4.weight.data = ExoMiner['FinalLin4.weight']
    ExoMinerRawModel.FinalLin1.weight.requires_grad_(True)
    ExoMinerRawModel.FinalLin2.weight.requires_grad_(True)
    ExoMinerRawModel.FinalLin3.weight.requires_grad_(True)
    ExoMinerRawModel.FinalLin4.weight.requires_grad_(True)
    ExoMinerRawModel.FinalLin1.bias.data = ExoMiner['FinalLin1.bias']
    ExoMinerRawModel.FinalLin2.bias.data = ExoMiner['FinalLin2.bias']
    ExoMinerRawModel.FinalLin3.bias.data = ExoMiner['FinalLin3.bias']
    ExoMinerRawModel.FinalLin4.bias.data = ExoMiner['FinalLin4.bias']

    return ExoMinerRawModel

def RunExoMiner(mission,lr,batch_size,training_size,epochs,type_sample='Oversample',load=True):  # mission
    torch.manual_seed(1)
    np.random.seed(1)
    print('Running ' + mission)
    if(mission=='TESS' or mission=='Kepler'):
        DataObject = gd.GatherData(mission=mission)
        data = UnPackData(mission)
        DataObject.IdData = DataObject.IdData[DataObject.IdData.WrapperId.isin(data.keys())]
        TrainDataset, ValidDataset, CandDataset, weight = GrabDatasets(data, DataObject, training_size, batch_size,Combo = False,type_sample=type_sample)
        model = ExoMiner.ExoMiner()
    elif(mission=='Combo'):
        DataObjectTESS = gd.GatherData(mission='TESS')
        dataTESS = UnPackData('TESS')
        DataObjectKepler = gd.GatherData(mission='Kepler')
        dataKepler = UnPackData('Kepler')
        DataObjectTESS.IdData = DataObjectTESS.IdData[DataObjectTESS.IdData.WrapperId.isin(dataTESS.keys())]
        DataObjectKepler.IdData = DataObjectKepler.IdData[DataObjectKepler.IdData.WrapperId.isin(dataKepler.keys())]
        TrainDatasetTESS, ValidDatasetTESS, CandDatasetTESS, weight = GrabDatasets(dataTESS, DataObjectTESS, training_size, batch_size,type_sample=type_sample)
        TrainDatasetKepler, ValidDatasetKepler, CandDatasetKepler, weight = GrabDatasets(dataKepler, DataObjectKepler, training_size, batch_size,type_sample=type_sample)
        train_data =pd.concat([TrainDatasetTESS,TrainDatasetKepler])
        valid_data =pd.concat([ValidDatasetTESS,ValidDatasetKepler])
        unk_data  =pd.concat([CandDatasetTESS,CandDatasetKepler])
        TrainDataset = D.MyDataset(train_data)
        ValidDataset = D.MyDataset(valid_data)
        CandDataset = D.MyDataset(unk_data)
        model = ExoMiner.ExoMiner()
    elif(mission=='ComboRaw'):
        DataObjectTESS = gd.GatherData(mission='TESS')
        dataTESS = UnpackRawData('TESS')
        DataObjectKepler = gd.GatherData(mission='Kepler')
        dataKepler = UnpackRawData('Kepler')
        DataObjectTESS.IdData = DataObjectTESS.IdData[DataObjectTESS.IdData.WrapperId.isin(dataTESS.keys())]
        DataObjectKepler.IdData = DataObjectKepler.IdData[DataObjectKepler.IdData.WrapperId.isin(dataKepler.keys())]
        TraindfTESS, ValiddfTESS, CanddfTESS, weight = GrabDatasets(dataTESS, DataObjectTESS,
                                                                                training_size, batch_size,
                                                                                type_sample=type_sample,Raw =True)
        TraindfKepler, ValiddfKepler, CanddfKepler, weight = GrabDatasets(dataKepler, DataObjectKepler,
                                                                                training_size, batch_size,
                                                                                type_sample=type_sample,Raw =True)
        train_data = pd.concat([TraindfTESS, TraindfKepler])
        valid_data = pd.concat([ValiddfTESS, ValiddfKepler])
        unk_data = pd.concat([CanddfTESS, CanddfKepler])
        TrainDataset = R.RawDataset(train_data)
        ValidDataset = R.RawDataset(valid_data)
        CandDataset = R.RawDataset(unk_data)
        model = ExoMinerRaw.ExoMinerRaw()

    elif (mission == 'TESSRaw'):
        DataObject = gd.GatherData(mission='TESS')
        data = UnpackRawData('TESS')
        DataObject.IdData = DataObject.IdData[DataObject.IdData.WrapperId.isin(data.keys())]
        TrainDataset, ValidDataset, CandDataset, weight = GrabDatasets(data, DataObject, training_size, batch_size,
                                                                       Combo=False, type_sample=type_sample,Raw =True)
        model = ExoMinerRaw.ExoMinerRaw()
    elif (mission == 'KeplerRaw'):
        print('Running '+mission)
        DataObject = gd.GatherData(mission='Kepler')
        data = UnpackRawData('Kepler')
        DataObject.IdData = DataObject.IdData[DataObject.IdData.WrapperId.isin(data.keys())]
        TrainDataset, ValidDataset, CandDataset, weight = GrabDatasets(data, DataObject, training_size, batch_size,
                                                                       Combo=False, type_sample=type_sample,Raw =True)
        model = ExoMinerRaw.ExoMinerRaw()
    if('Raw' in mission and load==True):
        checkpoint =LoadBestFitExoMiner()
        model = UseExoMinerWeights(ExoMinerRawModel=model, ExoMiner=checkpoint)

    TrainLoader = DataLoader(TrainDataset,batch_size=batch_size,drop_last=True)
    ValidLoader = DataLoader(ValidDataset,batch_size=batch_size,drop_last=True)
    #CandLoader = DataLoader(CandDataset,batch_size=batch_size,drop_last=True)
    #PlotLoader = DataLoader(ValidDataset, batch_size=1)

    criterion = nn.BCELoss(weight = weight, size_average=False)
    optimizer = optim.Adam(model.parameters(), lr=lr)
    model = TrainAndValidate(NumEpochs=epochs,
                     training_loader=TrainLoader,
                     validation_loader=ValidLoader,
                     optimizer=optimizer,
                     model=model,
                     criterion=criterion,
                     mission=mission)
    return None

def ModelChoice(model,mission,lr,training_size,batch_size,epochs,type_sample):
    if (model == "ExoMiner"):

        RunExoMiner(mission=mission,
                    lr=lr,
                    training_size=training_size,
                    batch_size=batch_size,
                    epochs=epochs,
                    type_sample=type_sample)

    elif(model == 'ExoMinerRaw'):
        RunExoMiner(mission=mission,
                    lr=lr,
                    training_size=training_size,
                    batch_size=batch_size,
                    epochs=epochs,
                    type_sample=type_sample)
    else:
        print('Choose either "ExoMiner" or "ExoMinerRaw" for model argument')
    return None

if __name__=='__main__':
    ModelChoice(model='ExoMinerRaw',
                  mission='ComboRaw',
                  lr=6.37e-5,
                  training_size=.8,
                  batch_size=10,
                  epochs=500,
                  type_sample='Oversample')


import uuid
from django.db import models

class SkillsOverview(models.Model):
    '''Skill overview for homepage'''
    overview = models.CharField(max_length=250, help_text='Overview of skills')

    def __str__(self):
        return self.overview[:80]

    class Meta:
        verbose_name_plural = 'Skills Overview'

class Skill(models.Model):
    '''Skill model. Enter skills for viewing on homepage'''
    CATEGORY_CHOICES = (
            ('FRONTEND', 'Frontend'),
            ('BACKEND', 'Backend'),
            ('OTHER', 'Other'),
        )
    category = models.CharField(default='OTHER', max_length=8, choices=CATEGORY_CHOICES, help_text='Category for the skill')
    name = models.CharField(max_length=50, help_text='Name of the skill')
    link = models.URLField(blank=True, help_text='Link to follow for more info on the skill.')
    years_experience = models.IntegerField(blank=True, help_text='Years experience with skill')
    notes = models.CharField(blank=True, default='', max_length=250, help_text='Anything extra worth annotating')

    def __str__(self):
        return self.name

    class Meta:
        unique_together = ['name', 'category',]


from email.policy import default
from flask.templating import render_template
from flask_wtf import FlaskForm
from wtforms import StringField, IntegerField, DecimalField
from flask_wtf.file import FileField, FileRequired, FileAllowed
from flask_debugtoolbar import DebugToolbarExtension
from wtforms.validators import DataRequired, Optional
from flask import Flask, send_file
import os
import numpy
from pydub import AudioSegment
from pathlib import Path
from scipy.io.wavfile import write
from PIL import Image, ImageFont, ImageDraw
from datetime import datetime, date
from moviepy.video import VideoClip
from moviepy.editor import *

global dict_data_store

def clearMedia():
        dirs = ["audio", "output", "uploads"]
        for dir in dirs:
            for f in os.listdir(dir):
                if("nodel" not in f):
                    os.remove(os.path.join(dir, f))

SECRET_KEY = os.urandom(32)

class appForm(FlaskForm):
    name = StringField("Name", validators=[DataRequired()])
    rr_code = StringField("RR Code", validators=[DataRequired()])
    wish = StringField("Wish", validators=[DataRequired()])
    photo = FileField("Photo", validators=[FileRequired(), FileAllowed(['jpg', 'png', 'jpeg'], 'Images only!')])
    frequency1 = DecimalField("Frequency1", validators=[DataRequired()], places=2)
    vol1 = IntegerField("Volume1", validators=[Optional()], default=10)
    frequency2 = DecimalField("Frequency2", places=2, default=0.0, validators=[Optional()])
    vol2 = IntegerField("Volume2", validators=[Optional()], default=10)
    frequency3 = DecimalField("Frequency3", places=2, default=0.0, validators=[Optional()])
    vol3 = IntegerField("Volume3", validators=[Optional()], default=10)
    frequency4 = DecimalField("Frequency4", places=2, default=0.0, validators=[Optional()])
    vol4 = IntegerField("Volume4", validators=[Optional()], default=10)
    duration = IntegerField("Duration", validators=[DataRequired()])
    clip = FileField("Audio Clip", validators=[Optional(), FileAllowed(['mp3', 'wav', 'mp4', 'm4a', 'flac'], 'Audio Files only!')])

def processPhoto(photo, msgs):
    old_im = Image.open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "uploads", photo))
    old_size = old_im.size
    aspect_ratio = old_size[1]/old_size[0]
    old_im = old_im.resize((1080, int(1080*aspect_ratio)), Image.NEAREST)
    old_size = old_im.size
    new_size = (1080, 1920)
    new_im = Image.new("RGB", new_size)   ## luckily, this is already black!
    new_im.paste(old_im, ((new_size[0]-old_size[0])//2, (new_size[1]-old_size[1])//2))

    textfont = ImageFont.truetype('Calibri.ttf', 50)
    W = 1080
    H = 1920
    draw = ImageDraw.Draw(new_im)

    msg = ""
    for x in msgs:
        msg += x + "\n\n"

    draw.text((W/2, H), msg, align="center", font=textfont, anchor="md")
    new_im.save(os.path.join(os.path.dirname(os.path.abspath(__file__)), "edited", photo))

def generateAudio(frequency_list, vol_list, duration, outfile):
    outlist = []
    sample_rate = 44100
    count = 0
    for freq in frequency_list:
        frequency = int(freq)
        print(" > Generating Frequency : ", frequency)
        samples = ""
        samples = (numpy.sin(2 * numpy.pi * numpy.arange(sample_rate * duration) * frequency / sample_rate)).astype(numpy.float32)
        write(os.path.join(os.path.dirname(os.path.abspath(__file__)), "audio", outfile + "_" + str(count) + ".wav"), sample_rate, samples)
        outlist.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "audio", outfile + "_" + str(count) + ".wav"))
        count += 1
        print(outlist)

    tmpsound = AudioSegment.from_file(file=outlist[0], format="wav")
    sound = tmpsound - 2 * (10 - vol_list[0])

    for x in range(1, len(outlist)):
        tmpsound2 = AudioSegment.from_file(file=outlist[x], format="wav")
        sound2 = tmpsound2 - 2 * (10 - vol_list[x])
        sound = sound.overlay(sound2, position=0)

    sound.export(os.path.join(os.path.dirname(os.path.abspath(__file__)), "audio", outfile + ".mp3"), format="mp3")

def generateAudioWithFile(frequency_list, vol_list, duration, outfile, audio_file):
    outname = os.path.join(os.path.dirname(os.path.abspath(__file__)), "audio", outfile + "_temp")
    generateAudio(frequency_list, vol_list, duration, outname)
    outname += ".mp3"
    overloop = AudioSegment.from_file(file = audio_file)
    audio = AudioSegment.from_file(file = outname)
    audio = audio.overlay(overloop, position=0, loop=True)
    audio.export(os.path.join(os.path.dirname(os.path.abspath(__file__)), "audio", outfile + ".mp3"), format="mp3")

def generateVideo(name, photo, rr, wish, freq, len, vols, audio):
    processPhoto(photo, [name, wish, "RR Code - " + rr, " - Radionic Tone Healing"])
    image_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), "edited")
    filename = Path(photo).stem
    duration = len * 60

    if audio is not None:
        generateAudioWithFile(freq, vols, duration, filename, audio)
    else:
        generateAudio(freq, vols, duration, filename)

    audioclip = AudioFileClip(os.path.join(os.path.dirname(os.path.abspath(__file__)), "audio", filename + ".mp3"))

    clip = ImageClip(os.path.join(image_folder, photo)).set_duration(audioclip.duration)
    clip = clip.set_audio(audioclip)
    clip.write_videofile(os.path.join(os.path.dirname(os.path.abspath(__file__)), "output", filename + ".mp4"), fps=1)
    return os.path.join(os.path.dirname(os.path.abspath(__file__)), "output", filename + ".mp4")

app = Flask("Radionic Tone Healing Script", template_folder=os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates"))
app.config['SECRET_KEY'] = SECRET_KEY
app.debug = False
toolbar = DebugToolbarExtension(app)

@app.route('/', methods = ['GET', 'POST'])
def index():
    # clearMedia()
    global dict_data_store
    form = appForm()
    if form.validate_on_submit():
        upload_dir = os.path.join(os.path.dirname(app.instance_path), 'uploads')

        p = form.photo.data
        photoname = datetime.now().strftime("%d%m%Y%H%M%S") + Path(p.filename).suffix
        p.save(os.path.join(upload_dir, photoname))
        freq_list = [form.frequency1.data, form.frequency2.data, form.frequency3.data, form.frequency4.data]
        vol_list = [form.vol1.data, form.vol2.data, form.vol3.data, form.vol4.data]

        freq_list = [x for x in freq_list if x is not None]
        vol_list = [x for x in vol_list if x is not None]

        video_op = generateVideo(form.name.data, photoname, form.rr_code.data, form.wish.data, freq_list, form.duration.data, vol_list, form.clip.data)

        return send_file(video_op, as_attachment=True)

    return render_template('index.html', form=form)

@app.route('/uploaded', methods=['GET'])
def postUpload():
    global dict_data_store

    return render_template

if __name__ == '__main__':
    # Bind to PORT if defined, otherwise default to 5000.
    port = int(os.environ.get('PORT', 5000))
    app.run(host='0.0.0.0', port=port)


import random, os, sys
import cv2
from glob import glob
classes = ['Human Face',
           'License Plate',
			]
typ = "jpeg";  print(f"{typ} types only. ")

colors = [(204, 204, 0), (51, 51, 255) ]

abs_path = str(sys.argv[1]) # absolute path of the images
thresh_folder= str(sys.argv[2]) # assumes that txt files are stored in threshold folder

img_paths = glob(abs_path+f"**/*.{typ}", recursive=True)
print()

# cv2.namedWindow("image_name")
for path in img_paths:
	image = cv2.imread(str(path))
	h, w, _ = image.shape

	img_name = path.split("/")[-1]
	common_path = path.split("/")[:-1]
	common_path.append(thresh_folder)
	# import ipdb; ipdb.set_trace()
	txt_path = "/".join(common_path)+str(img_name.replace(f".{typ}", ".txt"))

	with open(txt_path, 'r') as f:
		data = f.read().splitlines()
	if data:
		print(f"{img_name}")
		for line in data:
			class_ind, x, y, width, height = line.split(' ')
			class_ind, x, y, width, height = int(class_ind), float(x), float(y), float(width), float(height)
			# print(class_ind, x, y, w, h)
			x1, y1 = int((x - width / 2) * w), int((y - height / 2) * h)
			x2, y2 = int((x + width / 2) * w) , int((y + height / 2) * h)
			org = (x1, y1 - 30)
			fontScale = 1
			font = cv2.FONT_HERSHEY_SIMPLEX
			color = colors[class_ind]
			text = classes[class_ind]
			thickness = 1
			cv2.rectangle(image, (x1, y1), (x2, y2), color, 2)
			image = cv2.putText(image, text,
								org, font, fontScale, color, thickness, cv2.LINE_AA)

		img_name = path.split("/")[-1]
		cv2.imwrite(f'{ abs_path }/inference/{img_name}', image)

	# cv2.imshow("image_name", image )

	# cv2.waitKey()
	# if k==27:    # Esc key to stop
	# 	break

# and finally destroy/Closing all open windows
# cv2.destroyAllWindows()


from typing import List, Optional, Tuple, Union

from ..model import Compound, Parameter, ReferableElement, TypeRef
from .base import TranscoderBase

# https://kotlinlang.org/docs/reference/java-interop.html#mapped-types
_MAPPED_TYPES = {
    # Java primitives
    "byte": "Byte",
    "short": "Short",
    "int": "Int",
    "long": "Long",
    "char": "Char",
    "float": "Float",
    "double": "Double",
    "boolean": "Boolean",

    # Java non-primitive built-ins
    "java.lang.Object": "Any",
    "java.lang.Cloneable": "Cloneable",
    "java.lang.Comparable": "Comparable",
    "java.lang.Enum": "Enum",
    "java.lang.Annotation": "Annotation",
    "java.lang.CharSequence": "CharSequence",
    "java.lang.String": "String",
    "java.lang.Number": "Number",
    "java.lang.Throwable": "Throwable",
    "Object": "Any",

    # Java boxed types
    "java.lang.Byte": "Byte",
    "java.lang.Short": "Short",
    "java.lang.Integer": "Int",
    "Integer": "Int",
    "java.lang.Long": "Long",
    "java.lang.Character": "Char",
    "Character": "Char",
    "java.lang.Float": "Float",
    "java.lang.Double": "Double",
    "java.lang.Boolean": "Boolean",

    # Other
    "void": "Unit",
}

_NULLABLE_MAPPED_TYPES = [
    "java.lang.Byte",
    "java.lang.Short",
    "java.lang.Integer",
    "java.lang.Long",
    "java.lang.Character",
    "java.lang.Float",
    "java.lang.Double",
    "java.lang.Boolean",
    "Byte",
    "Short",
    "Integer",
    "Long",
    "Character",
    "Float",
    "Double",
    "Boolean",
]

_NULLABLE_OR_NOT_MAPPED_TYPES = [
    "java.lang.Object",
    "java.lang.Cloneable",
    "java.lang.Comparable",
    "java.lang.Enum",
    "java.lang.Annotation",
    "java.lang.CharSequence",
    "java.lang.String",
    "java.lang.Number",
    "java.lang.Throwable",
    "Object",
    "Cloneable",
    "Comparable",
    "Enum",
    "Annotation",
    "CharSequence",
    "String",
    "Number",
    "Throwable",
]

_NONNULL_MAPPED_TYPES = [
    "byte",
    "short",
    "int",
    "long",
    "char",
    "float",
    "double",
    "boolean",
    "void",
]

# https://kotlinlang.org/docs/reference/java-interop.html#nullability-annotations
# https://github.com/JetBrains/kotlin/blob/master/core/compiler.common.jvm/src/org/jetbrains/kotlin/load/java/JvmAnnotationNames.kt
_NULLABLE_ANNOTATIONS = [
    "@Nullable",
    "@CheckForNull",
    "@PossiblyNull",
    "@NullableDecl",
    "@RecentlyNullable",
    "@org.jetbrains.annotations.Nullable",
    "@androidx.annotation.Nullable",
    "@android.support.annotation.Nullable",
    "@android.annotation.Nullable",
    "@com.android.annotations.Nullable",
    "@org.eclipse.jdt.annotation.Nullable",
    "@org.checkerframework.checker.nullness.qual.Nullable",
    "@javax.annotation.Nullable",
    "@javax.annotation.CheckForNull",
    "@edu.umd.cs.findbugs.annotations.CheckForNull",
    "@edu.umd.cs.findbugs.annotations.Nullable",
    "@edu.umd.cs.findbugs.annotations.PossiblyNull",
    "@io.reactivex.annotations.Nullable",
    "@org.checkerframework.checker.nullness.compatqual.NullableDecl",
    "@androidx.annotation.RecentlyNullable",
]

_NONNULL_ANNOTATIONS = [
    "@NotNull",
    "@NonNull",
    "@NonNullDecl",
    "@RecentlyNonNull",
    "@org.jetbrains.annotations.NotNull",
    "@edu.umd.cs.findbugs.annotations.NonNull",
    "@androidx.annotation.NonNull",
    "@android.support.annotation.NonNull",
    "@android.annotation.NonNull",
    "@com.android.annotations.NonNull",
    "@org.eclipse.jdt.annotation.NonNull",
    "@org.checkerframework.checker.nullness.qual.NonNull",
    "@lombok.NonNull",
    "@io.reactivex.annotations.NonNull"
    "@javax.annotation.Nonnull",
    "@org.checkerframework.checker.nullness.compatqual.NonNullDecl",
    "@androidx.annotation.RecentlyNonNull",
]

# https://kotlinlang.org/docs/reference/java-interop.html#java-arrays
_PRIMITIVE_ARRAY_TYPES = [
    "byte",
    "short",
    "int",
    "long",
    "char",
    "float",
    "double",
    "boolean",
]

class KotlinTranscoder(TranscoderBase):
    SOURCE = "java"
    TARGET = "kotlin"

    def convert_name(self, source_element: Union[ReferableElement, TypeRef]) -> str:
        return _MAPPED_TYPES.get(source_element.name, source_element.name)

    def convert_full_name(self, source_element: ReferableElement) -> str:
        return _MAPPED_TYPES.get(source_element.full_name, source_element.full_name)

    def _compound(self, compound: Compound) -> Compound:
        transcoded = super()._compound(compound)
        transform_properties(transcoded.members)

        if (transcoded.returns is not None and transcoded.returns.type is not None
                and transcoded.returns.type.name == "Unit"):
            transcoded.returns = None

        return transcoded

    def type_ref(self, type_ref: TypeRef) -> TypeRef:
        transcoded = super().type_ref(type_ref)
        transcoded = transform_array(type_ref, transcoded)
        transcoded = set_nullability(type_ref, transcoded)

        if transcoded.prefix:
            transcoded.prefix = transcoded.prefix.replace("? extends", "out")
            transcoded.prefix = transcoded.prefix.replace("? super", "in")

        return transcoded

    def parameter(self, parameter: Parameter) -> Parameter:
        transcoded = super().parameter(parameter)
        transcoded = transform_vararg(transcoded)
        return transcoded

def transform_properties(members: List[Compound]) -> List[Compound]:
    getters = {
        m.name: m
        for m in members
        if ((m.name.startswith("get") or m.name.startswith("is")) and len(m.params) == 0)
        and m.returns is not None
    }
    setters = {
        m.name: m
        for m in members if (m.name.startswith("set") and len(m.params) == 1) and m.returns is None
    }

    for getter_name, getter in getters.items():
        if getter_name.startswith("get"):
            property_name = getter_name[3:]
        else:
            property_name = getter_name[2:]

        setter = setters.get(f"set{property_name}", None)
        if setter is None:
            continue

        if getter.name.startswith("get"):
            getter.name = f"{property_name[0].lower()}{property_name[1:]}"
            if getter.namespace:
                getter.full_name = f"{getter.namespace}.{getter.name}"
            else:
                getter.full_name = getter.name
        getter.kind = "property"
        members.remove(setter)

    return members

def set_nullability(type_ref: TypeRef, transcoded: TypeRef) -> TypeRef:
    nullable_prefix, transcoded.prefix = strip_annotations(transcoded.prefix, _NULLABLE_ANNOTATIONS)
    nullable_suffix, transcoded.suffix = strip_annotations(transcoded.suffix, _NULLABLE_ANNOTATIONS)
    nonnull_prefix, transcoded.prefix = strip_annotations(transcoded.prefix, _NONNULL_ANNOTATIONS)
    nonnull_suffix, transcoded.suffix = strip_annotations(transcoded.suffix, _NONNULL_ANNOTATIONS)

    kotlin_nullable = "!"
    if nullable_prefix or nullable_suffix:
        kotlin_nullable = "?"
    elif nonnull_prefix or nonnull_suffix:
        kotlin_nullable = ""
    elif type_ref.name in _NULLABLE_MAPPED_TYPES:
        kotlin_nullable = "?"
    elif type_ref.name in _NONNULL_MAPPED_TYPES:
        kotlin_nullable = ""

    if kotlin_nullable:
        if transcoded.suffix:
            transcoded.suffix = f"{kotlin_nullable}{transcoded.suffix}"
        else:
            transcoded.suffix = kotlin_nullable

    return transcoded

def strip_annotations(text: Optional[str], annotations: List[str]) -> Tuple[bool, Optional[str]]:
    if not text:
        return False, text

    result = " ".join(token for token in text.split(" ") if token not in annotations)
    return len(text) != len(result), result

def transform_array(type_ref: TypeRef, transcoded: TypeRef) -> TypeRef:
    if not transcoded.suffix or "[]" not in transcoded.suffix:
        return transcoded

    if type_ref.name in _PRIMITIVE_ARRAY_TYPES:
        transcoded.name = f"{type_ref.name[0].upper()}{type_ref.name[1:]}Array"
        transcoded.suffix = (type_ref.suffix.replace("[]", "!")
                             if type_ref.suffix is not None else "")
        return transcoded

    array_type = TypeRef("kotlin", "Array")
    array_type.prefix = transcoded.prefix
    array_type.suffix = transcoded.suffix.replace("[]", "")

    transcoded.prefix = "(out) "
    transcoded.suffix = ""
    array_type.nested = [transcoded]

    return array_type

def transform_vararg(param: Parameter) -> Parameter:
    if param.type is None or not param.type.suffix or "..." not in param.type.suffix:
        return param

    param.prefix = "vararg "
    param.type.suffix = param.type.suffix.replace("...", "")
    param.type.suffix = param.type.suffix.replace("?", "")
    param.type.suffix = param.type.suffix.replace("!", "")
    return param


from django.contrib import admin

# Register your models here.

from .models import ContactForm

class ContactFormAdmin(admin.ModelAdmin):
    search_fields = ['pk', 'full_name']
    list_filter = ['full_name']
    list_display = [
        'pk',
        'created_at',
        'full_name',
        'email_id',
        'contact_no',
        'message'
    ]
    list_editable = ['full_name']

admin.site.register(ContactForm, ContactFormAdmin)


# -*- coding: utf-8 -*-
import scrapy
from EnergyParser.items import EnergyparserItem

class MelillaSpider(scrapy.Spider):
    name = 'melilla'
    # Start URL
    start_urls = ['https://www.ree.es/en/balance-diario/melilla/2014/01/01/']

    url_template = 'https://www.ree.es/en/balance-diario/melilla/20{}/{:02d}/{:02d}/'

    # set custom settings
    custom_settings = {
        # 'DEPTH_LIMIT': 2,
        'FEED_FORMAT': "csv", # Save format
        'FEED_URI': 'energy_melilla.csv' # File name
    }

    def parse(self, response):
        '''
        Function for crawling
        '''
        for year in range(14, 20): # From 2014 to 2019
            for month in range(1, 13): # From Jun to Dec
                for day in range(1, 32): # Days from 1 to 31
                    NextUrl = self.url_template.format(year, month, day)
                    yield response.follow(NextUrl, callback=self.parse_data)

    def parse_data(self, response):
        '''
        Function for scraping
        '''
        info = EnergyparserItem()
        info['date'] = response.url.split("/")[6:9]
        info['Diesel_engines'] = response.xpath('//table[1]/tbody[1]/tr[2]/td[1]/text()').extract()
        info['Gas_turbine'] = response.xpath('//table[1]/tbody[1]/tr[3]/td[1]/text()').extract()
        info['Fuel_oil_gas'] = response.xpath('//table[1]/tbody[1]/tr[4]/td[1]/text()').extract()
        info['Solar_PV'] = response.xpath('//table[1]/tbody[1]/tr[5]/td[1]/text()').extract()
        info['Non_renewable_waste'] = response.xpath('//table[1]/tbody[1]/tr[6]/td[1]/text()').extract()
        info['Renewable_waste'] = response.xpath('//table[1]/tbody[1]/tr[7]/td[1]/text()').extract()
        info['Generation'] = response.xpath('//table[1]/tbody[1]/tr[8]/td[1]/text()').extract()
        info['Demand'] = response.xpath('//table[1]/tbody[1]/tr[9]/td[1]/text()').extract()
        info['Peak_Load'] = response.xpath('//table[2]/tbody[1]/tr[2]/td[1]/text()').extract()
        info['Daily_demand'] = response.xpath('//table[2]/tbody[1]/tr[3]/td[1]/text()').extract()

        yield info


from typing import Optional

class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val: int = val
        self.left: Optional[TreeNode] = left
        self.right: Optional[TreeNode] = right

# https://leetcode.com/problems/kth-smallest-element-in-a-bst/
class Solution:
    # 90th percentile - traversal short-circuits & we only store val of interest
    def kthSmallest(self, root: Optional[TreeNode], k: int) -> int:
        return self.kth_smallest(root, k)[0]

    def kth_smallest(self, root: Optional[TreeNode], k: int) -> tuple:  # (val, k)
        if not root:
            return None, k

        val, k = self.kth_smallest(root.left, k)
        if val is not None:
            return val, k

        # important note: $k is only modified for each root.val. this means parents need to accept children's $k's.
        k -= 1
        if not k:
            return root.val, k

        return self.kth_smallest(root.right, k)

    # 60th percentile - slower but dead simple; has a bit of elegance in its own way
    def slow_kth_smallest(self, root: Optional[TreeNode], k: int) -> int:
        return self.get_vals(root)[k - 1]

    def get_vals(self, root: Optional[TreeNode]):
        if not root:
            return []
        return self.get_vals(root.left) + [root.val] + self.get_vals(root.right)


"""SSL verification utility."""

import argparse
import csv
import logging
import socket
import ssl
from concurrent.futures import ThreadPoolExecutor
from typing import Dict, List, Optional, Tuple, Type, Union

logging.basicConfig()
logger = logging.getLogger("ssl_verify")

class CSVReport:
    """Report  writer."""

    def __init__(
        self,
        data: List[Tuple[str, str, str]],
        file_descriptor,
        dialect: Union[str, csv.Dialect, Type[csv.Dialect]] = "excel",
    ) -> None:
        self.header = ("Host", "SSL validity", "Expires")
        self.writer = csv.writer(file_descriptor, dialect=dialect)
        self.data = data

    def save(self) -> None:
        """Save report."""
        self.writer.writerow(self.header)
        for row in self.data:
            self.writer.writerow(row)

class SSLVerificator:
    """SSL verificator class."""

    THREADS = 5
    TIMEOUT = 10

    LEVEL_ERROR = 1
    LEVEL_INFO = 2
    LEVEL_DEBUG = 3

    VERBOSITY_LEVELS = {
        LEVEL_ERROR: logging.ERROR,
        LEVEL_INFO: logging.INFO,
        LEVEL_DEBUG: logging.DEBUG,
    }

    def __init__(self) -> None:
        parser = self._setup_params()
        self.args = self._get_args(parser)
        self._set_verbosity()

    def _set_verbosity(self) -> None:
        logger.setLevel(self.VERBOSITY_LEVELS[self.args.verbosity])

    def _setup_params(self) -> argparse.ArgumentParser:
        """Setup all utility parameters."""
        parser = argparse.ArgumentParser(
            description="This utility will help you to check domains SSL validity."
        )
        parser.add_argument(
            "-if",
            "--input_file",
            required=True,
            type=argparse.FileType("r"),
            help="Input file with list of domains names.",
        )
        parser.add_argument(
            "-of",
            "--output_file",
            type=argparse.FileType("w"),
            required=True,
            help="Output csv file with results of checking.",
        )
        parser.add_argument(
            "-v",
            "--verbosity",
            type=int,
            choices=tuple(self.VERBOSITY_LEVELS.keys()),
            default=self.LEVEL_ERROR,
            help="Verbosity level.",
        )
        parser.add_argument(
            "-t", "--threads", type=int, default=self.THREADS, help="Number of threads."
        )
        parser.add_argument(
            "-p", "--print", action="store_true", help="Print domains names to console."
        )
        return parser

    @staticmethod
    def _get_args(parser: argparse.ArgumentParser) -> argparse.Namespace:
        """Return list of parameters."""
        return parser.parse_args()

    def _get_cert(self, domain: str, port: Optional[int] = 443) -> Tuple[str, Union[None, Dict]]:
        context = ssl.SSLContext()
        context.verify_mode = ssl.CERT_REQUIRED
        context.check_hostname = True
        context.load_default_certs()

        with context.wrap_socket(socket.socket(socket.AF_INET), server_hostname=domain) as conn:
            try:
                conn.settimeout(self.TIMEOUT)
                conn.connect((domain, port))
                cert = conn.getpeercert()
            except (socket.timeout, ssl.CertificateError, OSError) as ex:
                logger.debug("Error for check %s: %s", domain, ex)
                return (domain, None)
        return (domain, cert)

    def _print_msg(self, msg: Tuple[str, str, str]) -> None:
        if self.args.print:
            print("{:<30} {:<20} {:<20}".format(*msg))

    @staticmethod
    def _get_expire_date(cert: Dict) -> str:
        return cert.get("notAfter", "")

    @staticmethod
    def _get_domain(line: str) -> str:
        """Return domain name from string."""
        line = line.strip().replace("https://", "").replace("http://", "").split("/")[0]
        return line

    def _get_report_writer(self) -> Type[CSVReport]:
        return CSVReport

    def _report(self, result: List[Tuple[str, str, str]]) -> None:
        """Save report."""
        report_class = self._get_report_writer()
        with self.args.output_file as out_fl:
            report = report_class(result, out_fl)
            report.save()

    def _get_domain_list(self) -> List[str]:
        """Return list of domains from file."""
        result = []
        with self.args.input_file as input_file:
            for idx, line in enumerate(input_file.readlines()):
                domain = self._get_domain(line)
                if not domain:
                    logger.warning("Wrong domain name at line %s.", idx + 1)
                    continue
                result.append(domain)
        return result

    def _validate_domains(self, domain_list: List[str]) -> List[Tuple[str, str, str]]:
        """Validate domain list."""
        result = []
        with ThreadPoolExecutor(max_workers=self.args.threads) as th_pool:
            for domain, cert_data in th_pool.map(self._get_cert, domain_list):
                domain_ssl_info = (
                    domain,
                    "valid" if cert_data else "NOT valid",
                    self._get_expire_date(cert_data) if cert_data else "",
                )
                self._print_msg(domain_ssl_info)
                result.append(domain_ssl_info)
        return result

    def process(self) -> None:
        """Entry point."""
        domain_list = self._get_domain_list()
        result_list = self._validate_domains(domain_list)
        self._report(result_list)

if __name__ == "__main__":
    verificator = SSLVerificator()
    verificator.process()


import json
import boto3

dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table('TodoTasks')

def lambda_handler(event, context):
    try:
        task = event['task']
        description = event['description']
        due_date = event['dueDate']  # Retrieve the due date from the event payload

        # Add task to DynamoDB table
        table.put_item(
            Item={
                'ID': task,
                'description': description,
                'dueDate': due_date  # Store the due date in DynamoDB
            }
        )

        return {
            'statusCode': 200,
            'body': json.dumps("You successfully added task: " + task)
        }
    except Exception as e:
        return {
            'statusCode': 500,
            'body': json.dumps({'message': str(e)})
        }


import numpy as np
import tensorflow as tf
import random
import pickle
import nltk
from collections import Counter
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer

pos_file='E:/dataset/NLP/Sentiment/rt-polaritydata/pos.txt'
neg_file='E:/dataset/NLP/Sentiment/rt-polaritydata/neg.txt'

def create_dic(pos_file,neg_file):
    dic=[]
    def process_file(file):
        with open(file,'r',encoding='utf-8') as f:
            dic=[]
            lines=f.readlines()
            for line in lines:
                words=word_tokenize(line.lower())  #令牌化
                dic+=words
            return dic
    dic+=process_file(pos_file)
    dic+=process_file(neg_file)

    lemmatizer=WordNetLemmatizer()
    dic=[lemmatizer.lemmatize(word) for word in dic] #将单词还原成原级形式

    word_count=Counter(dic) #计算单词出现次数

    dic=[]
    for word in word_count:
        if word_count[word]<2000 and word_count[word]>20: #去掉常用词和及不常用词
            dic.append(word)
    return dic

dic=create_dic(pos_file,neg_file)
print(len(dic))

def normalize_dataset(dic):
    dataset=[]
    def string_to_vector(dic,review,clf):
        words=word_tokenize(line.lower())
        lemmatizer=WordNetLemmatizer()
        words=[lemmatizer.lemmatize(word) for word in words]

        features=np.zeros(len(dic))
        for word in words:
            if word in dic:
                features[dic.index(word)]=1
        return np.array([features,clf])

    with open(pos_file,'r',encoding='utf-8') as f:
        lines=f.readlines()
        for line in lines:
            one_sample=string_to_vector(dic,line,[1,0])
            dataset.append(one_sample)

    with open(neg_file,'r',encoding='utf-8') as f:
        lines=f.readlines()
        for line in lines:
            one_sample=string_to_vector(dic,line,[0,1])
            dataset.append(one_sample)

    return dataset

dataset=normalize_dataset(dic)

random.shuffle(dataset)

'''
# 整理好数据保存到文件。
with open('E:/dataset/NLP/Sentiment/rt-polaritydata/save.pickle','wb') as f:
    pickle.dump(dataset,f)
'''

test_size=int(len(dataset)*0.2)

dataset=np.array(dataset)

train_dataset=dataset[:-test_size]
test_dataset=dataset[-test_size:]

n_input_layer=len(dic)

n_layer_1=1064
n_layer_2=2000

n_output_layer=2

def neural_network(data):
    layer_1_w_b={'w_':tf.Variable(tf.random_normal([n_input_layer,n_layer_1])),'b_':tf.Variable(tf.random_normal([n_layer_1]))}
    layer_2_w_b={'w_':tf.Variable(tf.random_normal([n_layer_1,n_layer_2])),'b_':tf.Variable(tf.random_normal([n_layer_2]))}
    layer_output_w_b={'w_':tf.Variable(tf.random_normal([n_layer_2,n_output_layer])),'b_':tf.Variable(tf.random_normal([n_output_layer]))}

    layer_1=tf.add(tf.matmul(data,layer_1_w_b['w_']),layer_1_w_b['b_'])
    layer_1=tf.nn.relu(layer_1)
    layer_2=tf.add(tf.matmul(data,layer_2_w_b['w_']),layer_2_w_b['b_'])
    layer_2=tf.nn.relu(layer_2)
    layer_output=tf.add(tf.matmul(layer_2,layer_output_w_b['w_']),layer_output_w_b['b_'])

    return layer_output

batch_size=50

X=tf.placeholder('float',[None,len(train_dataset[0][0])])
Y=tf.placeholder('float')

def train_neural_network(X,Y):
    predict=neural_network(X)
    cost_func=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predict,labels=Y))
    optimizer=tf.train.AdamOptimizer().minimize(cost_func)

    epochs=20
    with tf.Session() as session:
        session.run(tf.global_variables_initializer())
        epoch_loss=0

        i=0
        random.shuffle(train_dataset)

        train_x=dataset[:,0]
        train_y=dataset[:,1]

        for epoch in range(epochs):
            while i<len(train_x):
                start=1
                end=i+batch_size

                batch_x=train_x[start:end]
                batch_y=train_y[start:end]

                _,c=session.run([optimizer,cost_func],feed_dict={X:list(batch_x),Y:list(batch_y)})
                epoch_loss+=c
                i+=batch_size
            print(epoch,':',epoch_loss)

        test_x=test_dataset[:,0]
        test_y=test_dataset[:,1]
        correct=tf.equal(tf.argmax(predict,1),tf.argmax(Y,1))
        accuracy=tf.reduce_mean(tf.cast(correct,'float'))
        print('准确率：',accuracy.eval({X:list(test_x),Y:list(test_y)}))

train_neural_network(X,Y)


from django.contrib.auth.decorators import login_required
from django.http import JsonResponse
from django.shortcuts import render, get_object_or_404
from .models import Feedback
from lab.models import LabInSection
from cms.models import Task

@login_required
def process(request,task_id,labinsec_id,media_type,media_id):
    try:
        if labinsec_id == 'none':
            lab_in_sec = None
        else:
            lab_in_sec = LabInSection.objects.get(id=labinsec_id)
        task = Task.objects.get(id=task_id)
        properties = dict(
                    user=request.user,
                    task=task,
                    lab_in_sec=lab_in_sec,
                    media_id=media_id,
                    media_type=media_type,
                )
        if request.method == 'POST':
                try:
                    fb = Feedback.objects.get(**properties)
                except Feedback.DoesNotExist:
                    fb = Feedback(**properties)
                fb.comments = request.POST['comments']
                fb.rating = int(request.POST['rating'])
                fb.save()
                return JsonResponse({ 'status':'ok', })
        else:
            fb = get_object_or_404(Feedback,**properties)
            return JsonResponse({
                'comments': fb.comments,
                'rating'  : fb.rating
                })
    except Exception as e:
        return JsonResponse({
            'status':'error',
            'message':str(e),
            },status=400)


import datetime

from django.db.models.signals import post_save
from django.dispatch import receiver
from datetime import timedelta

from .models import Schedule, Talon

@receiver(post_save, sender=Schedule)
def generate_talons_for_schedule(sender, instance, created, **kwargs):
    if created:
        schedule = instance
        day_of_week = schedule.day_of_week

        today = datetime.date.today()
        offset = (day_of_week - today.weekday()) % 7

        next_weekday = today + timedelta(days=offset)
        weekdays = [next_weekday + timedelta(weeks=i) for i in range(4)]

        doctor = schedule.doctor

        for weekday in weekdays:
            current_time = schedule.start_time
            while current_time < schedule.end_time:
                talon = Talon.objects.create(
                    doctor=doctor,
                    date=weekday,
                    time=current_time,
                )
                current_datetime = datetime.datetime.combine(weekday, current_time)
                current_datetime += timedelta(minutes=30)
                current_time = current_datetime.time()


from __future__ import annotations

import pytest

from cursesmenu import CursesMenu
from cursesmenu.items import ExitItem, MenuItem

pytestmark = pytest.mark.usefixtures("mock_cursesmenu_curses", "mock_clear")

@pytest.fixture()
def sample_items():
    item0 = MenuItem("item0")
    item1 = MenuItem("item1")
    return [item0, item1]

@pytest.fixture()
def sample_menu(sample_items, mock_cursesmenu_curses_vary_window_size):  # noqa: ARG001
    menu = CursesMenu("menu", "TestSampleMenu")
    menu.items.append(sample_items[0])
    menu.items.append(sample_items[1])
    menu.start()
    menu.wait_for_start(timeout=10)
    yield menu
    menu.exit()
    menu.join(timeout=10)

@pytest.fixture()
def empty_menu():
    menu = CursesMenu("menu", "empty menu", show_exit_item=False)
    menu.start()
    menu.wait_for_start(timeout=10)
    yield menu
    menu.exit()
    menu.join(timeout=10)

@pytest.fixture()
def big_menu():
    menu = CursesMenu("Test Menu")
    for i in range(100):
        menu.items.append(MenuItem(f"item{i}", should_exit=True))
    menu.start()
    menu.wait_for_start(timeout=10)
    yield menu
    menu.exit()
    menu.join(timeout=10)

def test_empty_menu(empty_menu: CursesMenu):
    assert empty_menu.current_item is None
    assert empty_menu.selected_item is None
    empty_menu.select()
    empty_menu.go_to_exit()
    empty_menu.join(timeout=10)
    assert not empty_menu.is_alive()

def test_big_menu(big_menu: CursesMenu):  # noqa: ARG001
    pass

def test_go_down(sample_menu: CursesMenu, sample_items: list[MenuItem]):
    sample_menu.go_down()
    assert sample_menu.current_option == 1
    assert sample_menu.current_item is sample_items[1]
    sample_menu.go_down()
    assert sample_menu.current_option == 2
    assert type(sample_menu.current_item) == ExitItem
    sample_menu.go_down()
    assert sample_menu.current_option == 0
    assert sample_menu.current_item is sample_items[0]

def test_go_up(sample_menu: CursesMenu, sample_items: list[MenuItem]):
    sample_menu.go_up()
    assert sample_menu.current_option == 2
    assert type(sample_menu.current_item) == ExitItem
    sample_menu.go_up()
    assert sample_menu.current_option == 1
    assert sample_menu.current_item is sample_items[1]
    sample_menu.go_up()
    assert sample_menu.current_option == 0
    assert sample_menu.current_item is sample_items[0]

def test_go_to(sample_menu: CursesMenu, sample_items: list[MenuItem]):
    sample_menu.go_to(ord("2"))
    assert sample_menu.current_option == 1
    assert sample_menu.current_item is sample_items[1]
    sample_menu.go_to(ord("7"))
    assert sample_menu.current_option == 1
    assert sample_menu.current_item is sample_items[1]

def test_go_to_big(big_menu: CursesMenu):
    big_menu.go_to(ord("8"))
    assert big_menu.current_option == 7
    assert big_menu.current_item == big_menu.items[7]

def test_go_to_empty(empty_menu: CursesMenu):
    empty_menu.go_to(ord("1"))

def test_select(sample_menu: CursesMenu, sample_items: list[MenuItem]):
    sample_menu.select()
    assert sample_menu.current_option == 0
    assert sample_menu.current_item is sample_items[0]
    sample_menu.go_down()
    sample_menu.select()
    assert sample_menu.current_option == 1
    assert sample_menu.current_item is sample_items[1]
    sample_menu.go_down()
    sample_menu.select()
    assert sample_menu.current_option == 2
    assert type(sample_menu.current_item) == ExitItem
    sample_menu.join(timeout=10)
    assert not sample_menu.is_alive()

def test_go_to_exit(sample_menu: CursesMenu):
    sample_menu.go_to_exit()
    assert sample_menu.current_option == 2
    assert type(sample_menu.current_item) == ExitItem

def test_exit(sample_menu: CursesMenu):
    sample_menu.exit()
    sample_menu.join(timeout=10)
    assert not sample_menu.is_alive()

def test_basic_menu():
    menu = CursesMenu("Test Menu")
    menu.get_input = menu._exit_with_return
    menu.show()

def test_thread_stuff(sample_menu: CursesMenu):
    assert sample_menu.is_running()
    sample_menu.pause()
    assert not sample_menu.is_running()
    sample_menu.resume()
    assert sample_menu.is_running()

def test_append_while_running(sample_menu: CursesMenu):
    for i in range(12):
        new_item = MenuItem(f"item{i}")
        sample_menu.items.append(new_item)

def test_init():
    menu1 = CursesMenu()
    menu2 = CursesMenu("menu2", "test_init", show_exit_item=True)
    menu3 = CursesMenu(title="menu3", subtitle="test_init", show_exit_item=False)

    assert menu1.title == ""
    assert menu2.title == "menu2"
    assert menu3.title == "menu3"

    assert menu1.subtitle == ""
    assert menu2.subtitle == "test_init"
    assert menu3.subtitle == "test_init"

def test_null_screens_main_loop():
    menu = CursesMenu("menu", "empty menu", show_exit_item=False)
    CursesMenu.stdscr = None
    menu.get_input = menu._exit_with_return
    with pytest.raises((AttributeError, AssertionError)):
        menu._main_loop()

def test_repr(sample_menu: CursesMenu):
    assert (
        repr(sample_menu) == f"<{sample_menu.title}: {sample_menu.subtitle}. "
        f"{len(sample_menu.items)} items>"
    )

def test_draw_item(sample_menu):
    sample_menu.draw_item(0, sample_menu.items[0], "1")
    sample_menu.draw_item(0, sample_menu.items[0], None)

def test_resize(sample_menu):
    sample_menu.on_resize()

def test_deprecated_append(sample_menu):
    with pytest.deprecated_call():
        sample_menu.append_item(MenuItem("Deprecated Append Item"))

if __name__ == "__main__":
    pytest.main()


import pygame
class Animations:
    def __init__(self,rootAnimPath):
        self.rootAnimPath = rootAnimPath#the root animation path e.g. 'assets/animations' in the directory there are folders which contain the animations in images
        self.animationFrames = {}#the animation frames
        self.animationDatabase = {}#the animation database
        self.frame = 0#the current frame
        self.state = 'idle'#the current state

    def getAnimation(self,animName,frameDurations):#get the animation
        animationFrameData = []#the animation frame data
        n = 0#the frame number
        for frame in frameDurations:#for each frame in the frame durations
            animationFrameId = animName + str(n)#the animation frame id
            imgLoc = f"{self.rootAnimPath}/{animName}/{animationFrameId}.png"#the image location
            animationImage = pygame.image.load(imgLoc).convert()#load the image
            animationImage.set_colorkey((0,0,0))#set the color key
            self.animationFrames[animationFrameId] = animationImage.copy()#add the animation frame to the animation frames
            for i in range(frame):#for each frame in the frame duration
                animationFrameData.append(animationFrameId)#add the animation frame id to the animation frame data
            n += 1#increment the frame number
        self.animationDatabase[animName] = animationFrameData#add the animation frame data to the animation database

    def getImg(self):#get the image to draw
        if len(self.animationDatabase[self.state])-1 == self.frame:#if the current frame is the last frame
            self.frame = 0#reset frame
        else:
            self.frame += 1#increment the frame
        img = self.animationFrames[self.animationDatabase[self.state][self.frame]]#get the image
        return img#return the image
    @property
    def getCurrentImg(self):
        return self.animationDatabase[self.state][self.frame]#return the current image
    def changeState(self,newState):#change the state
        if self.state != newState:#if the state is not the same as the new state
            self.state = newState#set the state to the new state
            self.frame = 0#reset the frame
        return self.state,self.frame#return the state and frame


import torch
from torchvision import models
import os
import torchvision.transforms as transforms
from PIL import Image
from torch.utils.data import Dataset,DataLoader
import torch.nn as nn
from camera import takePic
import picamera as pic
import time

def Result(w_path):
    num_class=4

    model=models.mobilenet_v2(pretrained=True)
    model.classifier=nn.Linear(1280,num_class)

    model.load_state_dict(torch.load(w_path))

    trans_test = transforms.Compose([
        transforms.ToTensor(),
        transforms.Resize((224,224)),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])

    class CustomDataSet(Dataset):

        def __init__(self, main_dir, transform):
            self.main_dir = main_dir
            self.transform = transform

            all_imgs = os.listdir(main_dir)
            self.total_imgs = sorted(all_imgs)

        def __len__(self):
            return len(self.total_imgs)

        def __getitem__(self, idx):

            img_loc = os.path.join(self.main_dir, self.total_imgs[idx])
            image = Image.open(img_loc).convert("RGB")
            tensor_image = self.transform(image)

            return tensor_image

    test_data = CustomDataSet(takePic('/home/pi/Desktop/image/image.jpg'), transform=trans_test)
#    test_data = CustomDataSet('/home/pi/Desktop/image/', transform=trans_test)
    test_set = DataLoader(dataset = test_data, batch_size = 1)

    result =[]

    with torch.no_grad():
        result=[]
        for data in test_set:
            imgs = data
            pre_sum=0
            prediction=model(imgs)
            result.append(torch.argmax(prediction,1).tolist())

        print(result[0])

    return result[0] # result = [[label]], result[0] = [label]


import os
import time
from uuid import uuid4

import pyautogui

class ScreenCapture:
    def __init__(self, storage_path="screencaptures"):
        self.storage_path = storage_path
        self.storage_list = {}

    def make_print_screen(self):
        screenshot = pyautogui.screenshot()

        uuid = str(uuid4())
        filename = os.path.join(self.storage_path, (uuid + ".png"))

        screenshot.save(filename)

        self.storage_list.update(
            {
                uuid: {
                    "timestamp": time.time(),
                    "filename": filename,
                }
            }
        )
        return uuid

    def get_filename(self, uuid):
        return self.storage_list.get(uuid).get("filename")

    def remove_print_screen(self, id):
        file = self.storage_list.get(id)
        self.storage_list.pop(id)
        os.remove(file.get("filename"))

class ScreenClicker:
    def __init__(self):
        pass

    def click_at(self, x, y):
        pyautogui.click(x, y)