import torch
import pandas as pd
import numpy as np
import torch.nn as nn
import pickle
from pathlib import Path
from sklearn import metrics as sk_metrics
from tqdm import tqdm
import torch.optim as optim
import copy
import matplotlib.pyplot as plt

file_extension = ".csv"
parent_name ="checkpoints/"
output_directory ="outputs/"

class Loader(torch.utils.data.Dataset):
    def __init__(self):
        super(Loader, self).__init__()
        self.dataset = ''

    def __len__(self):
        return len(self.dataset)

    def __getitem__(self, idx):
        row = self.dataset.iloc[idx]
        row = row.drop(labels={'label'})
        data = torch.from_numpy(np.array(row)).float()
        return data

class Train_Loader(Loader):
    def __init__(self):
        super(Train_Loader, self).__init__()
        self.dataset = pd.read_csv(
                       'data/random_train.csv',
                       index_col=False
                       )

def ae_train(model, X_loader, l_r = 1e-2, w_d = 1e-5, n_epochs = 1, batch_size = 32, save_errors = True):
    device = 'cuda' if torch.cuda.is_available() else 'cpu'
    model.to(device)
    optimizer = torch.optim.SGD(model.parameters(), lr=l_r, weight_decay=w_d)
    criterion = nn.MSELoss(reduction='mean')
    model.train(True)
    errors = []
    for epoch in tqdm(range(n_epochs)):
        epoch_loss = []
        for step, batch in enumerate(X_loader):
            x_in = batch.type(torch.float32)
            x_in = x_in.to(device)
            x_out = model(x_in)
            loss = criterion(x_out, x_in)
            model.zero_grad()
            loss.backward()
            optimizer.step()
            torch.autograd.set_detect_anomaly(True)
            epoch_loss.append(loss.item())
        errors.append(sum(epoch_loss)/len(epoch_loss))
        print("epoch {}: {}".format(epoch+1, sum(epoch_loss)/len(epoch_loss)))
    if save_errors:
        np.savetxt(output_directory+"_training_loss_"+model.name+file_extension, errors, delimiter=',')

def mlp_train(model, X_train, y_train, X_val, y_val, l_r = 1e-2, w_d = 1e-5, n_epochs = 1, batch_size = 32, save_errors = True):
    # training parameters
    X_train = torch.tensor(X_train, dtype=torch.float32)
    y_train = torch.tensor(y_train, dtype=torch.float32).reshape(-1, 1)
    X_val = torch.tensor(X_val, dtype=torch.float32)
    y_val = torch.tensor(y_val, dtype=torch.float32).reshape(-1, 1)
    batch_start = torch.arange(0, len(X_train), batch_size)

    # Hold the best model
    best_mse = np.inf   # init to infinity
    best_weights = None
    history = []

    loss_fn = nn.MSELoss()  # mean square error
    optimizer = optim.Adam(model.regressor.parameters(), lr=0.0001)

    # training loop
    for epoch in range(n_epochs):
        model.train()
        with tqdm(batch_start, unit="batch", mininterval=0, disable=True) as bar:
            bar.set_description(f"Epoch {epoch}")
            for start in bar:
                # take a batch
                X_batch = X_train[start:start+batch_size]
                y_batch = y_train[start:start+batch_size]
                # forward pass
                y_pred = model(X_batch)
                loss = loss_fn(y_pred, y_batch)
                # backward pass
                optimizer.zero_grad()
                loss.backward()
                # update weights
                optimizer.step()
                # print progress
                bar.set_postfix(mse=float(loss))
        # evaluate accuracy at end of each epoch
        model.eval()
        y_pred = model(X_val)
        mse = loss_fn(y_pred, y_val)
        mse = float(mse)
        history.append(mse)
        if mse < best_mse:
            best_mse = mse
            best_weights = copy.deepcopy(model.state_dict())

    # restore model and return best accuracy
    model.load_state_dict(best_weights)
    print("MSE: %.5f" % best_mse)
    print("RMSE: %.5f" % np.sqrt(best_mse))
    #plt.plot(history)
    #plt.show()

def forest_train(model, X_train, y_train, max_depth = 3, save_errors = True):
    # training parameters

    model.regressor.fit(X_train, y_train)

def model_eval(model, x):
    loss_fn = nn.MSELoss()
    model.eval()
    x_pred = model(x)
    loss_val = loss_fn(x, x_pred)
    return loss_val

def compute_metrics(val_score, y_val, thresh, pos_label=1):
    """
    This function compute metrics for a given threshold

    Parameters
    ----------
    test_score
    y_test
    thresh
    pos_label

    Returns
    -------

    """
    y_pred = (val_score >= thresh).astype(int)
    y_true = y_val.astype(int)

    accuracy = sk_metrics.accuracy_score(y_true, y_pred)
    precision, recall, f_score, _ = sk_metrics.precision_recall_fscore_support(
        y_true, y_pred, average='binary', pos_label=pos_label
    )
    avgpr = sk_metrics.average_precision_score(y_true, val_score)
    roc = sk_metrics.roc_auc_score(y_true, val_score)
    cm = sk_metrics.confusion_matrix(y_true, y_pred, labels=[1, 0])

    return accuracy, precision, recall, f_score, roc, avgpr, cm

def estimate_optimal_threshold(val_score, y_val, pos_label=1, nq=100):
    ratio = 100 * sum(y_val == 0) / len(y_val)
    print("ratio", ratio)
    print(f"Ratio of normal data:{ratio}")
    q = np.linspace(ratio - 5, min(ratio + 5, 100), nq)
    thresholds = np.percentile(val_score, q)

    result_search = []
    confusion_matrices = []
    f1 = np.zeros(shape=nq)
    r = np.zeros(shape=nq)
    p = np.zeros(shape=nq)
    auc = np.zeros(shape=nq)
    aupr = np.zeros(shape=nq)
    qis = np.zeros(shape=nq)

    for i, (thresh, qi) in enumerate(zip(thresholds, q)):
        # print(f"Threshold :{thresh:.3f}--> {qi:.3f}")
        # Prediction using the threshold value
        accuracy, precision, recall, f_score, roc, avgpr, cm = compute_metrics(val_score, y_val, thresh, pos_label)

        confusion_matrices.append(cm)
        result_search.append([accuracy, precision, recall, f_score])
        # print(f"qi:{qi:.3f} ==> p:{precision:.3f}  r:{recall:.3f}  f1:{f_score:.3f}")
        f1[i] = f_score
        r[i] = recall
        p[i] = precision
        auc[i] = roc
        aupr[i] = avgpr
        qis[i] = qi

    arm = np.argmax(f1)

    return {
        "Precision": p[arm],
        "Recall": r[arm],
        "F1-Score": f1[arm],
        "AUPR": aupr[arm],
        "AUROC": auc[arm],
        "Thresh_star": thresholds[arm],
        "Quantile_star": qis[arm]
    }

def save(model):
    Path(parent_name).mkdir(parents=True, exist_ok=True)
    with open(parent_name+model.name+".pickle", "wb") as fp:
        pickle.dump(model.state_dict(), fp)

def load(model):
    with open(parent_name+model.name+".pickle", "rb") as fp:
        model.load_state_dict(pickle.load(fp))


# -*- coding: utf-8 -*-
"""
Created on Wed Apr 28 13:26:20 2021

@author: diego
"""

#import neccesary libraries
from progressbar import progressbar
import numpy as np
import random
import os
import tensorflow as tf # tensorflow 2.0 Google Deep learning release
import cv2 as cv
import imgaug.augmenters as iaa
import skimage
from skimage.morphology import disk
from skimage.filters import threshold_otsu, rank
from skimage.filters import try_all_threshold, threshold_minimum, threshold_mean
try:
    from . import hairReplacement
except:
    import hairReplacement
imageSize=128
seed=0
np.random.seed(seed) # fix random seed
tf.random.set_seed(seed)

kernel_size_for_edges=int(imageSize/10)#has to be odd

#allows the user to change adjust their desired processing method
######Takes a directory as input########
def adjustableProcessing(dataDir,labels,evenRatio=True, hairRemoval=True,hairRemovalMethod=3,borderRemoval=True,onlyBorderless=False,
                              segmentation=True,SegmentationMethod="otsu"):

    if borderRemoval==True and onlyBorderless==True:
        raise ValueError("Cannot have borderRemoval and onlyBorderless both set as True")
    randomIndices=[]
    counter=0
    melNum=0
    #declare array
    #imgArray=[]
    #colour arry
    imgArray=[]
            #get images and proccess them so the potential melanomas have more defined borders
    for file in progressbar(os.listdir(dataDir)):
        #get colour img
        colourImg=cv.resize(cv.imread(os.path.join(dataDir,file)), (300,300))#300 is the ideal size for hair removal method 2

        if hairRemoval==True:
            colourImg=hairReplacement.deleteHair(colourImg,method=hairRemovalMethod)

        colourImg= cv.resize(colourImg, (128,128))

        if borderRemoval==True:
            colourImg=BorderRemoval(colourImg)
        if onlyBorderless==True:
            if findBorders(colourImg)==True:#do this after hair removal so hairs do not count
            #remove labels and skip any following steps
                 labels=np.delete(labels, (counter), axis=0)
        #reshape img
        if segmentation==True:
            colourImg=Segmentation(colourImg,SegmentationMethod)

        #aka augmentation
        if evenRatio==True:
            #if the prediction is benign do not augment
            if labels[counter,1]==0:

                imgArray.append(np.float32(colourImg))
                counter=counter+1
            else:#duplicate and rotate twice and add to array
                #append original img
                imgArray.append(np.float32(colourImg))
                #append flipped and mirrored img
                imgArray.append(np.float32( cv.flip(colourImg, -1)))
                #append 90 deg. rotated img
                imgArray.append(np.float32(np.rot90(colourImg,axes=(0,1))))
                #append rotated and zoomed in img
                rotate=iaa.Affine(rotate=(-20, 20))
                rotatedImg=rotate.augment_image(colourImg)
                crop = iaa.Crop(percent=(0, 0.15)) # crop image (zoom in)
                imgArray.append(np.float32(crop.augment_image(rotatedImg)))
                #duplicate lables 3 times
                labels=np.insert(labels,counter,labels[counter],axis=0)
                labels=np.insert(labels,counter,labels[counter],axis=0)
                labels=np.insert(labels,counter,labels[counter],axis=0)

                counter=counter+4
                #keep track of the num of melanomas
                melNum=melNum+4
        else:
            imgArray.append(np.float32(colourImg))
    if evenRatio==True:
        #convert list to array
        imgArray=np.array(imgArray)
        nonMelIndex=np.where(labels[:,1]==0)
        nonMelIndex=np.asarray(nonMelIndex)
         #convert to 1d array and get the images with such index
        nonMelIndex=nonMelIndex[0,:]
        numOfExpendableImgs=(len(nonMelIndex)-melNum)
        print ("Number of non Mel. imgs: " + str(len(nonMelIndex)))

        print ("Number of non Mel. deleted to make a 1:1 ratio: " + str(numOfExpendableImgs))
        if numOfExpendableImgs >0:
            randomIndices=random.sample((nonMelIndex.tolist()),numOfExpendableImgs)# random (delete)
            #delete random non mel imgs
            labels=np.delete(labels, (randomIndices), axis=0)
            imgArray=np.delete(imgArray, (randomIndices), axis=0)
    else:
        #convert list to array
        imgArray=np.array(imgArray)
        labels=np.array(labels)

    return imgArray,labels,randomIndices#,imgArray

#allows the user to change adjust their desired processing method
######Takes an array of images as input########
def adjustableImageProcessing(imgs,labels,evenRatio=True, hairRemoval=True,hairRemovalMethod=3,hairLevels=None,
                              removalLowerBound=2,borderRemoval=True,
                              segmentation=True,imgSize=128,SegmentationMethod="otsu"):
    randomIndices=[]
    counter=0
    melNum=0
    noise_modes = ['gaussian', 'localvar', 'poisson', 'salt', 'pepper', 's&p', 'speckle']
    noise_mode = noise_modes[random.randint(0, 6)]
    #declare array
    #imgArray=[]
    #colour arry
    imgArray=[]
    imgCounter=0
        #get images and proccess them so the potential melanomas have more defined borders
    for img in progressbar(imgs):
        colourImg=img
        if hairRemoval==True:
            #only remove if there are hairs
            if hairLevels==None:

                colourImg=hairReplacement.deleteHair(colourImg,method=hairRemovalMethod)

            elif hairLevels[imgCounter]>removalLowerBound:#only delete hairs if image contains hairs
                colourImg=hairReplacement.deleteHair(colourImg,method=hairRemovalMethod)

        if borderRemoval==True:
            colourImg=BorderRemoval(colourImg)

        #reshape img
        colourImg= cv.resize(colourImg, (imgSize,imgSize))
        if segmentation==True:
            colourImg=Segmentation(colourImg,SegmentationMethod)
        #Augmentation
        if evenRatio==True:
            # if labels[counter,1]==0:
            #     imgArray.append(np.float32(colourImg))
            #     counter=counter+1
            # else:#duplicate and rotate twice and add to array
                #append og img
                imgArray.append(np.float32(colourImg))
                #append flipped and mirrored img
                #imgArray.append(np.float32( cv.flip(colourImg, -1)))
                #append 90 deg. rotated img cropped by 8%
                crop = iaa.Crop(percent=(0, 0.08)) # crop image
                croppedImg=crop.augment_image(colourImg)
                imgArray.append(np.float32(np.rot90(croppedImg,axes=(0,1))))
                #append rotated and zoomed in img
                # rotate=iaa.Affine(rotate=(-20, 20))
                # rotatedImg=rotate.augment_image(colourImg)
                # crop = iaa.Crop(percent=(0, 0.1)) # crop image
                # imgArray.append(np.float32(crop.augment_image(rotatedImg)))
                #add image with noise
                noisyImg = skimage.util.random_noise(colourImg, mode=noise_mode)
                rotate=iaa.Affine(rotate=(-180, 180))
                rotatedNoisyImg=rotate.augment_image(noisyImg)
                imgArray.append(np.float32(np.rot90(rotatedNoisyImg,axes=(0,1))))
                #duplicate groundtruth 3 times
                labels=np.insert(labels,counter,labels[counter],axis=0)
                labels=np.insert(labels,counter,labels[counter],axis=0)
                #labels=np.insert(labels,counter,labels[counter],axis=0)
                #labels=np.insert(labels,counter,labels[counter],axis=0)

                ###Change +3 if more images are added when augmented
                counter=counter+3
                #keep track of the num of melanomas
                melNum=melNum+3
        else:
            imgArray.append(np.float32(colourImg))
        imgCounter=imgCounter+1
    if evenRatio==True:
        #convert list to array
        imgArray=np.array(imgArray)
        melIndex=np.where(labels[:,1]==1)
        melIndex=np.asarray(melIndex)
         #convert to 1d array and get the images with such index
        melIndex=melIndex[0,:]

        nonMelIndex=np.where(labels[:,1]==0)
        nonMelIndex=np.asarray(nonMelIndex)
         #convert to 1d array and get the images with such index
        nonMelIndex=nonMelIndex[0,:]
        numOfExpendableImgs=(len(nonMelIndex)-len(melIndex))
        print("Number of melanoma pictures: %s" %len(melIndex))
        print ("Number of non Mel. imgs: " + str(len(nonMelIndex)))

        print ("Number of non Mel. deleted to make a 1:1 ratio: " + str(numOfExpendableImgs))
        if len(melIndex)<len(nonMelIndex):
            randomIndices=random.sample((nonMelIndex.tolist()),numOfExpendableImgs)# 6k random (delete)
        else:
            randomIndices=random.sample((melIndex.tolist()),numOfExpendableImgs*(-1))# 6k random (delete)

        #delete random non mel imgs
        labels=np.delete(labels, (randomIndices), axis=0)
        imgArray=np.delete(imgArray, (randomIndices), axis=0)
    else:
        #convert list to array
        imgArray=np.array(imgArray)
        labels=np.array(labels)

    return imgArray,labels,randomIndices#,imgArray
#use otsu to segment
def Segmentation(colourImg,method='otsu'):
        #black and white image
        colourImg=colourImg.astype('uint8')
        grayImg = cv.cvtColor(colourImg, cv.COLOR_BGR2GRAY)
        #otsu threshold removes backgroud and inter. like hair
        if method=='otsu':
            radius = 3
            selem = disk(radius)
            #apply otsu to img
            local_otsu = rank.otsu(grayImg, selem)
            threshold_global_otsu = threshold_otsu(grayImg)
            #use the global threshold to remove pixels that are not dark enough
            global_otsu = local_otsu >= threshold_global_otsu
            # mean_thresh = threshold_mean(grayImg)
            # #use the global threshold to remove pixels that are not dark enough
            # global_otsu = local_otsu >= mean_thresh
            #remove background from colour image
            colourImg[global_otsu[:,:]==True,:]=255
        else:
            print("No other methods avaliable at the moment")
        return colourImg

    #remove most part of the darkest borders which enhancest the segmenting process
def BorderRemoval(colourImg):
        blur = cv.GaussianBlur(colourImg, (5,5),2)#helps conserving dark values in lesions
        black=np.array([3,3,3])
        white=np.array([255,255,255])
        #conserve all the values lighter than rgb(3,3,3)
        mask = cv.inRange(blur, black, white)
        #trim edges
        kernel = cv.getStructuringElement(cv.MORPH_ELLIPSE, (int(kernel_size_for_edges),int(kernel_size_for_edges)))
        mask = cv.dilate(cv.bitwise_not(mask.astype('uint8')), kernel)
        mask =cv.bitwise_not(mask)

        if np.any(mask==0):

             #replace dark pixels with with pixels
             colourImg[mask[:,:]==0,:]=255
            # colourImg = cv.inpaint(colourImg, mask, inpaintRadius=25, flags=cv.INPAINT_TELEA)
        return colourImg
def findBorders(colourImg):
        blur = cv.GaussianBlur(colourImg, (5,5),2)#helps conserving small dark values in lesions
        black=np.array([3,3,3])
        white=np.array([255,255,255])
        #conserve all the values lighter than rgb(3,3,3)
        mask = cv.inRange(blur, black, white)
        #do not return an image if there are black values
        if np.any(mask==0):
            return True
        return False


import tensorflow as tf
from tensorflow.keras.layers import Conv2D, BatchNormalization, GlobalAveragePooling2D, Dense
import tensorflow_datasets as tfds
import time
import json
import numpy as np
import argparse

parser = argparse.ArgumentParser()
parser.add_argument("--seed", "-s", type=int, default=42, help="Random seed")
args = parser.parse_args()
tf.random.set_seed(args.seed)

# ResNet Layers
class IdentityResidual(tf.keras.layers.Layer):
    def __init__(self, out_channels, stride):
        super().__init__()
        self.out_channels = out_channels
        self.stride = stride

    def build(self, input_shape):
        b, h, w, in_channels = input_shape
        self.in_channels = in_channels
        self.h = h // self.stride
        self.w = w // self.stride
        self.b = b
        self.c = self.out_channels - self.in_channels

    def call(self, input_tensor):
        # Downsample spatially
        x = input_tensor[:, ::self.stride, ::self.stride, :]
        # Create padding tensor for extra channels
        if self.out_channels != self.in_channels:
            pad = tf.zeros((self.b, self.h, self.w, self.c))
            # Append padding to the downsampled identity
            x = tf.concat((x, pad), axis=-1)
        return x

class ResNetV2Layer(tf.keras.Model):
    def __init__(self, channels, stride=1):
        super().__init__()
        conv_kwargs = {
            "padding": "same",
            "use_bias": False
        }
        self.stride = stride
        self.channels = channels
        self.relu = tf.nn.relu
        self.residual = IdentityResidual(channels, stride)
        self.conv1 = Conv2D(filters=channels, kernel_size=3, strides=self.stride, **conv_kwargs)
        self.bn1 = BatchNormalization()
        self.conv2 = Conv2D(filters=channels, kernel_size=3, **conv_kwargs)
        self.bn2 = BatchNormalization()

    def call(self, input_tensor, training=False):
        residual = self.residual(input_tensor)
        x = self.bn1(input_tensor, training=training)
        x = self.relu(x)
        x = self.conv1(x)
        x = self.bn2(x, training=training)
        x = self.relu(x)
        x = self.conv2(x)
        return x + residual

class TimingCallback(tf.keras.callbacks.Callback):
    def __init__(self, logs={}):
        self.logs = []

        self.start_list = []
        self.end_list = []
        self.difference_list = []

        self.start_train_time = 0.0
        self.end_train_time = 0.0

        self.start_batch_inference_list = []
        self.end_batch_inference_list = []
        self.difference_batch_inference_list = []

        self.final_training_loss = 0.0
        self.final_evaluation_accuracy = 0.0

    def on_train_begin(self, logs={}):
        self.start_train_time = time.time()

    def on_train_end(self, logs={}):
        self.end_train_time = time.time()

    def on_test_batch_begin(self, batch, logs={}):
        self.starttime = time.time()
        self.start_batch_inference_list.append(self.starttime)

    def on_test_batch_end(self, batch, logs={}):
        self.endtime = time.time()
        self.end_batch_inference_list.append(self.endtime)
        self.difference_batch_inference_list.append(
            self.end_batch_inference_list[-1] - self.start_batch_inference_list[-1])

    def on_epoch_begin(self, epoch, logs={}):
        self.starttime = time.time()
        self.start_list.append(self.starttime)

    def on_epoch_end(self, epoch, logs={}):
        self.final_training_loss += logs["loss"]
        self.final_evaluation_accuracy += logs["sparse_categorical_accuracy"]
        self.endtime = time.time()
        self.end_list.append(self.endtime)
        self.difference_list.append(self.end_list[-1] - self.start_list[-1])

def get_datasets(batch_size):
    """
    Creates train, validation, and test datasets.
    Applies data normalization to all datasets and augmentation to training only.
    """
    train_ds, val_ds, test_ds = tfds.load(
        "cifar10",
        split=["train[:90%]", "train[90%:]", "test"],
        as_supervised=True,
        shuffle_files=True,
    )

    mean = [0.485, 0.456, 0.406]
    std = [0.229, 0.224, 0.225]
    var = [x ** 2 for x in std]

    augment_pipeline = tf.keras.Sequential([
        tf.keras.layers.Rescaling(scale=1./255),
        tf.keras.layers.Normalization(mean=mean, variance=var),
        tf.keras.layers.ZeroPadding2D(padding=(4, 4)),
        tf.keras.layers.RandomFlip(mode="horizontal"),
        tf.keras.layers.RandomCrop(height=32, width=32)
    ])

    evaluate_pipeline = tf.keras.Sequential([
        tf.keras.layers.Rescaling(scale=1./255),
        tf.keras.layers.Normalization(mean=mean, variance=var),
    ])

    augment_pipeline.compile()
    evaluate_pipeline.compile()

    AUTOTUNE = tf.data.AUTOTUNE

    train_ds = train_ds.cache().shuffle(1000).batch(batch_size, drop_remainder=True)
    train_ds = train_ds.map(lambda x, y: (augment_pipeline(x, training=True), y)).prefetch(AUTOTUNE)

    val_ds = val_ds.cache().shuffle(1000).batch(batch_size, drop_remainder=True)
    val_ds = val_ds.map(lambda x, y: (evaluate_pipeline(x, training=False), y)).prefetch(AUTOTUNE)

    test_ds = test_ds.cache().shuffle(1000).batch(batch_size, drop_remainder=True)
    test_ds = test_ds.map(lambda x, y: (evaluate_pipeline(x, training=False), y)).prefetch(AUTOTUNE)

    return train_ds, val_ds, test_ds

def get_compiled_model(batch_size):
    ResNetV2Model = tf.keras.Sequential([
        Conv2D(filters=16, kernel_size=3, padding="same", use_bias=False, data_format="channels_last"),
        ResNetV2Layer(16),
        ResNetV2Layer(16),
        ResNetV2Layer(16),
        ResNetV2Layer(32, stride=2),
        ResNetV2Layer(32),
        ResNetV2Layer(32),
        ResNetV2Layer(64, stride=2),
        ResNetV2Layer(64),
        ResNetV2Layer(64),
        GlobalAveragePooling2D(),
        Dense(10)
    ])

    # Model needs dummy input to build
    inputs = tf.random.normal((batch_size, 32, 32, 3))
    z = ResNetV2Model(inputs)

    ResNetV2Model.compile(
        optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3),
        loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
        metrics=[
            tf.keras.metrics.SparseCategoricalAccuracy(),
            tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
        ]
    )
    return ResNetV2Model

def main():
    BATCH_SIZE = 128
    train_ds, val_ds, test_ds = get_datasets(BATCH_SIZE)

    model = get_compiled_model(BATCH_SIZE)

    cb = TimingCallback()

    # Fitting the model
    history = model.fit(train_ds, epochs=10, validation_data=val_ds, callbacks=[cb])

    # Evaluate the model
    test_metrics = model.evaluate(test_ds)

    total_training_time = cb.end_train_time - cb.start_train_time
    average_epoch_training_time = np.mean(cb.difference_list)
    average_batch_inference_time = np.mean(cb.difference_batch_inference_list) * 1000
    final_eval_accuracy = history.history["val_sparse_categorical_accuracy"][-1]
    final_train_loss = history.history["loss"][-1]

    metrics = {
        "model_name": "ResNetV2-20",
        "framework_name": "TensorFlow",
        "dataset": "MNIST Digits",
        "task": "classification",
        "final_training_loss": final_train_loss,
        "final_evaluation_accuracy": final_eval_accuracy,
        "final_test_accuracy": test_metrics[1],  # Metrics is list of [loss, acc, ce]
        "total_training_time": total_training_time,  # in seconds
        "average_epoch_training_time": average_epoch_training_time,  # in seconds
        "average_batch_inference_time": average_batch_inference_time  # in milliseconds
    }

    for key, value in metrics.items():
        print(f'{key} : {value}')

    date_str = time.strftime("%Y-%m-%d-%H%M%S")
    with open(f"./output/m2-tensorflow-cnn-{date_str}.json", "w") as outfile:
        json.dump(metrics, outfile)

if __name__ == "__main__":
    main()


import torch
from torch import nn, optim
import pandas as pd
from torch.utils.data import Dataset, DataLoader
import numpy as np

import gc

class RNN(nn.Module):
    def __init__(self, input_size, hidden_size, num_layers):
        super(RNN, self).__init__()
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        self.lstm_1 = nn.LSTM(input_size=1, hidden_size=256, num_layers=1, batch_first=True)
        self.lstm_2 = nn.LSTM(input_size=256, hidden_size=128, num_layers=1, batch_first=True)
        self.lstm_3 = nn.LSTM(input_size=128, hidden_size=64, num_layers=1, batch_first=True)
        self.fc = nn.Linear(64, 1)

    def forward(self, x):
        # Set initial hidden and cell states
        # h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).cuda()
        # c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).cuda()

        # Forward propagate LSTM
        # out, _ = self.lstm_1(x, (h0, c0))  # out: tensor of shape (batch_size, seq_length, hidden_size)
        out, _ = self.lstm_1(x)  # out: tensor of shape (batch_size, seq_length, hidden_size)
        out = nn.Dropout(p=0.2)(out)
        out, _ = self.lstm_2(out)
        out = nn.Dropout(p=0.2)(out)
        out, _ = self.lstm_3(out)
        out = nn.Dropout(p=0.2)(out)

        # Decode the hidden state of the last time step
        out = self.fc(out[:, -1, :])
        return out
        # return out[:, -1, :]

class Model(nn.Module):
    def __init__(self, city_label_size=31, item_cat_size=84, shop_id_size=60):
        super().__init__()

        def embedding_network(input_size, emb_size=64):
            return nn.Sequential(
                nn.Linear(input_size, 128),
                nn.ReLU(inplace=True),
                nn.Dropout(0.4),
                nn.Linear(128, emb_size)
            )

        self.city_label_size = city_label_size
        self.item_cat_size = item_cat_size
        self.shop_id_size = shop_id_size

        self.city_embedding = embedding_network(self.city_label_size, emb_size=64)
        self.item_cat_embedding = embedding_network(self.item_cat_size, emb_size=64)
        self.shop_id_embedding = embedding_network(self.shop_id_size, emb_size=64)
        self.lstm = RNN(input_size=1, hidden_size=64, num_layers=2)

        self.fcnn = nn.Sequential(
                        nn.Linear(1+(64*3), 32),
                        nn.ReLU(inplace=True),
                        nn.Dropout(0.3),
                        nn.Linear(32, 1)
        )

    def forward(self, item_count_ts, item_cat_vector, shop_id_vector, city_label_vector):
        city_emb = self.city_embedding(city_label_vector)
        item_cat_emb = self.item_cat_embedding(item_cat_vector)
        shop_id_emb = self.shop_id_embedding(shop_id_vector)
        lstm_out = self.lstm(item_count_ts.unsqueeze(2))

        # # concat all
        concat_vector = torch.cat((lstm_out, city_emb), 1)
        return self.fcnn(concat_vector)

class SalesDataset(Dataset):
    def __init__(self, sales_train, sales_clean, test, is_train):
        self.sales_train = pd.read_csv(sales_train)
        self.sales_clean = pd.read_csv(sales_clean)
        self.test = pd.read_csv(test)
        self.is_train = is_train

        # removing shop id and item id which are not in test
        test_shop_ids = self.test['shop_id'].unique()
        test_item_ids = self.test['item_id'].unique()
        # Only shops that exist in test set.
        self.sales_train = self.sales_train[self.sales_train['shop_id'].isin(test_shop_ids)]
        # Only items that exist in test set.
        self.sales_train = self.sales_train[self.sales_train['item_id'].isin(test_item_ids)]
        self.sales_train['date'] = pd.to_datetime(self.sales_train['date'], format='%d.%m.%Y')
        # now we will create a pivot tabel by going so we get our data in desired form
        # we want get total count value of an item over the whole month for a shop
        # That why we made shop_id and item_id our indices and date_block_num our column
        # the value we want is item_cnt_day and used sum as aggregating function
        self.dataset = self.sales_train.pivot_table(index=['shop_id', 'item_id'], values=['item_cnt_day'],
                                                    columns=['date_block_num'], fill_value=0, aggfunc='sum')
        self.dataset.reset_index(inplace=True)

        if not self.is_train:
            # Now we will merge our pivot table with the test_data because we want to keep the data of items we have predict
            self.dataset = pd.merge(self.test, self.dataset, on=['item_id', 'shop_id'], how='left')

        extra = self.sales_clean[['shop_id', 'item_id', 'city_label', 'item_category_id', 'holidays_in_month']]
        extra = extra.drop_duplicates(['item_id', 'shop_id'], keep='last')
        # price = self.sales_train[['shop_id', 'item_id', 'item_price']]
        # price = price.drop_duplicates(['item_id', 'shop_id'], keep='last')

        # merging extra
        self.dataset = pd.merge(self.dataset, extra, on=['item_id', 'shop_id'], how='left')
        # self.dataset = pd.merge(self.dataset, price, on=['item_id', 'shop_id'], how='left')
        self.dataset.fillna(0, inplace=True)
        self.dataset.clip(0, )

        self.max_item_cat = int(max(self.dataset['item_category_id'])) + 1
        self.max_shop_id = int(max(self.dataset['shop_id'])) + 1
        self.max_city_label = int(max(self.dataset['city_label'])) + 1

        del self.sales_train
        del self.sales_clean
        gc.collect()

    def __len__(self):
        return len(self.dataset)

    def __getitem__(self, idx):
        row = self.dataset.iloc[idx]

        # one hot encoding for item_cat
        item_cat_vector = [0 for _ in range(self.max_item_cat)]
        item_cat_vector[int(row['item_category_id'])] = 1

        # one hot encoding for shop_id
        shop_id_vector = [0 for _ in range(self.max_shop_id)]
        shop_id_vector[int(row['shop_id'])] = 1

        # one hot encoding for city_label
        city_label_vector = [0 for _ in range(self.max_city_label)]
        city_label_vector[int(row['city_label'])] = 1

        if self.is_train:
            item_count_ts = list(row[3: 36])
            target = row[37]
        else:
            item_count_ts = list(row[4: 37])
            target = 0

        item_count_ts = torch.FloatTensor(item_count_ts)
        item_cat_vector = torch.FloatTensor(item_cat_vector)
        shop_id_vector = torch.FloatTensor(shop_id_vector)
        city_label_vector = torch.FloatTensor(city_label_vector)
        target = torch.FloatTensor([target])

        return item_count_ts, item_cat_vector, shop_id_vector, city_label_vector, target

    def get_test_data(self):
        return self.test

is_cuda = torch.cuda.is_available()
print("Is cuda availiable: ", is_cuda)

train_sales_dataset = SalesDataset(sales_train='../data/sales_train.csv',
                             sales_clean='../data/cleaned_sales.csv',
                             test='../data/test.csv',
                             is_train=True)

test_sales_dataset = SalesDataset(sales_train='../data/sales_train.csv',
                             sales_clean='../data/cleaned_sales.csv',
                             test='../data/test.csv',
                             is_train=False)

train_dataloader = DataLoader(train_sales_dataset, batch_size=4096, num_workers=4)
test_dataloader = DataLoader(test_sales_dataset, batch_size=4096, num_workers=4)

model = Model()
print(model)
no_epochs = 10
criterion = nn.MSELoss(reduction="mean")
optimizer = optim.RMSprop(model.parameters(), lr=0.001)

if is_cuda:
    model.cuda()

model.train()
for epoch in range(no_epochs):

    losses = []
    for idx, (item_count_ts, item_cat_vector, shop_id_vector, city_label_vector, target) in enumerate(train_dataloader):

        optimizer.zero_grad()
        if is_cuda:
            item_count_ts, item_cat_vector, shop_id_vector, city_label_vector, target = \
                item_count_ts.cuda(), item_cat_vector.cuda(), shop_id_vector.cuda(), city_label_vector.cuda(), target.cuda()

        pred = model(item_count_ts, item_cat_vector, shop_id_vector, city_label_vector)
        loss = criterion(pred, target)
        loss.backward()
        optimizer.step()

        losses.append(loss.item())

    print("Epoch: {} MSE Loss: {}".format(epoch+1, sum(losses)/len(losses)))

torch.save(model.state_dict(), 'hybrid_lstm')

model.eval()
submission_pfs = np.array([])
for idx, (item_count_ts, item_cat_vector, shop_id_vector, city_label_vector, target) in enumerate(test_dataloader):

    if is_cuda:
        item_count_ts, item_cat_vector, shop_id_vector, city_label_vector, target = \
            item_count_ts.cuda(), item_cat_vector.cuda(), shop_id_vector.cuda(), city_label_vector.cuda(), target.cuda()

    pred = model(item_count_ts, item_cat_vector, shop_id_vector, city_label_vector)
    submission_pfs = np.append(submission_pfs, pred.data.cpu().numpy())

# creating submission file
# we will keep every value between 0 and 20
submission_pfs = submission_pfs.clip(0,20)
test = train_sales_dataset.get_test_data()
# creating dataframe with required columns
submission = pd.DataFrame({'ID': test['ID'],'item_cnt_month':submission_pfs.ravel()})
# creating csv file from dataframe
submission.to_csv('hyb_sub_pfs.csv',index = False)


from brax.v1.experimental.braxlines.experiments import defaults

ENV_NAMES = ('ant', 'halfcheetah', 'humanoid', 'hopper', 'walker2d')
AGENT_MODULE = 'brax.experimental.braxlines.vgcrl.train'
CONFIG = [
    dict(
        env_name=[env_name],
        obs_indices='vel',
        algo_name=['gcrl', 'diayn', 'cdiayn', 'diayn_full'],
        obs_scale=5.0,
        seed=list(range(10)),
        normalize_obs_for_disc=False,
        evaluate_mi=True,
        evaluate_lgr=True,
        env_reward_multiplier=0.0,
        spectral_norm=True,
        ppo_params=defaults.get_ppo_params(env_name, 2))
    for env_name in ENV_NAMES
]


import pygame
import random
from lerp import lerp
from dudepose import DudePose

pygame.init()

DISPLAY_WIDTH = 600
DISPLAY_HEIGHT = 400

BLACK = (0,0,0)
WHITE = (255,255,255)

gameDisplay = pygame.display.set_mode((DISPLAY_WIDTH,DISPLAY_HEIGHT))
pygame.display.set_caption('My Cool Graphics')
clock = pygame.time.Clock()

families = []

class Tempo:
  startTime = 0
  msPerBeat = 500
  beatsPerMeasure = 4

  def __init__(self, beatsPerMinute, beatsPerMeasure, delay=0):

    self.msPerBeat = 1/(beatsPerMinute/(60*1000))

    self.startTime = pygame.time.get_ticks() + (delay * self.msPerBeat)
    self.beatsPerMeasure = beatsPerMeasure

  def getBeat(self):
    beats = self.getTotalBeats()
    fraction = beats - int(beats)
    whole = int(beats)
    measured = whole % self.beatsPerMeasure

    return measured + fraction

  def getTotalBeats(self):
    now = pygame.time.get_ticks()
    deltaTime = now - self.startTime
    beats = deltaTime/self.msPerBeat

    return beats

  def drawTempo(self, gameDisplay, position):
    beat = self.getBeat()
    beat = int(beat)

    for i in range(self.beatsPerMeasure):
      if i <= beat:
        pygame.draw.circle(gameDisplay, (0, 255, 0), [position[0] + i * 20, position[1]], 5)

pygame.mixer.init()
pygame.mixer.music.load('Village-People-YMCA.mp3')
pygame.mixer.music.play()

# create the shared Tempo object.
TEMPO = Tempo(126,4,2)

class Dude:
  animationFinished = True

  def __init__(self, position, size):
    self.position = position
    self.size = size
    self.leftHanded = random.randrange(100) < 20
    # default the time to a random time so that all animations don't start at exactly
    # the same time
    self.time = pygame.time.get_ticks() + random.randrange(3000)
    self.frameCount = 0

  def getExtentsRect(self):
    return pygame.Rect([0, 0, 16*self.size, 16*self.size])

  def drawLine(self, gameDisplay, startingPoint, endingPoint, color = (0,0,0)):
    scale = self.size
    posAdjusted = (self.position[0], self.position[1] - (16*scale))

    pygame.draw.line(gameDisplay, color, (posAdjusted[0]+(startingPoint[0] * scale),posAdjusted[1]+(startingPoint[1] * scale)), (posAdjusted[0]+(endingPoint[0] * scale),posAdjusted[1]+(endingPoint[1] * scale)))

  def drawCircle(self, gameDisplay, centerPoint, radius, color = (0,0,0), lineWidth=1):
    scale = self.size
    posAdjusted = (self.position[0], self.position[1] - (16*scale))

    pygame.draw.circle(gameDisplay, color, [posAdjusted[0] + (centerPoint[0]*scale),posAdjusted[1] + (centerPoint[1]*scale)], radius*scale, lineWidth)

  def drawStanding(self, gameDisplay):

    # this was just to show you that we could possibly do this another way.
    # ignore for now
    # L8,4,8,12,2,2,8,6,14,2,8,6,8,12,4,14,8,1,12,14
    # C8,2,2

    # body
    self.drawLine(gameDisplay, (8,4), (8,12) )

    # left arm
    self.drawLine(gameDisplay, (2,2), (8,6))

    # right arm

    self.drawLine(gameDisplay, (14,2), (8,6))

    # left leg

    self.drawLine(gameDisplay, (8,12), (4,16))
    # right leg

    self.drawLine(gameDisplay, (8,12), (12,16))

    # draw head
    self.drawCircle(gameDisplay, (8,2), 2)

    # wouldn't it be nice to just be able to do this instead!
    # [[(8,12), (4,6)], [(8,12), (12,16)]]

  def drawTap(self, gameDisplay):

    # body
    self.drawLine(gameDisplay, (8,4), (8.5,12) )

    # left arm
    self.drawLine(gameDisplay, (3,3), (8,6))
    self.drawLine(gameDisplay, (2,3), (3,3))

    # right arm

    self.drawLine(gameDisplay, (14,2), (8,6))

    # left leg

    self.drawLine(gameDisplay, (8.5,12), (4,16))
    # right leg

    self.drawLine(gameDisplay, (8.5,12), (11,14))
    self.drawLine(gameDisplay, (11,14), (12,16))

    # draw head
    self.drawCircle(gameDisplay, (8,2), 2)

  def drawM(self, gameDisplay):

    # draw the M in YMCA

    # body
    self.drawLine(gameDisplay, (8,4), (8,12) )

    # left arm
    self.drawLine(gameDisplay, (4,6), (8,6))
    self.drawLine(gameDisplay, (4,6), (4,8))

    # right arm

    self.drawLine(gameDisplay, (12,6), (8,6))
    self.drawLine(gameDisplay, (12,6), (12,8))

    # left leg

    self.drawLine(gameDisplay, (8,12), (4,16))
    # right leg

    self.drawLine(gameDisplay, (8,12), (12,16))

    # draw head
    self.drawCircle(gameDisplay, (8,2), 2, (255,0,0))

  def drawC(self, gameDisplay):

    # draw the C in YMCA

    # body
    self.drawLine(gameDisplay, (8,4), (8,12) )

    # left arm
    self.drawLine(gameDisplay, (4,6), (8,6))
    self.drawLine(gameDisplay, (4,6), (4,0))
    self.drawLine(gameDisplay, (4,0), (12,0))

    # right arm

    self.drawLine(gameDisplay, (12,6), (8,6))

    # left leg

    self.drawLine(gameDisplay, (8,12), (4,16))
    # right leg

    self.drawLine(gameDisplay, (8,12), (12,16))

    # draw head
    self.drawCircle(gameDisplay, (8,2), 1.9, (0,255,0))

  def drawA(self, gameDisplay):

    # draw the A in YMCA
    # body
    self.drawLine(gameDisplay, (8,4), (8,12) )

    # left arm
    self.drawLine(gameDisplay, (4,6), (8,6))
    self.drawLine(gameDisplay, (4,6), (7,1))

    # right arm

    self.drawLine(gameDisplay, (12,6), (8,6))
    self.drawLine(gameDisplay, (12,6), (9,1))

    # left leg

    self.drawLine(gameDisplay, (8,12), (4,16))
    # right leg

    self.drawLine(gameDisplay, (8,12), (12,16))

    # draw head
    self.drawCircle(gameDisplay, (8,2), 2, (255,0,0))

  def animateYMCA(self, gameDisplay):

    beat = TEMPO.getBeat()
    totalBeats = TEMPO.getTotalBeats()

    if beat < 2:
      self.drawStanding(gameDisplay)
    elif beat < 3:
      self.drawM(gameDisplay)
    elif beat < 3.5:
      self.drawC(gameDisplay)
    else:
      self.drawA(gameDisplay)

    if totalBeats - self.startingBeat >= 4:
      # we're done with our animation, so return True
      return True

    # we're not done with our animation so return False
    return False

  def animateStanding(self, gameDisplay):
    # the standing animation will have the Dude tap their hand

    beat = TEMPO.getBeat()
    totalBeats = TEMPO.getTotalBeats()

    if beat - int(beat) < .2:
      self.drawTap(gameDisplay)
    else:
      self.drawStanding(gameDisplay)

    if totalBeats - self.startingBeat >= 4:
      # we're done with our animation, so return True
      return True

    return False

  def draw(self,gameDisplay):
    # pose = DudePose(headRadius=4)
    # pose.draw(gameDisplay, self.position, self.size)

    pose = DudePose()

    poseM = DudePose(lowerArmLeft=[[12,6],[12,8]],
                      upperArmLeft=[[8,6],[12,6]],
                      lowerArmRight=[[4,6],[4,8]],
                      upperArmRight=[[8,6],[4,6]])

    poseBigHead = DudePose(headRadius=3)

    poseRunning1 = DudePose(upperLegLeft=[[8,12], [8,14]],
                            lowerLegLeft=[[8,14], [8,16]],
                            upperLegRight=[[8,12],[10,14]],
                            lowerLegRight=[[10,14],[9,16]])

    # pose.draw(gameDisplay, self.position, self.size)

    # poseHalfWay = DudePose.lerp(pose, poseM, .25)

    # poseHalfWay.draw(gameDisplay, self.position, self.size)

    beat = TEMPO.getBeat()

    if beat - int(beat) < .5:
        headBeat = DudePose.lerp(pose, poseBigHead, (beat-int(beat))*2, pose, ["headRadius"])
    else:
        headBeat = DudePose.lerp(poseBigHead, pose, (beat-int(beat)-.5)*2, pose, ["headRadius"])

    if beat - int(beat) < 1:
        lerp = DudePose.lerp(pose, poseRunning1, (beat-int(beat)), headBeat, ["upperLegLeft", "lowerLegLeft", "upperLegRight", "lowerLegRight"])

    if beat < 1:
      lerp.draw(gameDisplay, self.position, self.size)
    elif beat < 1.5:
      lerpPose = DudePose.lerp(pose, poseM, (beat-1)*2, lerp, ["lowerArmLeft", "upperArmLeft", "lowerArmRight", "upperArmRight"])
      lerpPose.draw(gameDisplay, self.position, self.size)
    else:
      lerpPose = DudePose.lerp(pose, poseM, 1, lerp, ["lowerArmLeft", "upperArmLeft", "lowerArmRight", "upperArmRight"])
      lerpPose.draw(gameDisplay, self.position, self.size)

    # if self.animationFinished:
    #   animations = [self.animateStanding, self.animateYMCA, self.animateYMCA, self.animateYMCA]
    #   self.startingBeat = TEMPO.getTotalBeats()

    #   # alternate between the standing animation and the YMCA
    #   # animation

    #   self.animation = animations[self.frameCount % len(animations)]
    #   self.frameCount = self.frameCount + 1

    # if self.animation:
    #   self.animationFinished = self.animation(gameDisplay)

class Family:
  def __init__(self, position, size):
    self.position = position
    self.size = size

    self.members = []

    x = position[0]
    # 80% chance that we have a mom
    if random.randrange(100) < 80:
      mom = Dude((x, position[1]), 5 * size)
      self.members.append(mom)
      x = x + mom.getExtentsRect().width
    # 80% chance that we have a dad
    if random.randrange(100) < 80:
      dad = Dude((x, position[1]), 4 * size)
      self.members.append(dad)
      x = x + dad.getExtentsRect().width

    for i in range(random.randrange(4)):
      kid = Dude((x, position[1]), (random.randrange(3) + 1) * size)
      self.members.append(kid)
      x = x + kid.getExtentsRect().width

  def getExtentsRect(self):
    width = 0
    height = 0
    for i in self.members:
      memberRect = i.getExtentsRect()
      width = width + memberRect.width
      if memberRect.height > height:
        height = memberRect.height

    return pygame.Rect([0, 0, width, height])

  def draw(self, gameDisplay):
    for i in self.members:
      i.draw(gameDisplay)

def setup():
  gameDisplay.fill(WHITE)
  pygame.display.update()

  global families

  rows = 5
  for y in range(rows):
    columns = 7 * (y+1)
    # columns = 10
    if y % 2 == 1:
      xPos = 25
    else:
      xPos = 0
    for x in range(columns):
      family = Family((xPos,DISPLAY_HEIGHT - 200 * (y/rows)), 1 - y/rows)
      xPos = xPos + family.getExtentsRect().width
      families.append(family)

  # family = Family((100,100), 1)
  # families.append(family)

def graphicsLoop():
    gameExit = False

    while not gameExit:

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                print("Goodbye.")
                quit()

        # erase the screen on each loop by filling the screen white
        gameDisplay.fill(WHITE)

        TEMPO.drawTempo(gameDisplay, (10,10))
        for i in families:
          i.draw(gameDisplay)

        # startingColor = (255,127,0)
        # endingColor = (0,127,255)

        # for i in range(100):
        #   pygame.draw.line(gameDisplay, lerp(startingColor, endingColor, i/100),[100 + i,0], [100+i, 100])

        pygame.display.update()

        # limit the refresh rate to 60 fps
        clock.tick(60)

print(lerp((0,0,0), (255,255,255), 2))

setup()
graphicsLoop()


from setuptools import find_packages
from setuptools import setup

install_requires = [
  'numpy',
  'scipy',
]

tests_require = [
  'pytest',
]

setup(name='pyAudioGraphInput',
      version='0.1',
      description='Python Audio Graph Input',
      author='Bruno Di Giorgi',
      author_email='bruno@brunodigiorgi.it',
      url='https://github.com/brunodigiorgi/pyAudioGraphInput',
      license="GPLv2",
      packages=find_packages(),
      include_package_data=False,
      zip_safe=False,
      install_requires=install_requires,
      extras_require={
        'testing': tests_require,
      },
      )


from django.views.decorators.csrf import csrf_exempt
from django.http import HttpResponse
from . import ip_config
import requests
import json

def graphql_request(request):
    if request.method == 'POST':
        data = request.body
        headers = {'Content-Type':'application/graphql'}
        response = requests.post(ip_config.CD_IP+'/graphql',
        data = data, headers = headers)
    return HttpResponse(response.text, content_type='application/graphql')

@csrf_exempt
def set_concierge_ip(request):
    data = json.loads(request.body.decode("utf-8"))
    print(ip_config.CD_IP)
    ip_config.CD_IP = data['ip']
    print(ip_config.CD_IP)
    return HttpResponse(status=204)


import nltk

class FeatureFactory(object):

    def __init__(self):
        self.featureFuncList = [
            self.pos_tag,
            self.string_normal,
        ]

        self.pos_tagger = nltk.pos_tag

    def getFeatures(self, tokenSeq):
        featureSeqs = []
        for featureFunc in self.featureFuncList:
            featureSeq = featureFunc(tokenSeq)
            assert len(featureSeq) == len(tokenSeq)
            featureSeqs.append(featureSeq)
        return featureSeqs

    def pos_tag(self, text_seq):
        return tuple(zip(*self.pos_tagger(text_seq)))[1]

    def string_normal(self, text_seq):
        return tuple(map(lambda w: self.normalization(w), text_seq))

    @staticmethod
    def normalization(word):
        normal_word = ''
        for char in word:
            if char.isupper():
                normal_word += 'A'
            elif char.islower():
                normal_word += 'a'
            elif char.isdigit():
                normal_word += '1'
            else:
                normal_word += '0'
        return normal_word

    def generateCoNLL(self, filename, outputName):
        with open(filename, 'r', encoding='utf-8') as inputFile:
            with open(outputName, 'w', encoding='utf-8') as outputFile:
                tokenSeq = []
                labelSeq = []

                for line in inputFile:
                    if line.startswith('<eos>') or not line.strip():
                        featureSeqs = self.getFeatures(tokenSeq)

                        for i in range(len(tokenSeq)):
                            write_line = tokenSeq[i] + '\t' + '\t'.join(map(lambda f: str(f[i]), featureSeqs)) + '\t' + \
                                         labelSeq[i] + '\n'
                            outputFile.write(write_line)

                        outputFile.write('\n')

                        tokenSeq = []
                        labelSeq = []
                    else:
                        label, token, after = line.strip('\n').split('\t')
                        tokenSeq.append(token)
                        labelSeq.append(labelSeq)

if __name__ == '__main__':
    featureFactory = FeatureFactory()
    featureFactory.generateCoNLL('../data/en_with_types/output-00001-of-00100', 'en_train_01.txt')


from collections import Counter
class Solution:
    def countPairs(self, deliciousness: List[int]) -> int:
        """LC 1711 Count Good Meals
        Time Complexity: O(22N)
        Space: O(N)
        Difficuty: Medium
        """
        res = 0
        counter = Counter()
        mod = 10**9 + 7
        for num in deliciousness:
            for i in range(22):
                c =  counter.get((1 << i) - num)
                if c is not None:
                    res += c
                    res %= mod
            counter[num] += 1
        return res


import torch.nn as nn
import torch.nn.functional as F
import math
import torch
from torch.nn.parameter import Parameter
from torch.nn.modules.module import Module

class SparseMM(torch.autograd.Function):
    """
    Sparse x dense matrix multiplication with autograd support.

    Implementation by Soumith Chintala:
    https://discuss.pytorch.org/t/
    does-pytorch-support-autograd-on-sparse-matrix/6156/7
    """

    def __init__(self, sparse):
        super(SparseMM, self).__init__()
        self.sparse = sparse

    def forward(self, dense):
        return torch.mm(self.sparse, dense)

    def backward(self, grad_output):
        grad_input = None
        if self.needs_input_grad[0]:
            grad_input = torch.mm(self.sparse.t(), grad_output)
        return grad_input

class GraphConvolution(Module):
    """
    Simple GCN layer, similar to https://arxiv.org/abs/1609.02907
    """

    def __init__(self, in_features, out_features, bias=True):
        super(GraphConvolution, self).__init__()
        self.in_features = in_features
        self.out_features = out_features
        self.weight = Parameter(torch.Tensor(in_features, out_features))
        if bias:
            self.bias = Parameter(torch.Tensor(out_features))
        else:
            self.register_parameter('bias', None)
        self.reset_parameters()

    def reset_parameters(self):
        stdv = 1. / math.sqrt(self.weight.size(1))
        self.weight.data.uniform_(-stdv, stdv)
        if self.bias is not None:
            self.bias.data.uniform_(-stdv, stdv)

    def forward(self, input, adj):
        support = torch.mm(input, self.weight)
        output = SparseMM(adj)(support)
        if self.bias is not None:
            return output + self.bias
        else:
            return output

    def __repr__(self):
        return self.__class__.__name__ + ' (' \
               + str(self.in_features) + ' -> ' \
               + str(self.out_features) + ')'

class GCN(nn.Module):
    def __init__(self, nfeat, nhid, nclass, dropout):
        super(GCN, self).__init__()

        # self.gc = []
        self.gc1 = GraphConvolution(nfeat, nhid)
        # for ite in range(nlayer - 2):
            # self.gc.append(GraphConvolution(nhid, nhid))
        self.gc2 = GraphConvolution(nhid, nclass)
        self.dropout = dropout

    def forward(self, x, adj):

        #for op in self.gc[:-1]:
        x = F.relu(self.gc1(x, adj))
        # x = F.dropout(x, self.dropout, training=self.training)
        x = F.softmax(self.gc2(x, adj), dim=-1)
        m = torch.distributions.Categorical(x)
        return m, m.sample()


# -*- coding: utf-8 -*-
# __author__ = 'konghualei, 20170421'
from mainapp.controllers.project.meta_controller import MetaController
from mainapp.libs.param_pack import group_detail_sort
from bson import ObjectId
import datetime
import shutil
import re,os
import time
from biocluster.workflow import Workflow
from biocluster.config import Config
from bson import SON

class ToolNetworkWorkflow(Workflow):
    """
    基因集venn图
    """
    def __init__(self, wsheet_object):
        self._sheet = wsheet_object
        super(ToolNetworkWorkflow, self).__init__(wsheet_object)
        options = [
            {"name": "task_id", "type": "string"},
            {"name": "submit_location", "type": "string"},
            {"name": "update_info", "type": "string"},
            {'name': 'group_dict', 'type': 'string'},
            {"name": "network_file", "type": "string"},
            {"name": "exp_id", "type": "string"},
            {"name": "cmp_list", "type": "string"},
            {'name': "gene_type", 'type': 'string'},
            {'name': "significant", 'type': 'string'},
            {'name': "regulate", 'type': 'string'},
            {'name': "stat_type", 'type': 'string'},
            {'name': "stat_type_operation", 'type': 'string'},
            {'name': "stat_type_value", 'type': 'string'},
            {'name': "log2fc", 'type': 'string'},
            {'name': "log2fc_operation", 'type': 'string'},
            {'name': "log2fc_value", 'type': 'string'},
            {'name': 'tool_type', 'type': 'string'},
            {'name': 'geneset_id', 'type': 'string'},
            {"name": "main_id", "type": "string"},
            {"name": "relate_name", "type": "string"},
            {'name': 'params', 'type': 'string'}
        ]
        self.add_option(options)
        self.set_options(self._sheet.options())

    def run(self):
        self.start_listener()
        self.fire("start")
        self.set_db()

    def end(self):
        result_dir = self.add_upload_dir(self.output_dir)
        try:
            main_id = ObjectId(self.option('main_id'))
        except:
            pass
        main_info = dict(
            task_id=self.option('task_id'),
            project_type='prok_rna',
            params=self.option('params'),
            status="end",
            main_id=main_id,
            relate_name=self.option('relate_name'),
            tool_type=self.option('tool_type'),
            relate_id=main_id,
            file_path=self.s3_path
        )
        collection_name = 'tool_thurl'
        project_type = 'tool_lab'
        db = Config().get_mongo_client(mtype=project_type)[Config().get_mongo_dbname(project_type)]
        main_id_ = db[collection_name].insert_one(SON(main_info)).inserted_id
        conn = db[collection_name]
        task_id = conn.find_one({'_id': main_id_})['task_id']
        conn.update({'_id': main_id_, "task_id": task_id}, {"$set": {'main_id': main_id_}}, upsert=True)
        super(ToolNetworkWorkflow, self).end()

    def set_db(self):
        api = self.api.api("prok_rna.tool_lab_api")
        self.s3_path = '{}/{}'.format(self._sheet.output, 'network.txt')
        api.add_network(self.option('main_id'))
        self.set_output()

    def set_output(self):
        file_path = os.path.join(self.work_dir, 'network.txt')
        os.link(file_path, os.path.join(self.output_dir, 'network.txt'))
        self.end()


from enum import Enum
from partition import Partition, PartitionKind
import templates
from utils import concat

class TableKind(str, Enum):
    GPT = 'gpt'
    MBR = 'msdos'

    def script(self, device: str):
        return templates.PARTED.substitute(
            device=device,
            command=f'mklabel {self}')

class Table:
    kind: TableKind
    partitions: list[Partition]

    def __init__(self, kind: TableKind, partitions: list[Partition]):
        self.kind = kind
        self.partitions = partitions

    def partition_script(self, device: str) -> str:
        cmds = [self.kind.script(device)]
        pointer = 1
        for partition in self.partitions:
            cmds.append(partition.script(device, pointer))
            if partition.size_mb: pointer += partition.size_mb
            else: break
        return concat(cmds)

    def format_script(self, device: str) -> str:
        cmds = []
        for i, partition in enumerate(self.partitions):
            cmds.append(partition.kind.format_script(
                Partition.device(device, i + 1)))
        return concat(cmds)

    def mount_script(self, device: str) -> str:
        cmds = []
        for i, partition in enumerate(self.partitions):
            partition_device = Partition.device(device, i + 1)
            script = partition.kind.mount_script(partition_device)
            if partition.kind == PartitionKind.ROOT: cmds = [script] + cmds
            else: cmds.append(script)
        return concat(cmds)

    def fstab_script(self, device: str) -> str:
        entries = []
        for i, partition in enumerate(self.partitions):
            entries.append(partition.kind.fstab_entry_script(
                Partition.device(device, i + 1)))
        return concat([
            'mkdir -p /mnt/etc',
            f'echo "{concat(entries)}" > /mnt/etc/fstab',
        ])

    def script(self, device: str) -> str:
        return concat([
            self.partition_script(device),
            self.format_script(device),
            self.mount_script(device),
            self.fstab_script(device),
        ], 2)


#%
import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import confusion_matrix
import tensorflow as tf

img_size = 28 # Dimensão das imagens de MNIST
img_shape = (img_size, img_size)

def set_data_cls(labels):
    return np.array([label.argmax() for label in labels])

def plot_images(images, labels, pred=None):

    # Pega as 9 primeiras imagens
    images = images[0:9]

    # Pega os 9 primeiros rotulos
    cls_true = set_data_cls(labels)[0:9]

    assert len(images) == len(cls_true) == 9

    # Create figure with 3x3 sub-plots.
    fig, axes = plt.subplots(3, 3)
    fig.subplots_adjust(hspace=0.3, wspace=0.3)

    for i, ax in enumerate(axes.flat):
        # Plot image.
        ax.imshow(images[i].reshape(img_shape), cmap='binary')

        # Show true and predicted classes.
        if pred is None:
            xlabel = "True: {0}".format(cls_true[i])
        else:
            cls_pred = set_data_cls(pred)[0:9]
            xlabel = "True: {0}, Pred: {1}".format(cls_true[i], cls_pred[i])

        ax.set_xlabel(xlabel)

        # Remove ticks from the plot.
        ax.set_xticks([])
        ax.set_yticks([])

def print_confusion_matrix(data, session, y_pred, feed_dict_test,num_classes):
    # Get the true classifications for the test-set.
    cls_true = set_data_cls(data.test.labels)

    # Get the predicted classifications for the test-set.
    pred = session.run(y_pred, feed_dict=feed_dict_test)

    # Get the confusion matrix using sklearn.
    cm = confusion_matrix(y_true=cls_true,
                          y_pred=set_data_cls(pred))

    # Print the confusion matrix as text.
    print(cm)

    # Plot the confusion matrix as an image.
    plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)

    # Make various adjustments to the plot.
    plt.tight_layout()
    plt.colorbar()
    tick_marks = np.arange(num_classes)
    plt.xticks(tick_marks, range(num_classes))
    plt.yticks(tick_marks, range(num_classes))
    plt.xlabel('Predicted')
    plt.ylabel('True')

def plot_example_errors(data, session, correct_prediction, y_pred, feed_dict_test):
    # Use TensorFlow to get a list of boolean values
    # whether each test-image has been correctly classified,
    # and a list for the predicted class of each image.
    correct, pred = session.run([correct_prediction, y_pred],
                                    feed_dict=feed_dict_test)

    # Negate the boolean array.
    incorrect = (correct == False)

    # Get the images from the test-set that have been
    # incorrectly classified.
    inc_images = data.test.images[incorrect]
    inc_labels = data.test.labels[incorrect]
    # Get the predicted classes for those images.
    inc_pred = pred[incorrect]

    # Get the true classes for those images.
    #cls_true = set_data_test_cls(data.test.labels)[incorrect]
    # Plot the first 9 images.
    plot_images(images=inc_images[0:9],
                labels=inc_labels[0:9],
                pred=inc_pred[0:9])

def plot_weights(weights, session):
    # Get the values for the weights from the TensorFlow variable.
    w = session.run(weights)

    # Get the lowest and highest values for the weights.
    # This is used to correct the colour intensity across
    # the images so they can be compared with each other.
    w_min = np.min(w)
    w_max = np.max(w)

    # Create figure with 3x4 sub-plots,
    # where the last 2 sub-plots are unused.
    fig, axes = plt.subplots(3, 4)
    fig.subplots_adjust(hspace=0.3, wspace=0.3)

    for i, ax in enumerate(axes.flat):
        # Only use the weights for the first 10 sub-plots.
        if i<10:
            # Get the weights for the i'th digit and reshape it.
            # Note that w.shape == (img_size_flat, 10)
            image = w[:, i].reshape(img_shape)

            # Set the label for the sub-plot.
            ax.set_xlabel("Weights: {0}".format(i))

            # Plot the image.
            ax.imshow(image, vmin=w_min, vmax=w_max, cmap='seismic')

        # Remove ticks from each sub-plot.
        ax.set_xticks([])
        ax.set_yticks([])


from django.urls import path, include, re_path
# from rest_framework.authtoken.views import obtain_auth_token
from rest_framework_simplejwt.views import TokenObtainPairView, TokenRefreshView
from django.conf import settings
from django.conf.urls.static import static

from .views import *

urlpatterns = [
    path('auth/', include('djoser.urls')),
    path('token/', TokenObtainPairView.as_view(), name='token_obtain_pair'),
    path('token/refresh', TokenRefreshView.as_view(), name='token_refresh'),

    # path('release/<int:pk>', ReleaseRetrieveView.as_view()),
    path('release/<slug>', ReleaseRetrieveView.as_view()),
    path('release/update/<int:pk>', ReleaseUpdateView.as_view()),
    path('release/create', ReleaseCreateView.as_view()),
    path('release/all', ReleaseListView.as_view()),

    path('platform/<int:pk>', PlatformRetrieveView.as_view()),
    path('platform/update/<int:pk>', PlatformUpdateView.as_view()),
    path('platform/create', PlatformCreateView.as_view()),
    path('platform/all', PlatformListView.as_view()),

    path('link/<int:pk>', LinkRetrieveView.as_view()),
    path('link/update/<int:pk>', LinkUpdateView.as_view()),
    path('link/create', LinkCreateView.as_view()),
    path('link/all', LinkListView.as_view()),
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


import matplotlib.pyplot as plt
import numpy as np
import os
from SALib.analyze import morris
from SALib.sample.morris import sample
from SALib.plotting.morris import horizontal_bar_plot

problem = {
    'num_vars':
        7,
    'names': [
        r'$\tilde{a}_B$', r'$\tilde{a}_N$', r'$\tilde{\beta}_B$',
        r'$\tilde{\beta}_N$', r'$\lambda$', r'$r_1$', r'$r_2$'
    ],
    'groups':
        None,
    'bounds': [[0.01, 2], [0.01, 2], [5, 40], [5, 40], [0.1, 4], [0.01, 2],
               [0.5, 1.5]]
}
# Generate samples
param_values = sample(problem, N=100, num_levels=4, grid_jump=2, \
                      optimal_trajectories=None)

saved_param_values = np.zeros_like(param_values)
saved_param_values[:, 0] = np.loadtxt("a1.txt", dtype=np.float32, delimiter=';')
saved_param_values[:, 1] = np.loadtxt("a2.txt", dtype=np.float16, delimiter=';')
saved_param_values[:, 2] = np.loadtxt("beta1.txt",
                                      dtype=np.float16,
                                      delimiter=';')
saved_param_values[:, 3] = np.loadtxt("beta2.txt",
                                      dtype=np.float16,
                                      delimiter=';')
saved_param_values[:, 4] = np.loadtxt("lambda.txt",
                                      dtype=np.float16,
                                      delimiter=';')
saved_param_values[:, 5] = np.loadtxt("r1.txt", dtype=np.float16, delimiter=';')
saved_param_values[:, 6] = np.loadtxt("r2.txt", dtype=np.float16, delimiter=';')

r_Nog = np.loadtxt(os.path.join("sweep_array_all_morris", "r_Nog.txt"),
                   dtype=np.float32)
r_BMP = np.loadtxt(os.path.join("sweep_array_all_morris", "r_BMP.txt"),
                   dtype=np.float32)
f_BMP = np.loadtxt(os.path.join("sweep_array_all_morris", "f_BMP.txt"),
                   dtype=np.float32)
f_Nog = np.loadtxt(os.path.join("sweep_array_all_morris", "f_Nog.txt"),
                   dtype=np.float32)

Y = (np.max(f_BMP, axis=1) - np.min(f_BMP, axis=1)) / (
    np.argmax(f_BMP, axis=1) - np.argmin(f_BMP, axis=1))

Si = morris.analyze(problem,
                    saved_param_values,
                    Y,
                    conf_level=0.95,
                    print_to_console=True,
                    num_levels=4,
                    grid_jump=2,
                    num_resamples=100)

fig, ax1 = plt.subplots(1, 1)
horizontal_bar_plot(ax1, Si, {}, sortby='mu_star')

plt.show()


from django.db.models.signals import post_migrate
from django.dispatch.dispatcher import receiver
from stackable.stackable import StackableSettings
def create_cache_table(table_name):
    """
    safely create the cache table
    """
    from django.db import connection
    cursor = connection.cursor()
    sql = """
     CREATE TABLE IF NOT EXISTS `{table_name}` (
      `cache_key` varchar(255) NOT NULL,
      `value` longtext NOT NULL,
      `expires` datetime NOT NULL,
      PRIMARY KEY (`cache_key`),
      KEY `django_db_cache_expires` (`expires`)
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
    """
    try:
        cursor.execute(sql.format(table_name=table_name))
    except Warning:
        # mysql warns about already existing table. we don't care
        pass

def setup_cache_table(settings, *args, **kwargs):
    """
    process all CACHES configs and create tables for database backend
    """
    CACHES = settings.get('CACHES')
    # create all cache tables
    for name, config in CACHES.items():
        if 'db.DatabaseCache' in config.get('BACKEND', ''):
            table_name = config['LOCATION']
            create_cache_table(table_name)

class Config_DjangoCacheDB(object):
    # db cache
    CACHES = {
        'default': {
            'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
            # cache table, create in setup_cache_table patch
            'LOCATION': 'django_db_cache',
        }
    }
    # setup cache table
    __patches__ = (
        StackableSettings.patch(setup_cache_table),
    )

@receiver(post_migrate)
def post_migrate_create_cache(*args, **kwargs):
    from django.conf import settings
    setup_cache_table({'CACHES': settings.CACHES})


from collections import deque

n, k = map(int, input().split())  # N: 크기 ,K: 바이러스종류 수

graph = []  # 전체 보드 정보를 담는 리스트
data = []  # 바이러스에 대한 정보를 담는 리스트

for i in range(n):
    graph.append(list(map(int, input().split())))
    for j in range(n):
        if graph[i][j] != 0:  # 바이러스가 존재하면,
            # (바이러스 종류, 시간, 위치x, 위치y)
            data.append((graph[i][j], 0, i, j))

data.sort()  # 바이러스 종류 순대로 오름차순 정렬

q = deque(data)  # 큐 시작

target_s, target_x, target_y = map(int, input().split())  # (S, X, Y)입력받기

dx = [0, 0, -1, 1]  # 상 하 좌 우
dy = [-1, 1, 0, 0]
while q:
    virus, s, x, y = q.popleft()
    if s == target_s:
        break

    # 현재노드에서 상하좌우 위치로 갈때 위치 확인
    for i in range(4):
        nx = x + dx[i]
        ny = y + dy[i]
        if 0 <= nx and nx < n and 0 <= ny and ny < n:  # 이동 가능한 경우
            # 아직 바이러스가 없는 곳일때
            if graph[nx][ny] == 0:
                graph[nx][ny] = virus
                q.append((virus, s+1, nx, ny))
print(graph[target_x - 1][target_y - 1])


import numpy as np
import os
from PIL import Image
from PIL import ImageFilter
import cv2

def video_length(video_path):
    cap = cv2.VideoCapture(video_path)
    length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
    return length

#extract frames from imput video
def extract_frames(video_path, save_path):
    print('***EXTRACTING FRAMES***')
    _, file_name = os.path.split(video_path)

    cap = cv2.VideoCapture(video_path)

    ct = 1
    ret, frame = cap.read()
    while ret:
        cv2.imwrite(os.path.join(save_path, '{}.jpg'.format(ct)), frame)
        ret, frame = cap.read()
        ct += 1
    cap.release()
    print('***FINISHED EXTRACTION***')

#process individual images
def image_process(save_path, output_path):
    for image_path in os.listdir(save_path):
        if image_path.endswith('.jpg'):
            #rotate and crop each frame
            input_path = os.path.join(save_path, image_path)
            im = Image.open(input_path)
            full_out_path = os.path.join(output_path, image_path)
            im.crop((960, 0, 962, 1080)).save(full_out_path)
        else:
            continue

def concat_images(image_list):
    _im = image_list.pop(0)
    _im = Image.open(os.path.join(output_path, _im))
    for im in image_list:
        if im.endswith('.jpg'):
            im = Image.open(os.path.join(output_path, im))
            dst = Image.new('RGB', (_im.width + im.width, im.height), (0, 0, 0))
            dst.paste(_im, (0, 0))
            dst.paste(im, (_im.width, 0))
            _im = dst
    print(type(_im))
    _im.save('out.jpeg')

def make_list(video):
    image_list = []
    ct = 1
    for i in range(int(video_length(video))):
        path = '{}.jpg'.format(ct)
        image_list.append(path)
        ct += 1
    return image_list

def delete_frames(save_path, output_path):
    for i in os.listdir(save_path):
        if i.endswith('.jpg'):
            os.remove(os.path.join(save_path, i))

    for i in os.listdir(output_path):
        if i.endswith('.jpg'):
            os.remove(os.path.join(output_path, i))

if __name__ == '__main__':
    video = 'input.mp4'
    save_path = 'frames'
    output_path = 'processed frames'
    extract_frames(video, save_path)
    image_process(save_path, output_path)
    image_list = make_list(video)
    concat_images(image_list)
    delete_frames(save_path, output_path)


import numpy as np
import pandas as pd
import numpy as np
import os
from pathlib import Path
from A_Infrastructure.A2_DB import DB
from A_Infrastructure.A1_CONS import CONS

def load_building_data_from_excel():
    base_path = Path().absolute().resolve()
    dynamic_data_path = Path("inputdata/AUT/001__dynamic_calc_data_bc_2017_AUT.csv")
    building_segment_path = Path("inputdata/AUT/040_aut__3__BASE__1_zz_new_bc_seg__b_building_segment_sh.csv")
    building_class_path = Path("inputdata/AUT/001_Building_Classes_2017.csv")

    dynamic_data = pd.read_csv(Path(base_path / dynamic_data_path), sep=None, engine="python")
    building_segment = pd.read_csv(Path(base_path / building_segment_path), sep=None, engine="python").drop(
        index=[0, 1, 2]).reset_index(drop=True)
    building_class = pd.read_csv(Path(base_path / building_class_path), sep=None, engine="python").drop(
        index=[0, 1, 2]).reset_index(drop=True)

    # create the data for calculations
    # for old buildings I use the data of the latest renovation (most likely to have heat pump installed)
    gen4 = building_class.loc[building_class["name"].str.contains("gen4", case=False)]
    sfh_gen4 = gen4.loc[gen4["name"].str.contains("sfh", case=False)]
    rh_gen4 = gen4.loc[gen4["name"].str.contains("rh", case=False)]

    gen3 = building_class.loc[building_class["name"].str.contains("gen3", case=False)]
    sfh_gen3 = gen3.loc[gen3["name"].str.contains("sfh", case=False)].iloc[-1,
               :].to_frame().transpose()  # only last one (F-series because no gen4)
    rh_gen3 = gen3.loc[gen3["name"].str.contains("rh", case=False)].iloc[-1, :].to_frame().transpose()

    gen2 = building_class.loc[building_class["name"].str.contains("gen2", case=False)]
    sfh_gen2 = gen2.loc[gen2["name"].str.contains("sfh", case=False)].iloc[-1,
               :].to_frame().transpose()  # only last one (G-series because no gen3)
    rh_gen2 = gen2.loc[gen2["name"].str.contains("rh", case=False)].iloc[-1, :].to_frame().transpose()

    modern_sfh = building_class.loc[building_class["name"].str.contains("sfh", case=False)].iloc[-2:, :]
    modern_rh = building_class.loc[building_class["name"].str.contains("rh", case=False)].iloc[-2:, :]

    all_classes = pd.concat([modern_sfh, sfh_gen2, sfh_gen3, sfh_gen4, modern_rh,
                             rh_gen2, rh_gen3, rh_gen4]).set_index("index", drop=True).sort_index()
    indices = all_classes.index.to_numpy().astype(int)

    all_dynamic_data = dynamic_data.set_index("bc_index").loc[indices, :]

    df_database = pd.concat([all_classes, all_dynamic_data], axis=1)
    # remove duplicate columns
    df_database = df_database.loc[:, ~df_database.columns.duplicated()]
    df_database = df_database.drop(columns=[
        "building_life_time_index",
        "length_of_building",
        "width_of_building",
        "number_of_floors",
        "room_height",
        "building_geometry_special_index",
        "percentage_of_building_surface_attached_length",
        "percentage_of_building_surface_attached_width",
        "share_of_window_area_on_gross_surface_area",
        "share_of_windows_oriented_to_south",
        "share_of_windows_oriented_to_north",
        "gebaudebauweise_fbw",
        "renovation_facade_year_start",
        "renovation_facade_year_end",
        "f_x3_facade",
        "f_x2_facade",
        "f_x1_facade",
        "f_x0_facade",
        "renovation_windows_year_start",
        "renovation_windows_year_end",
        "f_x3_windows",
        "f_x2_windows",
        "f_x1_windows",
        "f_x0_windows",
        "envelope_quality_def_index",
        "mech_ventilation_system_index",
        "user_profiles_index",
        "climate_region_index",
        "agent_mixes_index",
        "bc_specific_userfactor",
        "INPUT_DATA_STOP",
        "grossfloor_area",  # ist = Af
        "heated_area",
        "total_vertical_surface_area",
        "aewd",
        "areafloor",
        "areadoors",
        "grossvolume",
        "heatedvolume",
        "heated_norm_volume",
        "characteristic_length",
        "A_V_ratio",
        "LEK",
        "hwb",
        "hlpb",
        "orig_hlpb",
        "hlpb_dhw",
        "t_indoor_eff_factor",
        "effective_indoor_temp_jan",
        "prev_index",
        "last_action",
        "last_action_year",
        "mean_construction_period",
        "uedh_sh_norm",
        "uedh_sh_effective",
        "uedh_sh_original",
        "climate_change_factor1",
        "uedh_sh_climate1",
        "climate_change_factor1a",
        "uedh_sh_climate1a",
        "climate_change_factor1b",
        "uedh_sh_climate1b",
        "climate_change_factor2",
        "uedh_sh_climate2",
        "ued_cool",
        "ued_cool_climate1",
        "ued_cool_climate1a",
        "ued_cool_climate1b",
        "ued_cool_climate2",
        "ued_cool_no_int_gains",
        "uedh_sh_savings_mech_vent",
        "effective_heating_hours",
        "effective_operation_hours_solar",
        "boiler_operation_hours_dhw",
        "distribution_operation_hours_dhw",
        "rand_number_1",
        "rand_number_2",
        "rand_number_3",
        "u_value_ceiling",
        "u_value_exterior_walls",
        "u_value_windows1",
        "u_value_windows2",
        "u_value_roof",
        "u_value_zangendecke",
        "u_value_floor",
        "seam_loss_windows",
        "n_50",
        "envelope_quality_add_data_idx",
        "add_building_life_time",
        "facade_life_time_factor",
        "windows_life_time_factor",
        "envelope_lifetime_index",
        "trans_loss_walls",
        "trans_loss_ceil",
        "trans_loss_wind",
        "trans_loss_doors",
        "trans_loss_floor",
        "trans_loss_therm_bridge",
        "trans_loss_ventilation",
        "total_heat_losses",
        "uedh_sh_effective_gains",
        "uedh_sh_effective_gains_solar",
        "uedh_sh_effective_gains_lighting",
        "uedh_sh_effective_gains_internal_other",
        "uedh_sh_effective_gains_internal_people",
        "uedh_sh_norm_gains_solar_transparent",
        "ued_cool_gains_solar_transparent",
        "ued_cool_gains_solar_opaque",
        "ued_cool_gains_lighting",
        "ued_cool_gains_internal_other",
        "ued_cool_gains_internal_people",
        "internal_cool_gains_lighting",
        "internal_gains_lighting",
        "internal_cool_gains_other_appliances",
        "internal_gains_other_appliances",
        "internal_cool_gains_people",
        "internal_gains_people",
        "uedh_sh_effective_losses_before_gains",
        "estimated_electricity_internal_gains",
        "estimated_electricity_not_internal_gains",
        "uedh_sh_effective_jan",
        "uedh_sh_effective_feb",
        "uedh_sh_effective_mar",
        "uedh_sh_effective_apr",
        "uedh_sh_effective_may",
        "uedh_sh_effective_jun",
        "uedh_sh_effective_jul",
        "uedh_sh_effective_aug",
        "uedh_sh_effective_sep",
        "uedh_sh_effective_oct",
        "uedh_sh_effective_nov",
        "uedh_sh_effective_dec",
        "uedh_sh_effective_gains_solar_jan",
        "uedh_sh_effective_gains_solar_feb",
        "uedh_sh_effective_gains_solar_mar",
        "uedh_sh_effective_gains_solar_apr",
        "uedh_sh_effective_gains_solar_may",
        "uedh_sh_effective_gains_solar_jun",
        "uedh_sh_effective_gains_solar_jul",
        "uedh_sh_effective_gains_solar_aug",
        "uedh_sh_effective_gains_solar_sep",
        "uedh_sh_effective_gains_solar_oct",
        "uedh_sh_effective_gains_solar_nov",
        "uedh_sh_effective_gains_solar_dec",
        "annual_mech_vent_volume_heat_recovery_mode",
        "annual_mech_vent_volume_non_heat_recovery_mode",
        "ued_cooling_jan",
        "ued_cooling_feb",
        "ued_cooling_mar",
        "ued_cooling_apr",
        "ued_cooling_may",
        "ued_cooling_jun",
        "ued_cooling_jul",
        "ued_cooling_aug",
        "ued_cooling_sep",
        "ued_cooling_oct",
        "ued_cooling_nov",
        "ued_cooling_dec",
        "renovation_inv_costs",
        "of_which_maintenance_inv_costs_part",
        "target_hwb",
        "target_hwb_asterix",
        "hwb_lc_factor",
        "target_hwb_u_value_adoption_factor",
        "hwb_original_u_values_using_days",
        "hwb_using_days",
        "immissionFlaeche_sommer_nachweis_B8110_3",
        "immissionsflaechenbezogenerLuftwechsel_sommer_nachweis_B8110_3",
        "erforderliche_speicherwirksameMasse_sommer_nachweis_B8110_3",
        "immissionsflaechenbezogene_speicherwirksameMasse_sommer_nachweis_B8110_3",
        "heatstoragemass_ratio_sommer_nachweis_B8110_3",
        "start_refurb_obligation_year",
        "is_unrefurbished",
        "deep_refurb_level",
        "delta_hwb_refurb",
        "average_hwb_unrefurbished_simulation_start",
        "average_hwb_unrefurbished_current_period",
        "hwb_ref_climate_zone",
        "is_unrefurbished_age_not_considered",
        "cum_inv_env_exist_build",
        "cum_sub_env_exist_build",
        "Tset",
        "user_profile",
    ])
    df_database = df_database.reset_index().rename(columns={"index": "index_invert"})
    df_database["index"] = np.arange(1, len(df_database)+1)

    for index in indices:
        air_heatpump_buildings = building_segment.loc[
            (building_segment.loc[:, "buildingclasscsvid"] == str(index)) &
            (building_segment.loc[:, "heatingsystem"].str.contains("heatpump\(air/water\)", case=False))]
        number_of_air_heatpumps = air_heatpump_buildings.loc[:, "number_of_buildings"].sum()

        water_heatpump_buildings = building_segment.loc[
            (building_segment.loc[:, "buildingclasscsvid"] == str(index)) &
            (building_segment.loc[:, "heatingsystem"].str.contains("heatpump\(water/water\)", case=False))]
        number_of_water_heatpumps = water_heatpump_buildings.loc[:, "number_of_buildings"].sum()
    print(f"number of air HP: {number_of_air_heatpumps} \n number of ground HP: {number_of_water_heatpumps}")
         # TODO noch die anzahl bestimmen! entweder alle heat pumps von einer gebäudekategorie oder nur die die ich auswähle

    return df_database

if __name__=="__main__":
    building_data = load_building_data_from_excel()
    # write to DB
    column_names = {"index": "INTEGER", "index_invert": "INTEGER", "name": "TEXT",
                    "construction_period_start": "INTEGER", "construction_period_end": "INTEGER",
                    "building_categories_index": "INTEGER", "number_of_dwellings_per_building": "REAL",
                    "number_of_persons_per_dwelling": "REAL", "horizontal_shading_building": "REAL",
                    "areawindows": "REAL", "area_suitable_solar": "REAL", "hwb_norm": "REAL",
                    "ued_dhw": "REAL", "average_effective_area_wind_west_east_red_cool": "REAL",
                    "average_effective_area_wind_south_red_cool": "REAL",
                    "average_effective_area_wind_north_red_cool": "REAL",
                    "spec_int_gains_cool_watt": "REAL",
                    "Af": "REAL", "Hop": "REAL", "Htr_w": "REAL", "Hve": "REAL", "CM_factor": "REAL",
                    "Am_factor": "REAL"}
    DB().write_DataFrame(df_database, "ID_BuildingOption",
                         column_names=column_names.keys(),
                         conn=DB().create_Connection(CONS().RootDB),
                         dtype=column_names)


from django.db.models.signals import post_delete, post_save
from django.test import TestCase

from contentstore.models import Schedule
from contentstore.signals import schedule_deleted, schedule_saved

from .utils import normalise_metric_name

class NormaliseMetricNameTest(TestCase):
    def test_normalise_metric_name(self):
        """
        The normalise_metric_name function should replace all non-alphanumeric
        with underscores.
        """
        self.assertEqual(normalise_metric_name("foo^& bar"), "foo_bar")
        self.assertEqual(normalise_metric_name("foo   bar"), "foo_bar")
        self.assertEqual(normalise_metric_name("_foo!bar,"), "foo_bar")

post_save_signals = ((schedule_saved, Schedule),)

post_delete_signals = ((schedule_deleted, Schedule),)

def disable_signals():
    for (signal, model) in post_save_signals:
        post_save.disconnect(signal, sender=model)
    for (signal, model) in post_delete_signals:
        post_delete.disconnect(signal, sender=model)

def enable_signals():
    for (signal, model) in post_save_signals:
        post_save.connect(signal, sender=model)
    for (signal, model) in post_delete_signals:
        post_delete.connect(signal, sender=model)