import os
import string
import pickle
import re
import time
import subprocess
from itertools import compress
from typing import List
import numpy as np
import nltk
nltk.download('punkt')
from nltk.tokenize import word_tokenize
from keras.models import Model, Input, load_model
from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical
from keras.layers import LSTM, Dense, TimeDistributed, Embedding, Bidirectional, add
from keras.preprocessing.text import Tokenizer
from keras_contrib.layers import CRF
from keras_contrib.losses import crf_loss
from keras_contrib.metrics import crf_viterbi_accuracy
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
import matplotlib.pyplot as plt
from src.utils.config_loader import NamedEntityRecognitionConfigReader
from src.models.embedding_layers import FastTextEmbedding, Glove6BEmbedding

class DataPreprocessor:
    """
    Utility class performing several data preprocessing steps
    """
    def __init__(self, max_sequence_length: int, validation_split: float, vocab_size: int):
        self.max_sequence_length = max_sequence_length
        self.validation_split = validation_split
        self.vocab_size = vocab_size
        self.tokenizer_obj = None
        self.labels_to_idx = None

    def clean_data(self, X: List):
        """
        Performs data cleaning operations such as removing html breaks, lower case,
        remove stopwords ...
        Args:
            X: input reviews to be cleaned
        Returns:
            None
        """
        print("===========> data cleaning")
        review_lines = list()
        for line in X:
            tokens = [w.lower() for w in line]
            table = str.maketrans('', '', string.punctuation)
            stripped = [w.translate(table) for w in tokens]
            words = [word for word in stripped if word.isalpha()]
            # stop_words = set(stopwords.words('english'))
            # words = [word for word in words if word not in stop_words]
            review_lines.append(words)
        print("----> data cleaning finish")
        return review_lines

    def tokenize_text(self, X: List, y: List):
        """
        Performs data tokenization into a format that is digestible by the model
        Args:
            X: list of predictors already cleaned
        Returns:
            tokenizer object and tokenized input features
        """
        print("===========> data tokenization")
        # features tokenization
        self.tokenizer_obj = Tokenizer(num_words=self.vocab_size)
        self.tokenizer_obj.fit_on_texts(X)
        sequences = self.tokenizer_obj.texts_to_sequences(X)
        self.tokenizer_obj.word_index["pad"] = 0
        review_pad = pad_sequences(sequences, maxlen=self.max_sequence_length, padding="post",
                                   value=self.tokenizer_obj.word_index["pad"])
        # labels tokenization
        flat_list = [item for sublist in y for item in sublist]
        unique_labels = list(set(flat_list))
        self.labels_to_idx = {t: i + 1 for i, t in enumerate(unique_labels)}
        self.labels_to_idx["pad"] = 0
        tokenized_labels = [[self.labels_to_idx[word] for word in sublist] for sublist in y]
        tokenized_labels = pad_sequences(tokenized_labels, maxlen=self.max_sequence_length, padding="post",
                                         value=self.labels_to_idx["pad"])
        print("----> data tokenization finish")
        print("found %i unique tokens" % len(self.tokenizer_obj.word_index))
        print("features tensor shape ", review_pad.shape)
        print("labels tensor shape ", tokenized_labels.shape)
        return review_pad, tokenized_labels

    def split_train_test(self, X, y):
        """
        Wrapper method to split training data into a validation set and a training set
        Args:
            X: tokenized predictors
            y: labels
        Returns:
            tuple consisting of training predictors, training labels, validation predictors, validation labels
        """
        print("===========> data split")
        y = [to_categorical(i, num_classes=len(self.labels_to_idx)) for i in y]
        X_train, X_test, y_train, y_test = train_test_split(X, y, shuffle=True, test_size=self.validation_split)
        print("----> data split finish")
        print('training features shape ', X_train.shape)
        print('testing features shape ', X_test.shape)
        print('training target shape ', np.asarray(y_train).shape)
        print('testing target shape ', np.asarray(y_test).shape)
        return X_train, X_test, np.asarray(y_train), np.asarray(y_test)

    def save_preprocessor(self, file_name_prefix):
        """
        Stores the data preprocessor under 'models folder'
        Args:
            file_name_prefix: a file name prefix having the following format 'sentiment_analysis_%Y%m%d_%H%M%S'
        Returns:
            None
        """
        root_dir = os.environ.get("MARABOU_HOME")
        if not os.path.isdir(os.path.join(root_dir, "marabou/train/trained_models")):
            os.mkdir(os.path.join(root_dir, "marabou/train/trained_models"))
        model_folder = os.path.join(root_dir, "marabou/train/trained_models")
        file_url = os.path.join(model_folder, file_name_prefix + "_preprocessor.pkl")
        with open(file_url, 'wb') as handle:
            pickle.dump(self.tokenizer_obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
            pickle.dump(self.max_sequence_length, handle, protocol=pickle.HIGHEST_PROTOCOL)
            pickle.dump(self.vocab_size, handle, protocol=pickle.HIGHEST_PROTOCOL)
            pickle.dump(self.labels_to_idx, handle, protocol=pickle.HIGHEST_PROTOCOL)
        print("----> proprocessor object saved to %s" % file_url)

    @staticmethod
    def load_preprocessor(preprocessor_file_name):
        """
        Loads preprocessing tools for the model
        Args:
            preprocessor_file_name: data to evaluate
        Return:
            preprocessed object
        """
        root_dir = os.environ.get("MARABOU_HOME")
        if not os.path.isdir(os.path.join(root_dir, "marabou/evaluation/trained_models")):
            os.mkdir(os.path.join(root_dir, "marabou/evaluation/trained_models"))
        model_folder = os.path.join(root_dir, "marabou/evaluation/trained_models")
        preprocessor = {}
        preprocessor_file_name = os.path.join(model_folder, preprocessor_file_name)
        with open(preprocessor_file_name, 'rb') as f:
            preprocessor['tokenizer_obj'] = pickle.load(f)
            preprocessor['max_sequence_length'] = pickle.load(f)
            preprocessor['vocab_size'] = pickle.load(f)
            preprocessor['labels_to_idx'] = pickle.load(f)
        return preprocessor

    @staticmethod
    def preprocess_data(data, preprocessor):
        """
        Performs data preprocessing before inference
        Args:
            data: data to evaluate
            preprocessor: tokenizer object
        Return:
            preprocessed data
        """
        lines = list()
        n_tokens_list = list()
        for line in data:
            if not isinstance(line, list):
                line = word_tokenize(line)
            lines.append(line)
            n_tokens_list.append(len(line))
        data = preprocessor['tokenizer_obj'].texts_to_sequences(lines)
        data = pad_sequences(data, maxlen=preprocessor['max_sequence_length'], padding="post",
                             value=preprocessor['tokenizer_obj'].word_index["pad"])
        return data, lines, n_tokens_list

class RNNModel:
    """
    Handles the RNN model
    """
    def __init__(self, **kwargs):
        self.model_name = "rnn"
        self.use_pretrained_embedding = None
        self.vocab_size = None
        self.embedding_dimension = None
        self.embeddings_path = None
        self.max_length = None
        self.word_index = None
        self.embedding_layer = None
        self.model = None
        self.n_labels = None
        self.labels_to_idx = None
        self.n_iter = 5
        keys = kwargs.keys()
        if 'config' in keys and 'data_preprocessor' in keys:
            self.init_from_config_file(kwargs['config'], kwargs['data_preprocessor'])
        else:
            self.init_from_files(kwargs['h5_file'], kwargs['class_file'])

    def init_from_files(self, h5_file, class_file):
        """
        Initialize the class from a previously saved model
        Args:
            h5_file: url to a saved class
        Return:
            None
        """
        self.model = load_model(h5_file, custom_objects={'CRF': CRF,
                                                         'crf_loss': crf_loss,
                                                         'crf_viterbi_accuracy': crf_viterbi_accuracy})

        with open(class_file, 'rb') as f:
            self.use_pretrained_embedding = pickle.load(f)
            self.vocab_size = pickle.load(f)
            self.embedding_dimension = pickle.load(f)
            self.embeddings_path = pickle.load(f)
            self.max_length = pickle.load(f)
            self.word_index = pickle.load(f)

    def init_from_config_file(self, config: NamedEntityRecognitionConfigReader, data_preprocessor: DataPreprocessor):
        """
        Initializes the class for the first time from a given configuration file and data processor
        Args:
            config: .json configuration reader
            data_preprocessor: preprocessing tool for the training data
        Return:
            None
        """
        self.use_pretrained_embedding = config.pre_trained_embedding
        self.vocab_size = config.vocab_size
        self.embedding_dimension = config.embedding_dimension
        self.embeddings_name = config.embedding_algorithm
        if self.embeddings_name == "glove":
            self.embeddings_path = config.embeddings_path_glove
        elif self.embeddings_name == "fasttext":
            self.embeddings_path = config.embeddings_path_fasttext
        self.max_length = config.max_sequence_length
        self.n_labels = len(data_preprocessor.labels_to_idx)
        self.word_index = data_preprocessor.tokenizer_obj.word_index
        self.labels_to_idx = data_preprocessor.labels_to_idx
        self.embedding_layer = self.build_embedding()
        self.model = self.build_model()

    def build_embedding(self):
        """
        Builds the embedding layer. depending on the configuration, it will either
        load a pretrained embedding or create an empty embedding to be trained along
        with the data
        Return:
            None
        """
        if self.use_pretrained_embedding and self.embeddings_name == "glove":
            glove_embeddings = Glove6BEmbedding(self.embedding_dimension, self.word_index,
                                                self.vocab_size, self.embeddings_path, self.max_length)
            embedding_layer = glove_embeddings.embedding_layer
        elif self.use_pretrained_embedding and self.embeddings_name == "fasttext":
            fasttext_embeddings = FastTextEmbedding(self.word_index, self.vocab_size, self.embeddings_path,
                                                    self.max_length)
            embedding_layer = fasttext_embeddings.embedding_layer
        else:
            print("===========> embedding trained with the model")
            embedding_layer = Embedding(input_dim=self.vocab_size, output_dim=self.embedding_dimension,
                                        input_length=self.max_length, trainable=True)
        return embedding_layer

    def build_model(self):
        """
        Builds an RNN model according to fixed architecture
        Return:
            None
        """
        print("===========> build model")
        # Run the function
        input_layer = Input(shape=(self.max_length,), name='input')
        x = self.embedding_layer(input_layer)

        # # archi 1: f1-macro 0.3-fasttext 0.3-no embedding
        # x = Dropout(0.1)(x)
        # x = Bidirectional(LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(x)
        # x = TimeDistributed(Dense(self.n_labels, activation='softmax'))(x)
        # model = Model(inputs=input_layer, outputs=x)
        # model.compile(loss='categorical_crossentropy', optimizer="adam", metrics=['acc'])

        # # archi 2: f1-macro 0.35-fasttext
        # x = Bidirectional(LSTM(units=512, return_sequences=True, recurrent_dropout=0.2, dropout=0.2))(x)
        # x_rnn = Bidirectional(LSTM(units=512, return_sequences=True, recurrent_dropout=0.2, dropout=0.2))(x)
        # x = add([x, x_rnn])  # residual connection to the first biLSTM
        # x = TimeDistributed(Dense(self.n_labels, activation='softmax'))(x)
        # model = Model(inputs=input_layer, outputs=x)
        # model.compile(loss='categorical_crossentropy', optimizer="adam", metrics=['acc'])

        # # archi 3: crf layer
        x = Bidirectional(LSTM(units=50, return_sequences=True, recurrent_dropout=0.2, dropout=0.2))(x)
        x_rnn = Bidirectional(LSTM(units=50, return_sequences=True, recurrent_dropout=0.2, dropout=0.2))(x)
        x = add([x, x_rnn])  # residual connection to the first biLSTM
        x = TimeDistributed(Dense(50, activation='relu'))(x)
        crf = CRF(self.n_labels)
        x = crf(x)
        model = Model(inputs=input_layer, outputs=x)
        model.compile(loss=crf.loss_function, optimizer='adam', metrics=[crf.accuracy])

        print(model.summary())
        return model

    def convert_idx_to_labels(self, classes_vector, labels_to_idx):
        """
        Utility function to convert encoded target idx to original labels
        Args:
            classes_vector: target vector containing indexed classes
            labels_to_idx: a dictionary containing the conversion from each class label to its id
        Return:
            numpy array containing the corresponding labels
        """
        idx_to_labels = {v: k for k, v in labels_to_idx.items()}
        return [idx_to_labels[cl] for cl in classes_vector]

    def predict(self, encoded_text_list, labels_to_idx, n_tokens_list=None):
        """
        Inference method
        Args:
            encoded_text_list: a list of texts to be evaluated. the input is assumed to have been preprocessed
            n_tokens_list: number of tokens in each input string before padding
            labels_to_idx: a dictionary containing the conversion from each class label to its id
        Return:
            numpy array containing the class for token character in the sentence
        """
        probs = self.model.predict(encoded_text_list)
        labels_list = []
        for i in range(len(probs)):
            if n_tokens_list is not None:
                real_probs = probs[i][:n_tokens_list[i]]
            else:
                real_probs = probs[i]
            classes = np.argmax(real_probs, axis=1)
            labels_list.append(self.convert_idx_to_labels(classes, labels_to_idx))
        return labels_list

    def fit(self, X_train, y_train, X_test=None, y_test=None, labels_to_idx=None):
        """
        Fits the model object to the data
        Args:
            X_train: numpy array containing encoded training features
            y_train: numpy array containing training targets
            X_test: numpy array containing encoded test features
            y_test: numpy array containing test targets
            labels_to_idx: a dictionary containing the conversion from each class label to its id
        Return:
            list of values related to each datasets and loss function
        """
        wts = 10 * np.ones((y_train.shape[0], y_train.shape[1]))
        classes = np.argmax(y_train, axis=2)
        wts[classes == self.labels_to_idx["pad"]] = 1
        if (X_test is not None) and (y_test is not None):
            # history = self.model.fit(x=X_train, y=y_train, epochs=self.n_iter, batch_size=64,
            #                        sample_weight=wts, validation_split=0.1, verbose=2)
            history = self.model.fit(X_train, y_train, batch_size=64, epochs=self.n_iter,
                                     validation_split=0.1)
            y_hat = self.predict(X_test, labels_to_idx)
            true_classes = np.argmax(y_test, axis=2).tolist()
            y = [self.convert_idx_to_labels(sublist, labels_to_idx) for sublist in true_classes]
            y_flat = [val for sublist in y for val in sublist]
            y_hat_flat = [val for sublist in y_hat for val in sublist]
            report = classification_report(y_flat, y_hat_flat, output_dict=True)
            df = pd.DataFrame(report).transpose().round(2)
            print(df)
        else:
            report = None
            history = self.model.fit(x=X_train, y=y_train, epochs=self.n_iter, batch_size=64, verbose=2)
        return history, report

    def predict_proba(self, encoded_text_list, n_tokens_list):
        """
        Inference method
        Args:
            encoded_text_list: a list of texts to be evaluated. the input is assumed to have been preprocessed
            n_tokens_list: number of tokens in each input string before padding
        Return:
            numpy array containing the probabilities of a positive review for each list entry
        """
        probs = self.model.predict(encoded_text_list)
        real_probs_list = []
        for i in range(len(probs)):
            real_probs = probs[i][:n_tokens_list[i]]
            real_probs_list.append(real_probs)
        return real_probs_list

    def save_model(self, file_name_prefix):
        """
        Saves the trained model into a h5 file
        Args:
            file_name_prefix: a file name prefix having the following format 'sentiment_analysis_%Y%m%d_%H%M%S'
        Return:
            None
        """
        root_dir = os.environ.get("MARABOU_HOME")
        if not os.path.isdir(os.path.join(root_dir, "marabou/train/trained_models")):
            os.mkdir(os.path.join(root_dir, "marabou/train/trained_models"))
        model_folder = os.path.join(root_dir, "marabou/train/trained_models")
        file_url_keras_model = os.path.join(model_folder, file_name_prefix + "_rnn_model.h5")
        self.model.save(file_url_keras_model)
        file_url_class = os.path.join(model_folder, file_name_prefix + "_rnn_class.pkl")
        with open(file_url_class, 'wb') as handle:
            pickle.dump(self.use_pretrained_embedding, handle)
            pickle.dump(self.vocab_size, handle)
            pickle.dump(self.embedding_dimension, handle)
            pickle.dump(self.embeddings_path, handle)
            pickle.dump(self.max_length, handle)
            pickle.dump(self.word_index, handle)
        print("----> model saved to %s" % file_url_keras_model)
        print("----> class saved to %s" % file_url_class)

    def save_classification_report(self, report, file_name_prefix):
        """
        Saves the classification report to a txt file
        Args:
            report: a classification report object
            file_name_prefix: a file name prefix having the following format 'sentiment_analysis_%Y%m%d_%H%M%S'
        Return:
            None
        """
        root_dir = os.environ.get("MARABOU_HOME")
        if not os.path.isdir(os.path.join(root_dir, "marabou/train/perf")):
            os.mkdir(os.path.join(root_dir, "marabou/train/perf"))
        plot_folder = os.path.join(root_dir, "marabou/train/perf")
        report_file_url = os.path.join(plot_folder, file_name_prefix + "_report.txt")
        df = pd.DataFrame(report).transpose().round(2)
        df['classes'] = df.index
        f = open(report_file_url, "w")
        line = "{:15} |{:10} |{:10} |{:10} |{:10}|\n".format("classes", "precision", "recall", "f1-score", "support")
        f.write(line)
        for _, row in df.iterrows():
            line = "{:15} |{:10} |{:10} |{:10} |{:10}|\n".format(row[4], row[0], row[1], row[2], row[3])
            f.write(line)
        f.close()
        print("----> classification report saved to %s" % report_file_url)

    def save_learning_curve(self, history, file_name_prefix):
        """
        Saves the learning curve plot
        Args:
            history: a dictionary object containing training and validation dataset loss function values and
        objective function values for each training iteration
            file_name_prefix: a file name prefix having the following format 'sentiment_analysis_%Y%m%d_%H%M%S'
        Return:
            None
        """
        root_dir = os.environ.get("MARABOU_HOME")
        if not os.path.isdir(os.path.join(root_dir, "marabou/train/perf")):
            os.mkdir(os.path.join(root_dir, "marabou/train/perf"))
        plot_folder = os.path.join(root_dir, "marabou/train/perf")
        if not os.path.isdir(plot_folder):
            os.mkdir(plot_folder)

        acc = history.history['crf_viterbi_accuracy']
        val_acc = history.history['val_crf_viterbi_accuracy']
        loss = history.history['loss']
        val_loss = history.history['val_loss']
        epochs = range(len(acc))

        fig, ax = plt.subplots(1, 2)
        ax[0].plot(epochs, acc, 'bo', label='Training acc')
        ax[0].plot(epochs, val_acc, 'b', label='Validation acc')
        ax[0].set_title('Training and validation accuracy')
        ax[0].set_ylim([0.6, 1])
        ax[0].legend()
        fig.suptitle('model performance')
        ax[1].plot(epochs, loss, 'bo', label='Training loss')
        ax[1].plot(epochs, val_loss, 'b', label='Validation loss')
        ax[1].set_title('Training and validation loss')
        ax[1].set_ylim([0, 1])
        ax[1].legend()
        plot_file_url = os.path.join(plot_folder, file_name_prefix + "_learning_curve.png")
        plt.savefig(plot_file_url)
        plt.close()
        print("----> learning curve saved to %s" % plot_file_url)

    @staticmethod
    def load_model(h5_file_url=None, class_file_url=None, preprocessor_file_url=None, collect_from_gdrive=False):
        """
        Extracts a model saved using the save_model function
        Args:
            h5_file_url: gdrive link for the trained model
            class_file_url: gdrive link for the class file
            preprocessor_file_url: gdrive link for the preprocessor file
            collect_from_gdrive: whether to collect the model file from google drive
        Return:
            model object and a tokenizer object
        """
        trained_model = None
        root_dir = os.environ.get("MARABOU_HOME")
        if not os.path.isdir(os.path.join(root_dir, "marabou/evaluation/trained_models")):
            os.mkdir(os.path.join(root_dir, "marabou/evaluation/trained_models"))
        model_dir = os.path.join(root_dir, "marabou/evaluation/trained_models")
        if not collect_from_gdrive:
            model_files_list = os.listdir(model_dir)
            if len(model_files_list) > 0:
                rnn_models_idx = [("named_entity_recognition" in f) and ("rnn" in f) for f in model_files_list]
                if np.sum(rnn_models_idx) > 0:
                    rnn_model = list(compress(model_files_list, rnn_models_idx))
                    model_dates = [int(''.join(re.findall(r'\d+', f))) for f in rnn_model]
                    h5_file_name = rnn_model[np.argmax(model_dates)]
                    preprocessor_file = h5_file_name.replace("rnn_model.h5", "preprocessor.pkl")
                    class_file = h5_file_name.replace("rnn_model.h5", "rnn_class.pkl")
                    if (os.path.isfile(os.path.join(model_dir, preprocessor_file))) and\
                            (os.path.isfile(os.path.join(model_dir, class_file))):
                        trained_model = RNNModel(h5_file=os.path.join(model_dir, h5_file_name),
                                                 class_file=os.path.join(model_dir, class_file))
                        return trained_model, preprocessor_file
                    return None, None
                return None, None
            return None, None
        else:
            bash_script_folder = os.path.join(root_dir, "marabou/train/bash_scripts")
            print("===========> collecting model file from link")
            script_path = os.path.join(bash_script_folder, "load_named_entity_recognition_model_file.sh")
            file_prefix = "named_entity_recognition_loaded_%s" % time.strftime("%Y%m%d_%H%M%S")
            h5_file_name = file_prefix + "_rnn_model.h5"
            class_file_name = h5_file_name.replace("rnn_model.h5", "rnn_class.pkl")
            preprocessor_file_name = h5_file_name.replace("rnn_model.h5", "preprocessor.pkl")
            h5_file_local_url = os.path.join(model_dir, h5_file_name)
            class_file_local_url = os.path.join(model_dir, class_file_name)
            preprocessor_file_local_url = os.path.join(model_dir, preprocessor_file_name)
            subprocess.call("%s %s %s %s %s %s %s" % (script_path, h5_file_url, h5_file_local_url,
                                                      class_file_url,
                                                      class_file_local_url, preprocessor_file_url,
                                                      preprocessor_file_local_url),
                            shell=True)
            if (os.path.isfile(h5_file_local_url) and os.path.isfile(class_file_local_url)):
                trained_model = RNNModel(h5_file=h5_file_local_url, class_file=class_file_local_url)
                return trained_model, preprocessor_file_name
            else:
                return None, None

    def visualize(self, questions_list_tokenized, labels_list):
        """
        Visualization method for the nlp model classes
        Args:
            questions_list_tokenized: a tokenized list corresponding to the input text
            labels_list: a list of predicted labels for each input text
        Return:
            display of the result stored in a string
        """
        result = "{:15} | {:5}\n".format("Word", "Pred")
        result += "=" * 20
        result += "\n"
        for i in range(len(labels_list)):
            for word, label in zip(questions_list_tokenized[i], labels_list[i]):
                label = label.replace("B-geo", "Geographical Entity")
                label = label.replace("I-geo", "Geographical Entity")
                label = label.replace("B-tim", "Time indicator")
                label = label.replace("I-tim", "Time indicator")
                label = label.replace("B-org", "Organization")
                label = label.replace("I-org", "Organization")
                label = label.replace("B-gpe", "Geopolitical Entity")
                label = label.replace("I-gpe", "Geopolitical Entity")
                label = label.replace("B-per", "Person")
                label = label.replace("I-per", "Person")
                label = label.replace("B-eve", "Event")
                label = label.replace("I-eve", "Event")
                label = label.replace("B-art", "Artifact")
                label = label.replace("I-art", "Artifact")
                label = label.replace("B-nat", "Natural Phenomenon")
                label = label.replace("I-nat", "Natural Phenomenon")
                label = label.replace("O", "no Label")
                result += "{:15} | {:5}\n".format(word, label)
            result += "\n"
        return result


"""training.py

Helper functions for running experiments.
Author: Alex Kyllo
Date: 2021-06-06
"""
import logging
import os
from datetime import datetime
from pathlib import Path

from tensorflow.keras import callbacks

from musiclearn import processing, vae_models

LOG = logging.getLogger(__name__)
LOG.setLevel(logging.DEBUG)

def build_callbacks(exp_path: os.PathLike, patience: int = 10):
    """Build an array of callbacks for model training."""
    os.makedirs(exp_path, exist_ok=True)
    cbacks = [
        callbacks.ModelCheckpoint(
            exp_path / "checkpoints",
            save_weights_only=True,
            monitor="val_loss",
            save_best_only=True,
        ),
        callbacks.EarlyStopping(
            monitor="val_loss",
            patience=patience,
            verbose=1,
            restore_best_weights=True,
        ),
        callbacks.CSVLogger(exp_path / "history.csv", separator=",", append=True),
        callbacks.TensorBoard(log_dir=exp_path / "tensorboard", histogram_freq=1),
    ]

    return cbacks

def log_start(exp_name, model_name, **hparams):
    LOG.info(f"Running experiment {exp_name} for model {model_name} with hparams: {hparams}")

def log_end(exp_name, model_name, start_time):
    end_time = datetime.now()
    duration = end_time - start_time
    LOG.info(f"Experiment {exp_name} for model {model_name} finished in {duration}")

def train_mtvae(
    exp_name,
    model_class,
    ticks_per_beat,
    beats_per_phrase,
    epochs,
    batch_size,
    learning_rate,
    lstm_units,
    latent_dim,
    embedding_dim,
    dropout_rate,
    gru,
    bidirectional,
    patience=10,
    augment=False,
):
    """Train the MultiTrack VAE once with a given set of hyperparameters."""
    # String quartet MIDI programs
    x = processing.get_string_quartets(ticks_per_beat)
    if augment:
        x = processing.augment(x)
    mtvae = model_class(
        lstm_units,
        embedding_dim,
        latent_dim,
        batch_size,
        learning_rate,
        dropout_rate,
        gru,
        bidirectional,
    )
    model_name = type(mtvae).__name__
    start_time = datetime.now()
    start_str = start_time.strftime("%Y-%m-%dT%H:%M:%S")
    exp_path = Path(f"experiments/{exp_name}/{start_str}")
    cbacks = build_callbacks(exp_path, patience=patience)
    log_start(
        exp_name,
        model_name,
        ticks_per_beat=ticks_per_beat,
        beats_per_phrase=beats_per_phrase,
        epochs=epochs,
        batch_size=batch_size,
        learning_rate=learning_rate,
        lstm_units=lstm_units,
        latent_dim=latent_dim,
        embedding_dim=embedding_dim,
        dropout_rate=dropout_rate,
        patience=patience,
        gru=gru,
        bidirectional=bidirectional,
        augment=augment,
    )
    mtvae.train(x, ticks_per_beat, beats_per_phrase, epochs, callbacks=cbacks)
    log_end(exp_name, model_name, start_time)
    mtvae.save(exp_path / "saved_model")
    LOG.info(f"Model configuration saved to {exp_path / 'saved_model'}")
    return mtvae


from collections import defaultdict

class ATM:

    def __init__(self):
        self.notectr = defaultdict(int)

    def deposit(self, banknotesCount: List[int]) -> None:
        for i in range(5):
            self.notectr[[20, 50, 100, 200, 500][i]] += banknotesCount[i]

    def withdraw(self, amount: int) -> List[int]:
        currnotectr = self.notectr.copy()
        for note in [500, 200, 100, 50, 20]:
            notesused = min(amount // note, currnotectr[note])
            amount -= notesused * note
            currnotectr[note] -= notesused
            if amount == 0:
                break
        if amount != 0:
            return [-1]
        used = [self.notectr[note] - currnotectr[note] for note in [20, 50, 100, 200, 500]]
        self.notectr = currnotectr
        return used

# Your ATM object will be instantiated and called as such:
# obj = ATM()
# obj.deposit(banknotesCount)
# param_2 = obj.withdraw(amount)


from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.common.by import By
from Tasks.task10.pages.campaign_page import CampaignPage
from Tasks.task10.pages.checkout_page import CheckoutPage
import time

class Application:

    def __init__(self):
        self.driver = webdriver.Chrome(ChromeDriverManager().install())
        #self.driver = webdriver.Firefox()
        self.driver.implicitly_wait(1)
        self.base_url = "http://localhost/litecart"
        self.campaign_tab = CampaignPage(self.driver)
        self.checkout_page = CheckoutPage(self.driver)

    def quit(self):
        self.driver.quit()

    def open_campaign(self):
        self.campaign_tab.open(self.base_url)

    def buy_first_product(self):
        # Condition: items in cart should increase:
        self.init_cart = self.get_items_from_cart()
        self.final_cart = self.init_cart + 1

        # Add it:
        self.campaign_tab.add_item_to_cart()
        self.campaign_tab.close_item()

        # Wait until cart's items is updated:
        self.campaign_tab.wait_for_cart(str(self.final_cart))

    def get_items_from_cart(self):
        # Check current items in the cart:
        return self.campaign_tab.check_items()

    def purge_cart(self):
        self.checkout_page.open()
        self.checkout_page.remove_items()

    def is_cart_empty(self):
        try:
            self.checkout_page.open()
        finally:
            return not self.checkout_page.check_shopping_cart()

    def is_cart_full(self):
        try:
            self.checkout_page.open()
        finally:
            return self.checkout_page.check_shopping_cart()


from collections import namedtuple
from operator import attrgetter

metro_data = [
    ('Tokyo', 'JP', 36.93, (35.68, 139.69)),
    ('Delhi NCR', 'IN', 21.94, (28.61, 77.21)),
    ('Mexico City', 'MX', 20.14, (19.43, -99.13)),
    ('New York-Newark', 'US', 20.1, (40.81, -74.02)),
    ('Sao Paulo', 'BR', 19.65, (-23.55, -46.64))
]
LatLon = namedtuple('LatLon', 'lat lon')
Metropolis = namedtuple('Metropolis', 'name cc pop coord')
metro_areas = [Metropolis(name, cc, pop, LatLon(lat, lon)) for name, cc, pop, (lat, lon) in metro_data]
print(metro_areas[0].coord.lat)

# attrgetter 创建的函数会根据名称来提取对象的属性
name_let = attrgetter('name', 'coord.lat')
for city in sorted(metro_areas, key=attrgetter('coord.lat')):
    print(name_let(city))


import copy
import json
from django.http import HttpResponse
import time
import calendar
import datetime

from reports_mjt.utils.decime import to_string

def utc(local_st):
    utc_st = time.localtime(local_st - 28800)
    result = time.strftime("%Y-%m-%d %H:%M:%S", utc_st)
    return result
day_now = time.localtime()
day_begin = '%d-%02d-01' % (day_now.tm_year, day_now.tm_mon)  # 月初肯定是1号
wday, monthRange = calendar.monthrange(day_now.tm_year, day_now.tm_mon)  # 得到本月的天数 第一返回为月第一日为星期几（0-6）, 第二返回为此月天数
day_end = '%d-%02d-%02d' % (day_now.tm_year, day_now.tm_mon, monthRange)

#获取各种时间戳
tday = datetime.date.today()
yesterday = tday - datetime.timedelta(days=1)
tomorrow = tday + datetime.timedelta(days=1)
weekday = tday - datetime.timedelta(days=6)
month = tday - datetime.timedelta(days=29)
months = tday - datetime.timedelta(days=89)
yesterday_start = int(time.mktime(time.strptime(str(yesterday), '%Y-%m-%d')))
yesterday_end = int(time.mktime(time.strptime(str(tday), '%Y-%m-%d'))) - 1
weekday_start = int(time.mktime(time.strptime(str(weekday), '%Y-%m-%d')))
month_start = int(time.mktime(time.strptime(str(month), '%Y-%m-%d')))
months_start = int(time.mktime(time.strptime(str(months), '%Y-%m-%d')))
today_start = yesterday_end + 1
today_end = int(time.mktime(time.strptime(str(tomorrow), '%Y-%m-%d'))) - 1
datedict = {
    'month': [utc(month_start), utc(today_end)],
    'week': [utc(weekday_start), utc(today_end)],
    'total': ['2018-11-25', utc(today_end)],
    'today': [utc(today_start), utc(today_end)],
    'yesday': [utc(yesterday_start), utc(yesterday_end)],
    'months': [utc(months_start), utc(today_end)],
    'day_begin': [day_begin, day_begin],
    'day_end': [day_end, day_end],
     }

def time_stamp():
    # 获取各种时间戳
    tday = datetime.date.today()
    yesterday = tday - datetime.timedelta(days=1)
    tomorrow = tday + datetime.timedelta(days=1)
    weekday = tday - datetime.timedelta(days=6)
    month = tday - datetime.timedelta(days=29)
    months = tday - datetime.timedelta(days=89)
    yesterday_start = int(time.mktime(time.strptime(str(yesterday), '%Y-%m-%d')))
    yesterday_end = int(time.mktime(time.strptime(str(tday), '%Y-%m-%d'))) - 1
    weekday_start = int(time.mktime(time.strptime(str(weekday), '%Y-%m-%d')))
    month_start = int(time.mktime(time.strptime(str(month), '%Y-%m-%d')))
    months_start = int(time.mktime(time.strptime(str(months), '%Y-%m-%d')))
    today_start = yesterday_end + 1
    today_end = int(time.mktime(time.strptime(str(tomorrow), '%Y-%m-%d'))) - 1
    datedict = {
        'month': [utc(month_start), utc(today_end)],
        'week': [utc(weekday_start), utc(today_end)],
        'total': ['2018-11-25', utc(today_end)],
        'today': [utc(today_start), utc(today_end)],
        'yesday': [utc(yesterday_start), utc(yesterday_end)],
        'months': [utc(months_start), utc(today_end)],
        'day_begin': [day_begin, day_begin],
        'day_end': [day_end, day_end],
    }
    return datedict

class FunctionObject(object):
    # 转换时间类型拼接字典
    def timedict(self, startTime, endTime):
        self.startTime = startTime
        self.endTime = endTime
        datestart = datetime.datetime.strptime(self.startTime, '%Y-%m-%d')
        dateend = datetime.datetime.strptime(self.endTime, '%Y-%m-%d')
        date_list = []
        datestart = datestart - datetime.timedelta(days=1)
        while datestart < dateend:
            datestart += datetime.timedelta(days=1)
            qu = datestart.strftime('%Y-%m-%d')
            date_list.append(qu)
        total_list = [0 for _ in range(len(date_list))]
        dict = {
            'date': date_list,
            'number': copy.deepcopy(total_list),
        }
        return dict

    def buytake(self, takeorder, buyorder, splilist, pages, numberbars):
        self.takeorder = takeorder
        self.buyorder = buyorder
        self.splilist = splilist
        self.pages = pages
        self.numberbars = numberbars
        for takekey, take in enumerate(self.takeorder):
            for key, id in enumerate(self.splilist):
                if take[0] == id['id']:
                    self.splilist[key]['tabkegoods'] = self.takeorder[takekey][1]
                    break
        for buykey, take in enumerate(self.buyorder):
            for key, id in enumerate(self.splilist):
                if take[0] == id['id']:
                    self.splilist[key]['purchase'] = self.buyorder[buykey][1]
                    self.splilist[key]['total'] = to_string(self.buyorder[buykey][2])
                    break
        dict = {'data': self.splilist[(self.pages - 1) * self.numberbars:self.pages * self.numberbars]}
        totalnum = len(self.splilist)
        dict['totalnum'] = totalnum
        data = {'data': dict, "errmsg": "成功", "errno": "0"}
        return data

    def startend(self, start_day, end_day):
        self.start_day = start_day
        self.end_day = end_day
        s_time = time.mktime(time.strptime(self.start_day, '%Y-%m-%d'))
        e_time = time.mktime(time.strptime(self.end_day, '%Y-%m-%d'))
        if (float(s_time) > float(e_time)):
            return HttpResponse(json.dumps({"errmsg": "订单统计时间顺序不对", "errno": "4001"}), content_type="application/json")
        return self.start_day, self.end_day

#时间戳转日期
def localutc(local_st):
    utc_st = time.localtime(local_st)
    result = time.strftime("%Y-%m-%d %H:%M:%S", utc_st)
    return result

#时间戳转日期
def yearmonthday(local_st):
    utc_st = time.localtime(local_st)
    result = time.strftime("%Y-%m-%d", utc_st)
    return result

#时间戳转日期
def monthtime(local_st):
    utc_st = time.localtime(local_st)
    result = time.strftime("%Y-%m", utc_st)
    return result

#时间戳转日期
def yeartime(local_st):
    utc_st = time.localtime(local_st)
    result = time.strftime("%Y", utc_st)
    return result

#时间戳转日期减去3小时
def userutc(local_st):
    usertime = datetime.datetime.strptime(local_st, "%Y-%m-%d %H:%M:%S")
    data = time.mktime(usertime.timetuple())-10800
    utc_st = time.localtime(data)
    result = time.strftime("%Y-%m-%d %H:%M:%S", utc_st)
    return result

#29100   633600
#代言券 已入账时间设置
def commission_price(local_st):
    utc_st = time.localtime(local_st - 633600)
    result = time.strftime("%Y-%m-%d %H:%M:%S", utc_st)
    return result

#字符串转时间戳
def str_time_stamp(data):
    timeArray = time.strptime(data, "%Y-%m-%d %H:%M:%S")
    timeStamp = int(time.mktime(timeArray))
    return timeStamp

#字符串转时间戳
def str_time_stamp_date(data):
    timeArray = time.strptime(data, "%Y-%m-%d")
    timeStamp = int(time.mktime(timeArray))
    return timeStamp


'''

Initilizing maze and point-based features
No walls, no collision with features
Known correspondence which means all features have unique labels
screen visualization
'''
import pygame
from visualization import *
from shapely.geometry import *

#### Variables ####
map_width = 500
map_height = 600
borders = [Point(20, 20), Point(20, map_height-20), Point(map_width-20, map_height-20), Point(map_width-20, 20), Point(20, 20)]
wall1 = [Point(20, 100), Point(map_width-150, 100)]
wall2 = [Point(map_width-20, 200), Point(200, 200), Point(200, map_height-120)]
wall3 = [Point(map_width-150, map_height-20), Point(map_width-150, 300)]
maze = [borders, wall1, wall2, wall3]
features = []
[[features.append(j) for j in i] for i in maze]

testfeatures = [Point(20,map_height-20)]
beacons = [features]

def draw_borders(win):
    for i in range(len(borders) - 1):
        pygame.draw.line(win, (255, 0, 0), (int(borders[i].x), int(borders[i].y)),
                         (int(borders[i + 1].x), int(borders[i + 1].y)), 5)

def draw_walls(win):
    #### Wall1 ####
    for i in range(len(wall1)-1):
        pygame.draw.line(win, (255, 0, 0), (int(wall1[i].x), int(wall1[i].y)),
                         (int(wall1[i + 1].x), int(wall1[i + 1].y)), 2)
    #### Wall2 ####
    for i in range(len(wall2)-1):
        pygame.draw.line(win, (255, 0, 0), (int(wall2[i].x), int(wall2[i].y)),
                         (int(wall2[i + 1].x), int(wall2[i + 1].y)), 2)
    #### Wall3 ####
    for i in range(len(wall3)-1):
        pygame.draw.line(win, (255, 0, 0), (int(wall3[i].x), int(wall3[i].y)),
                         (int(wall3[i + 1].x), int(wall3[i + 1].y)), 2)

def draw_beacons(win):
    for i in range(len(features)):
        pygame.draw.circle(win, (0,140,140), (int(features[i].x), int(features[i].y)), 5)

def update_screen(win, bot_c, angle, radius, deltaT=30):

    ### Draw robot ###
    pygame.draw.circle(win, (0, 255, 0), (int(bot_c.x), int(bot_c.y)), radius)
    pygame.draw.line(win, (255, 255, 0), (int(bot_c.x), int(bot_c.y)),
                     (bot_c.x + radius * -np.cos(angle), (bot_c.y + radius * np.sin(angle))),
                     int(radius / 10))
    pygame.display.update()
    pygame.time.delay(deltaT)

def draw_predict_trail(win, index, start, end,width):
    pygame.draw.aaline(win, (252, 3, 236), start, end, width)

def draw_true_trail(win, index, start, end, width):
    pygame.draw.aaline(win, (0, 0, 0), start, end, width)

def quit_screen(): #TOBE TEST
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()


from django.conf import settings as django_settings
from datetime import datetime
from fabric.api import *
import django
import yaml
import os

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "microsite.settings")
django.setup()

def build_documents():
    books_dir = os.path.join(django_settings.BASE_DIR, 'books')
    books_source_dir = os.path.join(django_settings.BASE_DIR, 'books/source')
    documents_file = open(os.path.join(books_dir, 'books.yml'), 'r')
    data = yaml.load(documents_file)

    # Update the books modified date in books.yml file
    for position, book in data.get('documents').iteritems():
        book["updated_date"] = datetime.fromtimestamp(
            os.path.getmtime(books_source_dir + '/' + book.get("folder_name")))

    with open(os.path.join(books_dir, 'books.yml'), 'w') as f:
        yaml.dump(data, f, default_flow_style=False)

    # Remove the templates(build) directory, if exists
    books_templates_dir = os.path.join(
        django_settings.BASE_DIR, 'books/templates')
    if os.path.exists(books_templates_dir):
        local("rm -rf %s" % books_templates_dir)

    # Build documents(.rst files to HTML files) using Sphinx command.
    local("cd books && make html")


# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy.http import Request

class SteamSpider(CrawlSpider):
    name = 'steam'
    #allowed_domains = ['https://store.steampowered.com/']
    start_urls = ['https://store.steampowered.com/search/?sort_by=Released_DESC/']

    rules = (
        Rule(
            LinkExtractor(
                restrict_css=[".search_result_row",]
            ),
            callback='parse_item',
            follow=True
        ),

        #Rule(
        #    LinkExtractor(
        #        restrict_css=[".search_pagination_right",]
        #        restrict_xpath=["./a[@class='pagebtn'][last()]",]
        #    ),
        #    follow=True
        #)
    )

    def parse_item(self, response):
        i = {}

        i['title'] = response.css(".details_block").xpath("./text()")[1].extract()

        return i


import pygame
import sys

pygame.init()

size= (800,500)
screen= pygame.display.set_mode(size)
clock = pygame.time.Clock()

pygame.mouse.set_visible(0)

while True:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()

    mouse_pos = pygame.mouse.get_pos()
    x = mouse_pos[0]
    y = mouse_pos[1]

    screen.fill("Black")

    pygame.draw.rect(screen, "Red", (x,y, 100,100))

    #60 seran los circulos de la ventana
    pygame.display.flip()
    clock.tick(30) #Creamos puntos aleatorios cada 30 seg


import os
import binascii
from datetime import datetime

def get_file_header(path):
    """
    Retrieve the first two hexadecimal numbers of a file
    """
    data = open(path, 'rb')
    hex_str = binascii.b2a_hex(data.read(2))
    return str(hex_str[:2], 'utf8'), str(hex_str[2:4], 'utf8')

"""
JPEG file header hexadecimal is 0xFFD8FF
PNG file header hexadecimal is 0x89504E
GIF file header hexadecimal is 0x474946
"""
def identify_file_type(directory):
    first_value, second_value = get_file_header(directory)
    first_value = int(first_value, 16)
    second_value = int(second_value, 16)
    code_key = first_value ^ 0xFF
    file_type = 'jpg'

    if first_value ^ 0xFF == second_value ^ 0xD8:
        code_key = first_value ^ 0xFF
        file_type = 'jpg'
    elif first_value ^ 0x89 == second_value ^ 0x50:
        code_key = first_value ^ 0x89
        file_type = 'png'
    elif first_value ^ 0x47 == second_value ^ 0x49:
        code_key = first_value ^ 0x47
        file_type = 'gif'

    return code_key, file_type

def process_file(path, filename):
    file_path = os.path.join(path, filename)
    modification_time = datetime.fromtimestamp(os.path.getmtime(file_path)).strftime('%Y-%m-%d %H.%M.%S')
    code_key, file_type = identify_file_type(file_path)

    directory_name = path.split('\\')[-1]
    if not os.path.exists(directory_name):
        os.mkdir(directory_name)

    source_file = open(file_path, "rb")
    destination_file = open(directory_name + '\\' + modification_time + '-' + filename[:10] + '.' + file_type, "wb")

    for current_byte in source_file:
        for byte in current_byte:
            destination_file.write(bytes([byte ^ code_key]))

    source_file.close()
    destination_file.close()

def process_directory(path):
    files = os.listdir(path)
    print('Total files:{}'.format(len(files)))
    i = 0
    for filename in files:
        try:
            print('{} ... {}'.format(i, filename))
            process_file(path, filename)
            i += 1
        except:
            pass

if __name__=="__main__":
    process_directory(r"C:\Users\FrogHarkness\Documents\WeChat Files\wxid_5d6v5u1a026912\FileStorage\Image\2021-11")


# Dash & Plotly Modules
import dash
import dash_core_components as dcc
import dash_html_components as html
import dash_bootstrap_components as dbc
import dash_flexbox_grid as dfx
from dash.dependencies import Input, Output, State
import dash_table
import dash_dangerously_set_inner_html
from dash.exceptions import PreventUpdate
import plotly.express as px
import dash_dangerously_set_inner_html

#Python Generic & Local Modules
import base64
import shap, eli5
from config import *
from ml_utils_xai import *
import pandas as pd

external_stylesheets = [dbc.themes.BOOTSTRAP]
app = dash.Dash(__name__, external_stylesheets=external_stylesheets)

df = pd.read_csv('Data/Output/test.csv')
app.layout = html.Div([
    dcc.Tabs([
        dcc.Tab(label='Metrics', children=[
            html.Div([html.H2("Performance Metrics: ")], style={'marginLeft': 10, 'marginBottom': 50, 'marginTop': 25}),
            dbc.Row(
                [
                    dbc.Col(html.Div([html.Img(src = encodedImage('ROCAUC.png'), id = 'rocauc_image', height=700, width=720)])),
                    dbc.Col(html.Div([html.Img(src = encodedImage('ClassificationReport.png'), id = 'classificationreport_image', height=700, width=720)])),
                ]
            ),
            dbc.Row(
                [
                    dbc.Col(html.Div([html.Img(src = encodedImage('PrecisionRecallCurve.png'), id = 'precisionrecall_image', height=700, width=720)])),
                    dbc.Col(html.Div([html.Img(src = encodedImage('ConfusionMatrix.png'), id = 'confusion_matrix_image', height=700, width=720)])),
                ]
            )
        ]),

        dcc.Tab(label='Explainable AI - SHAP', children=[
            html.Div([html.H2(" Feature Importance With SHAP: ")], style={'marginLeft': 10, 'marginBottom': 50, 'marginTop': 25}),
            html.Div([html.H4("Please Choose a Death Event Label: ")], style={'marginLeft': 10, 'marginTop': 25}),
            dcc.Dropdown(id = 'shapLabelOption',  options=[
                    {'label':'Benign', 'value': 0},
                    {'label':'Death', 'value':1}
                ]
                , value = 1
                , style={'width':300,'paddingLeft':10 }),

            dbc.Row(
            [
                dbc.Col(html.Div([dcc.Graph(id='shapBarPlot')], style = {'marginTop':55}))
            ]
            ),
        ]),
        dcc.Tab(label='Explainable AI - ELI5', children=[
            html.Div([html.H2(" Explain Datapoint Prediction with ELI5: ")], style={'marginLeft': 10, 'marginBottom': 50, 'marginTop': 25}),

            dbc.Row(
                [
                    dbc.Col([
                            html.Div([html.H4("Test Dataset: ")], style={'marginLeft': 10, 'marginBottom': 50, 'marginTop': 25}),
                            html.Div([
                            dash_table.DataTable(
                                    data=df.to_dict('records'),
                                    columns=[{'id': c, 'name': c} for c in df.columns],
                                    style_table={'height': '400px', 'overflowY': 'auto'}
                                )
                            ], style={'marginLeft': 10, 'marginTop': 150, 'marginTop': 25}),
                            html.Div([html.H4("Enter an index row for analysis: ")], style={'marginTop':10,'marginLeft': 10, 'marginBottom': 15, 'marginTop': 25}),
                            html.Div(dbc.Input(id='rowIndex', type='number', value = 0 , style={'marginLeft': 10,'width':125})),
                            html.Div(dbc.Button('Submit', id='eli5Button', n_clicks=0, style={'marginLeft': 10, 'marginBottom': 50, 'marginTop': 10})),
                            html.Div(id='eli5Table', style = {'marginLeft': 10, 'marginBottom': 50,'marginTop':10})]),
                ],
            ),
        ]),
    ]),

])

@app.callback(Output('shapBarPlot', 'figure'),
              [Input('shapLabelOption','value')])
def shapCallback(shapLabelOption):
    """
    SHAP Global Interpretability Callback - Bar chart.
    Purpose: Visualize feature importance from SHAP package
    Steps:
        1) Convert shap_values into a df
        2) Get the columns of the training dataset
        3) Assign the same column of the training dataset to the shap values dataframe
        4) Copy the train data df and reset the index so that it is compatible with shap value df
        5) Get  the correlation coefficient and append it to a new df (corr_df) - This will be use to plot the bar chart
    """
    # Load SHAP correlation Dataframe
    shapObjectName = 'RandomForestClassifier_SHAP_LABEL_{}'.format(shapLabelOption)
    outputShapCorrelationDataframe = loadObject(shapObjectName, OUTPUT_PATH)

    # Generate Shap figure
    shapFigure = px.bar(outputShapCorrelationDataframe, x="Features", y="Correlation",title="<b>Feature Importance (Postive/Negative) Correlation <br>",
            height=650, width = 1200,color = np.where(outputShapCorrelationDataframe['Correlation']>0,'Positive Correlation','Negative Correlation'))
    return shapFigure

@app.callback(Output('eli5Table', 'children'),
    [Input('eli5Button','n_clicks')],
    [State('rowIndex', 'value')])
def eli5Callback(clicks, rowIndex):
    """
    ELI5 Local Interpretability Callback - Output results table.
    Purpose: Visualize ELI5 explanation table
    """
    modelName = "RandomForestClassifier"
    trainedModel = loadObject(modelName, OUTPUT_PATH)
    testDataset = pd.read_csv(OUTPUT_PATH + 'test.csv')

    #Calls EL5 explain prediction API - Format as HTML
    explainPredictionObject = eli5.explain_prediction(trainedModel, testDataset.iloc[rowIndex], top=10, target_names= {0: ' Benign', 1:' Death'})
    HTMLFormat = eli5.format_as_html(explainPredictionObject, show=('targets','decision_tree','transition_features'))

    #Construct ELI5 table
    tableHeader = [html.Thead(html.Tr([html.Th("Row #"+ str(rowIndex))]), style={"fontSize":14})]
    tableBody = [html.Tbody(dash_dangerously_set_inner_html.DangerouslySetInnerHTML(HTMLFormat), style={"fontSize":14})]
    outputELI5Table = dbc.Table(tableHeader + tableBody, hover=True, responsive=True, striped=True)

    return outputELI5Table
if __name__ == '__main__':
    app.run_server(debug=False)


import pytest
from src.lib.string_util import StringUtil

@pytest.fixture(scope="module", autouse=True)
def stu_object():
    stu = StringUtil()
    yield stu

class TestStringUtil:
    @pytest.mark.parametrize("test_input, expected", [('県外一例目', False), ('県内1例目', True), ('県内2外', True)])
    def test_exclude_outside(self, stu_object, test_input, expected):
        result = stu_object.exclude_outside(test_input)
        assert result == expected

    @pytest.mark.parametrize("test_input, expected_bool, expected_list", [('県内一例目', False, '県内一例目'), ('県内3,4例目', True, ['県内3例目', '県内4例目'])])
    def test_is_duplicate_data(self, stu_object, test_input, expected_bool, expected_list):
        result_bool, result_list = stu_object.is_duplicate_data(test_input)
        assert result_bool == expected_bool
        assert result_list == expected_list

if __name__ == '__main__':
    pytest.main(['-v', __file__])


import numpy as np
import matplotlib.pyplot as plt

X= [20,30,47,78,765,89,4]
y = [33,5,72,366,5,4,66]

plt.plot(X,y)
plt.xlabel('X - axis')
plt.ylabel('Y - axis')
plt.title('Example')
plt.legend('temp')

plt.show()


# -*- coding: utf-8 -*-
"""
Created on Fri Feb 17 11:39:58 2017

@author: luzhangqin
"""

import numpy as np
from sklearn.cluster import KMeans
#计算两个输入集合的每对之间的距离。
from scipy.spatial.distance import cdist
from matplotlib.font_manager import FontProperties
import matplotlib.pyplot as plt

font = FontProperties(fname = r'/usr/share/fonts/truetype/arphic/ukai.ttc', size = 10)

#输出间隔的下限。 所有生成的值将大于或等于低。 默认值为0。
#t或ints的元组，可选输出形状。
#如果给定形状是例如（m，n，k），则绘制m * n * k个样本。
#默认值为None，在这种情况下返回单个值。
cluster1 = np.random.uniform(0.5, 1.5, (2, 10))
cluster2 = np.random.uniform(3.5, 4.5, (2, 10))
#取一个数组序列，并水平堆叠，形成一个数组。
#转置
X = np.hstack((cluster1, cluster2)).T
plt.figure()
plt.axis([0, 5, 0, 5])
plt.grid(True)
plt.plot(X[:, 0], X[:, 1], 'k.')
plt.show()
#取一个数组序列，并将它们垂直堆叠以构成一个数组。
#X = np.vstack((cluster1, cluster2)).T

K = range(1, 10)
meandistortions = []
for k in K:
    kmeans = KMeans(n_clusters = k)
    kmeans.fit(X)
    #kmeans.cluster_centers_ 集群中心的坐标
    #axis如果这是一个int的元组，则选择多个轴的最小值，而不是单个轴或所有轴，如前所述。
    meandistortions.append(sum(np.min(cdist(X, kmeans.cluster_centers_,
                                            'euclidean'), axis = 1)) / X.shape[0])
plt.plot(K, meandistortions, 'bx-')
plt.xlabel('k')
plt.ylabel('平均畸变程度', fontproperties = font )
plt.title('肘部法则', fontproperties = font )


import numpy as np
import math

import torch
import torch.nn as nn
import torchvision.transforms as transforms

class OhemCELoss(nn.Module):
    """
    算法本质：
    Ohem本质：核心思路是取所有损失大于阈值的像素点参与计算，但是最少也要保证取n_min个

    """
    def __init__(self, thresh, lb_ignore=255):
        super(OhemCELoss, self).__init__()
        # self.thresh = 0.3567     -ln(0.7) = 0.3567
        # CELoss = -log(Pt)
        self.thresh = -torch.log(torch.tensor(thresh, requires_grad=False, dtype=torch.float))
        # self.lb_ignore = 255
        self.lb_ignore = lb_ignore
        self.criteria = nn.CrossEntropyLoss(ignore_index=lb_ignore, reduction='none')

    def forward(self, logits, labels):
        # logits: [2,11,1088,896]  batch,classNum,height,width
        # labels: [2,1088,896]  batch,height,width

        # 1、计算n_min（最少算多少个像素点）
        # n_min的大小：一个batch的n张h*w的label图的所有的像素点的十六分之一
        n_min = labels[labels != self.lb_ignore].numel() // 16

        # 2、交叉熵预测得到loss之后，打平成一维的
        loss = self.criteria(logits, labels).view(-1)

        # 3、所有loss中大于阈值的，这边叫做loss hard，这些点才参与损失计算
        # 注意，这里是优化了pytorch中 Ohem 排序的，不然排序太耗时间了
        loss_hard = loss[loss > self.thresh]

        # 4、如果总数小于了n_min，那么肯定要保证有n_min个
        if loss_hard.numel() < n_min:
            loss_hard, _ = loss.topk(n_min)

        # 5、如果参与的像素点的个数大于了n_min个，那么这些点都参与计算
        loss_hard_mean = torch.mean(loss_hard)

        # 6、返回损失的均值
        return loss_hard_mean


# from argparse import ArgumentParser
from PIL import Image, ImageFont, ImageDraw
from config import Input_shape, channels, threshold, ignore_thresh, path
from network_function import YOLOv3
from detect_function import predict
from utils.yolo_utils import read_anchors, read_classes, letterbox_image  # , resize_image
from timeit import default_timer as timer  # to calculate FPS
from pathlib import Path
import numpy as np
import tensorflow as tf
import argparse
import colorsys
import random
import sys
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
# from tensorflow.python.client import device_lib
# print(device_lib.list_local_devices())

class YOLO(object):
    def __init__(self):

        self.anchors_path = path + '/yolo3/model/yolo_anchors.txt'
        self.COCO = False
        self.trainable = False

        args1 = sys.argv[2]
        if args1=='COCO':
            print("-----------COCO-----------")
            self.COCO = True
            self.classes_path = path + '/yolo3/model/coco_classes.txt'
            self.trainable = False
        elif args1=='VOC':
            print("-----------VOC------------")
            self.classes_path = path + '/yolo3/model/voc_classes.txt'
        elif args1=='boat':
            print("-----------boat-----------")
            self.classes_path = path + '/yolo3/model/boat_classes.txt'

        # args = self.argument()
        # if args.COCO:
        #     print("-----------COCO-----------")
        #     self.COCO = True
        #     self.classes_path = self.PATH + '/model/coco_classes.txt'
        #     self.trainable = False
        # elif args.VOC:
        #     print("-----------VOC------------")
        #     self.classes_path = self.PATH + '/model/voc_classes.txt'
        # elif args.boat:
        #     print("-----------boat-----------")
        #     self.classes_path = self.PATH + '/model/boat_classes.txt'

        self.class_names = read_classes(self.classes_path)
        self.anchors = read_anchors(self.anchors_path)
        self.threshold = 0.5# threshold
        self.ignore_thresh = ignore_thresh
        self.INPUT_SIZE = (Input_shape, Input_shape)  # fixed size or (None, None)
        self.is_fixed_size = self.INPUT_SIZE != (None, None)
        # LOADING SESSION...
        self.boxes, self.scores, self.classes, self.sess = self.load()

    @staticmethod
    def argument():
        parser = argparse.ArgumentParser(description='COCO or VOC or boat')
        parser.add_argument('--COCO', action='store_true', help='COCO flag')
        parser.add_argument('--VOC', action='store_true', help='VOC flag')
        parser.add_argument('--boat', action='store_true', help='boat flag')
        args = parser.parse_args()
        return args
    def load(self):
        # Remove nodes from graph or reset entire default graph
        tf.reset_default_graph()

        # Generate colors for drawing bounding boxes.
        hsv_tuples = [(x / len(self.class_names), 1., 1.) for x in range(len(self.class_names))]
        self.colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
        self.colors = list(map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), self.colors))
        random.seed(10101)  # Fixed seed for consistent colors across runs.
        random.shuffle(self.colors)  # Shuffle colors to decorrelate adjacent classes.
        random.seed(None)  # Reset seed to default.

        # Generate output tensor targets for filtered bounding boxes.
        self.x = tf.placeholder(tf.float32, shape=[None, Input_shape, Input_shape, channels])
        self.image_shape = tf.placeholder(tf.float32, shape=[2, ])
        # self.is_training = tf.placeholder(tf.bool)
        # image_shape = np.array([image.size[0], image.size[1]])  # tf.placeholder(tf.float32, shape=[2,])

        # Generate output tensor targets for filtered bounding boxes.
        # scale1, scale2, scale3 = YOLOv3(self.x, len(self.class_names), trainable=self.trainable, is_training=self.is_training).feature_extractor()
        scale1, scale2, scale3 = YOLOv3(self.x, len(self.class_names), trainable=self.trainable).feature_extractor()
        scale_total = [scale1, scale2, scale3]

        # detect
        boxes, scores, classes = predict(scale_total, self.anchors, len(self.class_names), self.image_shape,
                                         score_threshold=self.threshold, iou_threshold=self.ignore_thresh)

        # Add ops to save and restore all the variables
        saver = tf.train.Saver(var_list=None if self.COCO==True else tf.trainable_variables())

        # Allowing GPU memory growth

        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        sess = tf.Session(config = config)
        sess.run(tf.global_variables_initializer())
        epoch = input('Entrer a check point at epoch:')
        # For the case of COCO
        epoch = epoch if self.COCO == False else 2000
        checkpoint = path + "/yolo3/save_model/SAVER_MODEL_boat10/model.ckpt-" + str(epoch)
        try:
            aaa = checkpoint + '.meta'
            my_abs_path = Path(aaa).resolve()
        except FileNotFoundError:
            print("Not yet training!")
        else:
            saver.restore(sess, checkpoint)
            print("checkpoint: ", checkpoint)
            print("already training!")

        return boxes, scores, classes, sess

    def detect_image(self, image):
        # Generate colors for drawing bounding boxes.
        hsv_tuples = [(x / len(self.class_names), 1., 1.) for x in range(len(self.class_names))]
        self.colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
        self.colors = list(map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), self.colors))
        random.seed(10101)  # Fixed seed for consistent colors across runs.
        random.shuffle(self.colors)  # Shuffle colors to decorrelate adjacent classes.
        random.seed(None)  # Reset seed to default.

        if self.is_fixed_size:
            assert self.INPUT_SIZE[0] % 32 == 0, 'Multiples of 32 required'
            assert self.INPUT_SIZE[1] % 32 == 0, 'Multiples of 32 required'
            boxed_image, image_shape = letterbox_image(image, tuple(reversed(self.INPUT_SIZE)))
            # boxed_image, image_shape = resize_image(image, tuple(reversed(self.INPUT_SIZE)))
        else:
            new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32))
            boxed_image, image_shape = letterbox_image(image, new_image_size)
            # boxed_image, image_shape = resize_image(image, new_image_size)
        image_data = np.array(boxed_image, dtype='float32')

        print("heights, widths:", image_shape)
        image_data /= 255.
        inputs = np.expand_dims(image_data, 0)  # Add batch dimension. #

        out_boxes, out_scores, out_classes = self.sess.run([self.boxes, self.scores, self.classes],
                                                           feed_dict={self.x: inputs,
                                                                      self.image_shape: image_shape,
                                                                      # self.is_training: False
                                                                      })

        print('Found {} boxes for {}'.format(len(out_boxes), 'img'))

        # Visualisation#################################################################################################
        font = ImageFont.truetype(font=path + '/yolo3/model/font/FiraMono-Medium.otf', size=np.floor(3e-2 * image.size[1] + 0.5).astype(np.int32))
        thickness = (image.size[0] + image.size[1]) // 500  # do day cua BB

        for i, c in reversed(list(enumerate(out_classes))):
            predicted_class = self.class_names[c]
            box = out_boxes[i]
            score = out_scores[i]

            label = '{} {:.2f}'.format(predicted_class, score)
            draw = ImageDraw.Draw(image)
            label_size = draw.textsize(label, font)

            top, left, bottom, right = box  # y_min, x_min, y_max, x_max
            top = max(0, np.floor(top + 0.5).astype(np.int32))
            left = max(0, np.floor(left + 0.5).astype(np.int32))
            bottom = min(image.size[1], np.floor(bottom + 0.5).astype(np.int32))
            right = min(image.size[0], np.floor(right + 0.5).astype(np.int32))
            print(label, (left, top), (right, bottom))  # (x_min, y_min), (x_max, y_max)

            if top - label_size[1] >= 0:
                text_origin = np.array([left, top - label_size[1]])
            else:
                text_origin = np.array([left, top + 1])

            # My kingdom for a good redistributable image drawing library.
            for j in range(thickness):
                draw.rectangle([left + j, top + j, right - j, bottom - j], outline=self.colors[c])
            draw.rectangle([tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c])
            draw.text(text_origin, label, fill=(0, 0, 0), font=font)
            del draw
        return image

def detect_video(yolo, video_path=None, output_video=None):
    import urllib.request as urllib
    import cv2

    fourcc = cv2.VideoWriter_fourcc(*'XVID')
    fps = 20.0  # display fps frame per second
    accum_time = 0
    curr_fps = 0
    prev_time = timer()
    if video_path=='stream':
        url = 'http://10.18.97.1:8080/shot.jpg'
        out = cv2.VideoWriter(output_video, fourcc, fps, (1280, 720))
        while True:

            # Use urllib to get the image and convert into a cv2 usable format
            imgResp = urllib.urlopen(url)
            imgNp = np.array(bytearray(imgResp.read()), dtype=np.uint8)
            img = cv2.imdecode(imgNp, -1)
            # print(np.shape(img))  # get w, h from here

            image = Image.fromarray(img)
            image = yolo.detect_image(image)
            result = np.asarray(image)

            curr_time = timer()
            exec_time = curr_time - prev_time
            prev_time = curr_time
            accum_time = accum_time + exec_time
            curr_fps = curr_fps + 1
            if accum_time > 1:
                accum_time = accum_time - 1
                fps = "FPS: " + str(curr_fps)
                curr_fps = 0
            cv2.putText(result, text=fps, org=(3, 15), fontFace=cv2.FONT_HERSHEY_SIMPLEX,
                        fontScale=0.50, color=(255, 0, 0), thickness=2)
            # cv2.namedWindow("result", cv2.WINDOW_NORMAL)
            cv2.imshow("Result", result)
            out.write(result)

            # To give the processor some less stress
            # time.sleep(0.1)

            if cv2.waitKey(1) & 0xFF == ord('q'):
                break
        out.release()
        # Closes all the frames
        cv2.destroyAllWindows()

        yolo.sess.close()
    else:
        cap = cv2.VideoCapture(video_path)
        if not cap.isOpened():
            raise IOError("Couldn't open webcam or video")
        # The size of the frames to write
        w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
        h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
        out = cv2.VideoWriter(output_video, fourcc, fps, (w, h))
        while True:
            ret, frame = cap.read()
            if ret==True:
                image = Image.fromarray(frame)

                image = yolo.detect_image(image)
                result = np.asarray(image)

                curr_time = timer()
                exec_time = curr_time - prev_time
                prev_time = curr_time
                accum_time = accum_time + exec_time
                curr_fps = curr_fps + 1
                if accum_time > 1:
                    accum_time = accum_time - 1
                    fps = "FPS: " + str(curr_fps)
                    curr_fps = 0
                cv2.putText(result, text=fps, org=(3, 15), fontFace=cv2.FONT_HERSHEY_SIMPLEX,
                            fontScale=0.50, color=(255, 0, 0), thickness=2)
                cv2.namedWindow("Result", cv2.WINDOW_NORMAL)
                cv2.imshow("Result", result)

                out.write(result)

                if cv2.waitKey(1) & 0xFF == ord('q'):
                    break
            else:
                break
        cap.release()
        out.release()
        # Closes all the frames
        cv2.destroyAllWindows()

        yolo.sess.close()

def detect_img(yolo, output=''):
    while True:
        img = input('Input image filename:')
        try:
            img = path + '/yolo3' + str(img)
            image = Image.open(img)
        except:
            print('Open Error! Try again!')
            continue
        else:
            r_image = yolo.detect_image(image)
            r_image.save(output)
            r_image.show()

    yolo.sess.close()

if __name__ == '__main__':
    choose = sys.argv[1]
    if choose=='image':
        output = sys.argv[3]
        detect_img(YOLO(), output)
    elif choose=='video':
        video_path = sys.argv[3]
        output = sys.argv[4]
        detect_video(YOLO(), video_path, output)


import re

import numpy as np
import torch
from nltk.translate.bleu_score import corpus_bleu
from nltk.translate.meteor_score import meteor_score
from rouge import Rouge

def cosine_similarity(v1, v2, t):
    return ((v1 * v2).sum() / (v1.norm() * v2.norm() * 2 + 1e-8) + 0.5).cpu().item()

def exponential_similarity(v1, v2, t):
    return torch.exp(-(v1 - v2).norm() / t * np.log(2)).cpu().item()

def inverse_similarity(v1, v2, t):
    return torch.pow(1 + (v1 - v2).norm() / t, -1).cpu().item()

def simple_preprocess(text):
    text = text.strip()
    text = text[0].upper() + text[1:].lower()
    text = text.replace('(', ' (').replace(')', ') ')
    text = re.sub('\s+', ' ', text).replace(' .', '.')
    if not text.endswith('.'):
        text = text + '.'
    return text

def calc_rouge(s1, s2):
    rouge = Rouge()
    score = (rouge.get_scores(s1, s2)[0]['rouge-l']['f'] + rouge.get_scores(s2, s1)[0]['rouge-l']['f']) / 2
    return score

def calc_bleu(s1, s2):
    bleu_weights = (1/4, 1/4, 1/4, 1/4)
    score = (corpus_bleu([s1.split()], [s2.split()], weights=bleu_weights) +  corpus_bleu([s2.split()], [s1.split()], weights=bleu_weights)) / 2
    return score

def calc_meteor(s1, s2):
    score = (meteor_score([s1], s2) + meteor_score([s2], s1)) / 2
    return score

def intercluster_similarity(cl1, cl2, fn):
    values = []
    for el1 in cl1:
        for el2 in cl2:
            values.append(fn(el1, el2))
    return sum(values) / len(values)

def cluster_centroid(cl, fn):
    if len(cl) == 1:
        return cl[0]
    else:
        values = [[] for _ in cl]
        for i in range(len(cl) - 1):
            for j in range(i + 1, len(cl)):
                score = fn(cl[i], cl[j])
                values[i].append(score)
                values[j].append(score)
        values = [sum(v) / len(v) for v in values]
        return cl[sorted(list(range(len(cl))), key=lambda x: values[x])[-1]]

def make_string(ranked_list):
    add_flag = True
    up_to_500 = ''
    entire_str = ''
    for l in ranked_list:
        if len(up_to_500 + ' ' + l[1].strip()) > 500:
            add_flag = False
        elif add_flag:
            up_to_500 = up_to_500 + ' ' + l[1].strip()
        entire_str += '%.6f\t%s\n' % (l[0], l[1])
    return up_to_500, entire_str


from skimage.io import imread
from matplotlib import pyplot as plt
import numpy
from scipy import stats

RANGE = 256

# utility function plotting histogram
def plot_hist(img, title, pixel_value_range = RANGE):
    image_hist, bins = numpy.histogram(img, bins=range(pixel_value_range))
    center = (bins[:-1] + bins[1:]) / 2
    plt.title(title)
    plt.bar(center, image_hist, width=1)
    plt.show()

# utility function plotting image
def grey_image_show(img, title):
    plt.title(title)
    plt.imshow(img, 'Greys_r')
    plt.show()

# g(x,y) = T[f(x,y)] = L - f(x,y)
def hist_flip_transform_function(pixel, pixel_value_range):
    return pixel_value_range - pixel

# flip whole image
def hist_flip(img, pixel_value_range=RANGE):
    return hist_flip_transform_function(img, pixel_value_range)

# get CDF of an image as a discrete function
def get_cumulative_density_vector(img, value_range=RANGE):
    hist, bins = numpy.histogram(img, bins=value_range, range=(0, value_range))
    h_r = hist / img.size
    return numpy.cumsum(h_r)

# get exponential CDF over range as a discrete function
def get_exponential_cumulative_density_vector(exp_base, value_range=RANGE):
    nu = numpy.float64(exp_base ** value_range) / (exp_base - 1)
    alpha = numpy.divide(1, nu - 1)

    def ps(j):
        return numpy.multiply(alpha, numpy.float64(exp_base ** j))

    h_s = ps(numpy.linspace(0, RANGE - 1, RANGE))
    return numpy.cumsum(h_s)

# main functionality of hw6_a
def hw6_a(img):
    transformed_image = hist_flip(img)

    grey_image_show(img, 'Original')
    plot_hist(img, 'Original Histogram')
    grey_image_show(transformed_image, 'Transformed image')
    plot_hist(transformed_image, 'Transformed histogram')

# main functionality of hw6_b
def hw6_b(img, exp_base):

    H_r = get_cumulative_density_vector(img)
    H_s = get_exponential_cumulative_density_vector(exp_base)

    lookup = numpy.ndarray((RANGE,))

    for i in range(RANGE):
        diff = 10
        prominent_j = 0
        for j in range(RANGE):
            j_diff = numpy.abs(H_r[i] - H_s[j])
            if j_diff <= diff:
                diff = j_diff
                prominent_j = j
            else:
                break
        lookup[i] = prominent_j

    def t(pixel):
        return lookup[int(pixel)]

    vt = numpy.vectorize(t)

    transformed_image = vt(img)

    grey_image_show(transformed_image, 't_img')
    plot_hist(transformed_image, 'hist')

def main():
    image = numpy.floor(imread('./data/ax.jpg', as_gray=True) * RANGE) - 1
    hw6_a(image)
    hw6_b(image, exp_base=1.05)


from dataclasses import dataclass
from typing import Dict, Generator, NamedTuple, OrderedDict, Union

from starknet_py.serialization._context import (
    DeserializationContext,
    SerializationContext,
)
from starknet_py.serialization.data_serializers._common import (
    deserialize_to_dict,
    serialize_from_dict,
)
from starknet_py.serialization.data_serializers.cairo_data_serializer import (
    CairoDataSerializer,
)
from starknet_py.serialization.tuple_dataclass import TupleDataclass

@dataclass
class NamedTupleSerializer(
    CairoDataSerializer[Union[Dict, NamedTuple, TupleDataclass], TupleDataclass]
):
    """
    Serializer for tuples with named fields.
    Can serialize a dictionary, a named tuple and TupleDataclass.
    Deserializes data to a TupleDataclass.

    Example:
    {"a": 1, "b": 2} => [1,2]
    """

    serializers: OrderedDict[str, CairoDataSerializer]

    def deserialize_with_context(
        self, context: DeserializationContext
    ) -> TupleDataclass:
        as_dictionary = deserialize_to_dict(self.serializers, context)
        return TupleDataclass.from_dict(as_dictionary)

    def serialize_with_context(
        self,
        context: SerializationContext,
        value: Union[Dict, NamedTuple, TupleDataclass],
    ) -> Generator[int, None, None]:
        # We can't use isinstance(value, NamedTuple), because there is no NamedTuple type.
        context.ensure_valid_type(
            value,
            isinstance(value, (dict, TupleDataclass)) or self._is_namedtuple(value),
            "dict, NamedTuple or TupleDataclass",
        )

        # noinspection PyUnresolvedReferences, PyProtectedMember
        values: Dict = value if isinstance(value, dict) else value._asdict()

        yield from serialize_from_dict(self.serializers, context, values)

    @staticmethod
    def _is_namedtuple(value) -> bool:
        return isinstance(value, tuple) and hasattr(value, "_fields")


#!/usr/bin/env python3
# coding: utf-8
"""
@file: test_reasder.py
@description:
@author: Yiran Wu
@email: wuyiran@genomics.cn
@last modified by: Yiran Wu

change log:
    2021/08/13  create file.
"""
import numpy as np

import pandas as pd

from stereo.core.stereo_exp_data import StereoExpData
from stereo.preprocess.qc import cal_qc
from stereo.plots.qc import plot_violin_distribution
from stereo.plots.qc import save_fig
from stereo.plots.qc import plot_genes_count
from stereo.plots.qc import plot_spatial_distribution
from stereo.io.reader import read
from stereo.io.reader import read_txt
from stereo.io.reader import read_stereo
from stereo.io.reader import read_ann_h5ad
from stereo.io.writer import write
import sys
import scanpy as sc

def make_data():
    import numpy as np
    from scipy import sparse
    genes = ['g1', 'MT-111', 'g3']
    rows = [0, 1, 0, 1, 2, 0, 1, 2]
    cols = [0, 0, 1, 1, 1, 2, 2, 2]
    cells = ['c1', 'c2', 'c3']
    v = [2, 3, 4, 5, 6, 3, 4, 5]
    exp_matrix = sparse.csr_matrix((v, (rows, cols)))
    # exp_matrix = sparse.csr_matrix((v, (rows, cols))).toarray()
    position = np.random.randint(0, 10, (len(cells), 2))
    out_path = sys.argv[0]+"/F5.h5ad"
    data = StereoExpData(bin_type='cell_bins', exp_matrix=exp_matrix, genes=np.array(genes),
                         cells=np.array(cells), position=position, output=out_path)
    return data

def plot(data,out):
    """

    :param data:
    :return:
    """
    plot_violin_distribution(data)
    save_fig(out+"/F5.vio.png")
    plot_genes_count(data)
    save_fig(out+"/F5.gene.png")
    plot_spatial_distribution(data)
    save_fig(out+"/F5.spa.png")
    # plot_genes_count(data)

def check_read_txt(out) :
    data = read_txt(out+"/F5.gem.txt",bin_type="bins",bin_size=200)
    myout = out+"/check_read_txt"
    check_out(myout)
    data=cal_qc(data)
    plot(data,myout)
    write(data, output=myout + "/F5.bin200.h5ad")
    print("finished in " + myout)
    print(data.to_df())

def check_read_stereo(out):
    data = read_stereo(out+"/check_write/F5.bin200.h5ad")
    myout = out+"/check_read_stereo"
    check_out(myout)
    data=cal_qc(data)
    plot(data,myout)
    write(data, output=myout + "/F5.bin200.h5ad")
    print("finished in " + myout)
    print(data.to_df())

def check_read_ann_h5ad(out):
    data = read_ann_h5ad(out+"/scanpy/raw_andata.bin200.h5ad")
    myout = out+"/check_read_ann_h5ad"
    check_out(myout)
    plot(data,myout)
    write(data,output=myout+"/F5.bin200.h5ad")
    print("finished in " + myout)
    print(data.to_df())

def check_stereo_to_andata(out):
    data = read(file_path=out+"/F5.gem.txt",file_format='txt',bin_type="bins",bin_size=200)
    #data=make_data()
    data = cal_qc(data)
    adata=data.to_andata()
    sc.write(out+"/check_stereo_to_andata/F5.stereo_to_andata.h5ad",adata)
    print(adata)

def check_andata_to_stereo(out):
    andata = sc.read(out+"/scanpy/raw_andata.bin200.h5ad")
    data=StereoExpData()
    data.andata_to_stereo(andata)

def check_read_h5ad(out):
    import numpy as np
    data=StereoExpData()
    data.read_h5ad_anndata(out+"/scanpy/raw_andata.bin200.h5ad")
    plot(data, out+"/check_read_h5ad")
    #print(data.cells.to_df())

def check_out(out):
    from pathlib import Path
    myout = Path(out)
    if not myout.exists():
        myout.mkdir()

if __name__ == '__main__':
    out=sys.argv[1]
    print(out)
    check_read_txt(out)
    check_read_stereo(out)
    check_read_ann_h5ad(out)
    check_stereo_to_andata(out)
    '''
    data=make_data()
    data = cal_qc(data)
    print(data.cells.to_df())
    print(data.genes.to_df())
    #data = read_stereo(out+"/F5.gem","bins",200)
    ##test andata_to_stereo
    #print(data.gene_names)
#    print(data.cells.n_genes_by_counts)
    #plot(data,out)
    #plot_genes_count(data)
#    print(data.cells.total_counts)
#    print(data.cells.n_genes_by_counts)
#    print(data.position)
'''


# An implementation of a linear SVM that uses gradient descent for training
import numpy as np
import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split
from sklearn.datasets import load_breast_cancer
from sklearn.preprocessing import StandardScaler
from datetime import datetime
from util import get_clouds

# Define the Linear SVM using a class
class LinearSVM:
    def __init__(self, C = 1.0):
        self.C = C

    def _objective(self, margins):
        return 0.5 * self.w.dot(self.w) + self.C * np.maximum(0, 1 - margins).sum()

    def fit(self, X, Y, lr = 1e-5, n_iters=400):
        N, D = X.shape
        self.N = N
        self.w = np.random.randn(D)
        self.b = 0

        # Gradient descent
        losses = []
        for _ in range(n_iters):
            margins = Y * self._decision_function(X)
            loss = self._objective(margins)
            losses.append(loss)

            idx = np.where(margins < 1)[0]
            grad_w = self.w - self.C * Y[idx].dot(X[idx])
            self.w -= lr*grad_w
            grad_b = -self.C * Y[idx].sum()
            self.b -= lr * grad_b

        self.support_ = np.where((Y * self._decision_function(X)) <= 1)[0]
        print("num SVs:", len(self.support_))

        print("w:", self.w)
        print("b:", self.b)

        plt.plot(losses)
        plt.title("Loss per iteration")
        plt.show()

    def _decision_function(self, X):
        return X.dot(self.w) + self.b

    def predict(self, X):
        return np.sign(self._decision_function(X))

    def score(self, X, Y):
        P = self.predict(X)
        return np.mean(Y == P)

def plot_decision_boundary(model, X, Y, resolution=100, colors=('b', 'k', 'r')):
    np.warnings.filterwarnings('ignore')
    fig, ax = plt.subplots()

    # Generate a coordinate grid of shape [resolution x resolution]
    # Evaluate the model over the entire space
    x_range = np.linspace(X[:,0].min(), X[:,0].max(), resolution)
    y_range = np.linspace(X[:,1].min(), X[:,1].max(), resolution)
    grid = [[model._decision_function(np.array([[xr, yr]])) for yr in y_range] for xr in x_range]
    grid = np.array(grid).reshape(len(x_range), len(y_range))

    # Plot decision contours using grid and make a scatter plot of training data
    ax.contour(x_range, y_range, grid.T, (-1, 0, 1), linewidths=(1,1,1), linestyles = ('--', '-', '--'), colors=colors)
    ax.scatter(X[:,0], X[:,1], c=Y, lw=0, alpha=0.3, cmap='seismic')

    # Plot support vectors (non-zero alphas) as circled points (linewidth > 0)
    mask = model.support_
    ax.scatter(X[:,0][mask], X[:,1][mask], c=Y[mask], cmap='seismic')

    # Debug
    ax.scatter([0], [0], c='black', marker='x')
    plt.show()

def clouds():
    X, Y = get_clouds()
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.33)
    return X_train, X_test, Y_train, Y_test, 1e-3, 200

def medical():
    data = load_breast_cancer()
    X, Y = data.data, data.target
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.33)
    return X_train, X_test, Y_train, Y_test, 1e-3, 200

if __name__ == '__main__':
    X_train, X_test, Y_train, Y_test, lr, n_iters = clouds()
    print("Possible labels:", set(Y_train))

    # Make sure the targets are (-1, +1)
    Y_train[Y_train == 0] = -1
    Y_test[Y_test == 0] = -1

    # Scale the data
    scaler = StandardScaler()
    X_train = scaler.fit_transform(X_train)
    X_test = scaler.transform(X_test)

    model = LinearSVM(C=1.0)

    t0 = datetime.now()
    model.fit(X_train, Y_train, lr=lr, n_iters=n_iters)
    print("Train duration:", datetime.now() - t0)
    t0 = datetime.now()
    print("Train score:", model.score(X_train, Y_train), "Duration:", datetime.now() -t0)
    t0 = datetime.now()
    print("Test score:", model.score(X_test, Y_test), "Duration:", datetime.now() - t0)

    if X_train.shape[1] == 2:
        plot_decision_boundary(model, X_train, Y_train)


import flet as ft

def main(page: ft.Page):
    page.title = "Flet Band Name Generator"
    # page.vertical_alignment = "center"
    def reset_textfields():
        city.error_text = ""
        pet.error_text = ""

    def button_clicked(e):
        reset_textfields()
        valid_form = True

        if city.value == "":
            city.error_text = "City cannot be blank."
            valid_form = False
        if pet.value == "":
            pet.error_text = "Pet name connot be blank."
            valid_form = False

        if valid_form:
            band_name.value = f"Your band name could be {city.value} {pet.value}."
        page.update()

    city = ft.TextField(label="Name of the city you grew up in")
    pet = ft.TextField(label="Name of your pet")
    band_name = ft.Text("Your band name could be [wait for it].")

    page.add(
        city,
        pet,
        ft.ElevatedButton(text="Generate your band name", on_click=button_clicked),
        band_name,
    )

ft.app(target=main)


#!/usr/bin/env python3

# Version 1.0
# Author Alexis Blanchet-Cohen
# Date: 24/02/2014

import argparse
import os
import subprocess

parser = argparse.ArgumentParser(description='Set up analysis directories and configuration files.')
parser.add_argument("directory", nargs=1, help="Project directory. If the directory does not exist, it will be created.")
parser.add_argument("-c", "--configuration", help="Configuration file. Choices=bisseq, chipseq, rnaseq, or user-defined file", default="rnaseq")
args = parser.parse_args()

pipelines_folder = os.environ["PIPELINES_FOLDER"]

# Process command line arguments
directory = args.directory[0]

if args.configuration == "bisseq":
    configuration_file = os.path.join(pipelines_folder, "configuration_files/bisseq/configuration_project.txt")
elif args.configuration == "chipseq":
    configuration_file = os.path.join(pipelines_folder, "configuration_files/chipseq/configuration_project.txt")
elif args.configuration == "rnaseq":
    configuration_file = os.path.join(pipelines_folder, "configuration_files/rnaseq/configuration_project.txt")
else:
    configuration_file = os.path.abspath(args.configuration)

# Create project directory, if they do not exist yet, and change to main project directory.
if not os.path.exists(directory):
    os.mkdir(directory)
    print("Project directory created: " + directory)
else:
    print("No need to create project directory. Directory already exists: " + directory)
os.chdir(directory)

#Create subdirectoroes, if they do not exist yet.
#if not os.path.exists("data/BAM_files"):
#    os.makedirs("data/BAM_files")

if not os.path.exists("data/FASTQ_files/untrimmed"):
    os.makedirs("data/FASTQ_files/untrimmed")

if not os.path.exists("scripts"):
    os.makedirs("scripts")

if not os.path.exists("results"):
    os.makedirs("results")

# Create link to report file
subprocess.call("ln -s scripts/report/report.pdf report.pdf", shell=True)

# Copy configuration file to Analysis/scripts
subprocess.call("cp " + configuration_file + " scripts", shell=True)
print("Configuration file copied to " + directory + "/scripts:\n" + configuration_file)


import requests
import string
from flask import Flask, render_template, request, redirect, url_for, flash, jsonify
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)

app.config['DEBUG'] = True
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///weather.db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SECRET_KEY'] = 'thisisasecret'
db = SQLAlchemy(app)

#SQL Alchemy class database
class City(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(50), nullable=False)

#Get API informations
def get_weather_data(city):
    url = f"http://api.openweathermap.org/data/2.5/weather?q={city}&units=metric&appid=b21a2633ddaac750a77524f91fe104e7"
    r = requests.get(url).json()
    return r

# Get information from API, store the data in database
@app.route('/')
def index_get():
    cities = City.query.all()

    weather_data = []

    for city in cities:
        r = get_weather_data(city.name)
        weather = {
            'city': city.name,
            'temperature': r['main']['temp'],
            'description': r['weather'][0]['description'],
            'icon': r['weather'][0]['icon'],
        }
        weather_data.append(weather)

    return render_template('weather.html', weather_data=weather_data) #Render in HTML file
    #return jsonify(weather)  #Render in JSON file

#informs validation message (If you already have the city or
# not in the database)
@app.route('/', methods=['POST'])
def index_post():
    err_msg = ''
    new_city = request.form.get('city')
    new_city = new_city.lower()
    new_city = string.capwords(new_city)
    if new_city:
        existing_city = City.query.filter_by(name=new_city).first()

        if not existing_city:
            new_city_data = get_weather_data(new_city)
            if new_city_data['cod'] == 200:
                new_city_obj = City(name=new_city)

                db.session.add(new_city_obj)
                db.session.commit()
            else:
                err_msg = 'Sorry. We couldnt find the specific city!'
        else:
            err_msg = 'City already exists in the database!'

    if err_msg:
        flash(err_msg, 'error')
    else:
        flash('City added successfully!', 'success')

    return redirect(url_for('index_get'))

#Delete the data (cities)
@app.route('/delete/<name>')
def delete_city(name):
    city = City.query.filter_by(name=name).first()
    db.session.delete(city)
    db.session.commit()

    flash(f'Successfully deleted {city.name}!', 'success')
    return redirect(url_for('index_get'))


import sys
import requests
import datetime
import time
from pytz import timezone
import sendgrid
from sendgrid.helpers.mail import *
import os
from dotenv import load_dotenv

def send_email(subject, content):
	load_dotenv()
	sg = sendgrid.SendGridAPIClient(api_key=os.environ.get('SENDGRID_API_KEY'))
	from_email = Email(os.environ.get('SENDGRID_FROM_EMAIL'))
	to_email = To(os.environ.get('SENDGRID_TO_EMAIL'))
	subject = subject
	content = Content("text/html", content)
	mail = Mail(from_email, to_email, subject, content)
	response = sg.client.mail.send.post(request_body=mail.get())

def check_updates(url, frequency=60):

	#Time Zone formatting
	gmt = timezone('GMT')
	eastern = timezone('US/Eastern')
	input_datetime_format = "%a, %d %b %Y %H:%M:%S %Z"
	output_datetime_format = "%b %d %Y %H:%M %Z"

	prev_check = None

	while url:
		response = requests.head(url)

		if response.status_code==200:

			datetime_str = response.headers["last-modified"] #Thu, 29 Oct 2020 14:42:12 GMT
			current = datetime.datetime.now(eastern)
			most_recent_check = gmt.localize(datetime.datetime.strptime(datetime_str, input_datetime_format)).astimezone(eastern)

			if (prev_check and most_recent_check > prev_check):
				subject = "[HTTP Update Checker] File has been updated on {most_recent_check}"\
					.format(most_recent_check=most_recent_check.strftime(output_datetime_format))
				content = "[{current}] The last-modified date for  monitored file \
				 changed from: {prev_check} to: {most_recent_check}.
\
				 This is an automated message. Do not reply."\
					.format(current=current.strftime("%b %d %Y %H:%M:%S %Z"), url=url,\
					prev_check=prev_check.strftime(output_datetime_format),\
					most_recent_check=most_recent_check.strftime(output_datetime_format))
				print(subject)
				send_email(subject, content)
			else:
				print("[{current}] No updates, most recent update on: {most_recent_check}"\
					.format(current=current.strftime(output_datetime_format),\
					most_recent_check=most_recent_check.strftime(output_datetime_format)))

			prev_check = most_recent_check

		time.sleep(frequency)

if __name__ == '__main__':
	# First argument must be a file url, i.e:
	# python check_updates.py "https://cdn.star.nesdis.noaa.gov/GOES16/ABI/FD/GEOCOLOR/678x678.jpg"
	check_updates(*sys.argv[1:])


import json
import requests

game_id = 453253378
r = requests.post(
	'https://www.codingame.com/services/gameResultRemoteService/findByGameId',
	json = [str(game_id), None]
)
replay = r.json()
print(replay)
with open(f'{game_id}.json', 'w+') as f:
	f.write(json.dumps(replay))


from datetime import datetime

# GMT string to GMT Time

dateString = "Oct 28 23:59:59 2020 GMT"
dateFormatter = "%b %d %H:%M:%S %Y GMT"
print(datetime.strptime(dateString, dateFormatter))

# GMT Time to GMT String

x = datetime.now()       #現在時間
print(x.strftime("%Y-%m-%d %H:%M:%S"))

# x = datetime(2018, 6, 1)    # 設定 datetime 物件的時間


from collections import defaultdict

class Solution(object):
    def shortestAlternatingPaths(self, n, red_edges, blue_edges):
        """
        :type n: int
        :type red_edges: List[List[int]]
        :type blue_edges: List[List[int]]
        :rtype: List[int]
        """
        redGraph = defaultdict(list)
        blueGraph = defaultdict(list)

        for i, j in red_edges:
            redGraph[i].append(j)
        print(redGraph)
        for i, j in blue_edges:
            blueGraph[i].append(j)

        red_seen = set()
        blue_seen = set()
        queue = []
        queue.append((0, 0))
        queue.append((0, 1))
        red_seen.add(0)
        blue_seen.add(0)

        step = 0
        res = [float("inf") for i in range(n)]

        while queue:
            size = len(queue)
            while size > 0:
                node, color = queue.pop(0)
                if color == 0:
                    for nex in blueGraph[node]:
                        if nex not in blue_seen:
                            queue.append((nex, 1))
                            blue_seen.add(nex)
                            res[nex] = min(res[nex], step + 1)
                else:
                    for nex in redGraph[node]:
                        if nex not in red_seen:
                            queue.append((nex, 0))
                            red_seen.add(nex)
                            res[nex] = min(res[nex], step + 1)
                size -= 1
            step += 1
        res[0] = 0
        return [i if i != float("inf") else -1 for i in res]

python/BFS/Shortest Path with Alternating Colors/Shortest Path with Alternating Colors.py


import discord

users = {
    'Dark#6189': {
        'rank': 'F',
        'wins': 0,
        'gold': 0,  # gold gained from fighting monsters
        'inventory': {}  # inventory to store items from shop
    },
    'user2': {
        'rank': 'A',
        'wins': 3,
        'gold': 0,
        'inventory': {}
    },
    'user3': {
        'rank': 'A',
        'wins': 5,
        'gold': 0,
        'inventory': {}
    },
}

def leaderboard(dic):
    """
    Returns an embed of the leaderboard
    :param dic: a dictionary of users
    :return: embeds of the leaderboard
    """
    s = []
    a = []
    b = []
    c = []
    d = []
    f = []

    for key in dic:
        if dic[key]['rank'] == 'S':
            s.append({key: dic[key]['wins']})
        elif dic[key]['rank'] == 'A':
            a.append({key: dic[key]['wins']})
        elif dic[key]['rank'] == 'B':
            b.append({key: dic[key]['wins']})
        elif dic[key]['rank'] == 'C':
            c.append({key: dic[key]['wins']})
        elif dic[key]['rank'] == 'D':
            d.append({key: dic[key]['wins']})
        elif dic[key]['rank'] == 'F':
            f.append({key: dic[key]['wins']})

    embed = discord.Embed(title="Leaderboard!", colour=discord.Color.from_rgb(255, 255, 0),
                          description="Our Top 5 Players From Each Rank")
    embed.set_thumbnail(
        url="https://media.istockphoto.com/id/1176397624/vector/vector-flat-golden-trophy.jpg?s=612x612&w=0&k=20&c=kjnN3SB3l1cAMMt5xUvnyJDfPzQKzZ_pZHt3jaFnmF0=")
    embed.add_field(name="Rank S Users", value=list_to_string(s), inline=False)
    embed.add_field(name="Rank A Users", value=list_to_string(a), inline=False)
    embed.add_field(name="Rank B Users", value=list_to_string(b), inline=False)
    embed.add_field(name="Rank C Users", value=list_to_string(c), inline=False)
    embed.add_field(name="Rank D Users", value=list_to_string(d), inline=False)
    embed.add_field(name="Rank F Users", value=list_to_string(f), inline=False)

    return embed

def quick_sort(sequence):
    """
    Sorts a list of dictionaries by the value of the dictionary
    :param sequence: a list of dictionaries
    :return: a sorted list of dictionaries
    """
    length = len(sequence)
    if length <= 1:
        return sequence
    else:
        pivot = sequence.pop()

    items_greater = []
    items_lower = []

    for item in sequence:
        if list(item.values())[0] > list(pivot.values())[0]:
            items_greater.append(item)
        else:
            items_lower.append(item)

    return quick_sort(items_greater) + [pivot] + quick_sort(items_lower)

def list_to_string(items):
    """
    Returns a string of the top 5 users from each rank
    :param items: a list of dictionaries
    :return: a string of the top 5 users from each rank
    """
    rank = ""
    new = quick_sort(items)
    for index, info in enumerate(new):
        if index == len(new) - 1:
            tmp = f"{list(info.keys())[0]}: {list(info.values())[0]} wins"
            rank += tmp
        else:
            tmp = f"{list(info.keys())[0]}: {list(info.values())[0]} wins\n"
            rank += tmp

    if len(rank) == 0:
        rank = "<none>"

    return rank