import logging
import math
import sys
from collections import defaultdict

import numpy as np
import pandas as pd
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import explained_variance_score
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_squared_log_error
from sklearn.metrics import median_absolute_error
from sklearn.metrics import precision_score
from sklearn.metrics import r2_score
from sklearn.metrics import recall_score
from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve

from common.python.utils import log_utils
from kernel.components.evaluation.param import EvaluateParam
from kernel.model_base import ModelBase
from kernel.utils import consts

LOGGER = log_utils.get_logger()

class PerformanceRecorder():
    """
    This class record performance(single value metrics during the training process)
    """

    def __init__(self):

        # all of them are single value metrics
        self.allowed_metric = [consts.AUC,
                               consts.EXPLAINED_VARIANCE,
                               consts.MEAN_ABSOLUTE_ERROR,
                               consts.MEAN_SQUARED_ERROR,
                               consts.MEAN_SQUARED_LOG_ERROR,
                               consts.MEDIAN_ABSOLUTE_ERROR,
                               consts.R2_SCORE,
                               consts.ROOT_MEAN_SQUARED_ERROR,
                               consts.PRECISION,
                               consts.RECALL,
                               consts.ACCURACY,
                               consts.KS
                               ]

        self.larger_is_better = [consts.AUC,
                                 consts.R2_SCORE,
                                 consts.PRECISION,
                                 consts.RECALL,
                                 consts.EXPLAINED_VARIANCE,
                                 consts.ACCURACY,
                                 consts.KS
                                 ]

        self.smaller_is_better = [consts.ROOT_MEAN_SQUARED_ERROR,
                                  consts.MEAN_ABSOLUTE_ERROR,
                                  consts.MEAN_SQUARED_ERROR,
                                  consts.MEAN_SQUARED_LOG_ERROR]

        self.cur_best_performance = {}

        self.no_improvement_round = {}  # record no improvement round of all metrics

    def has_improved(self, val: float, metric: str, cur_best: dict):

        if metric not in cur_best:
            return True

        if metric in self.larger_is_better and val > cur_best[metric]:
            return True

        elif metric in self.smaller_is_better and val < cur_best[metric]:
            return True

        return False

    def update(self, eval_dict: dict):
        """

        Parameters
        ----------
        eval_dict dict, {metric_name:metric_val}, e.g. {'auc':0.99}

        Returns stop flag, if should stop return True, else False
        -------
        """
        if len(eval_dict) == 0:
            return

        for metric in eval_dict:
            if metric not in self.allowed_metric:
                continue
            if self.has_improved(eval_dict[metric], metric, self.cur_best_performance):
                self.cur_best_performance[metric] = eval_dict[metric]
                self.no_improvement_round[metric] = 0
            else:
                self.no_improvement_round[metric] += 1

class Evaluation(ModelBase):

    def __init__(self):
        super().__init__()
        self.model_param = EvaluateParam()
        self.eval_results = defaultdict(list)

        self.save_single_value_metric_list = [consts.AUC,
                                              consts.EXPLAINED_VARIANCE,
                                              consts.MEAN_ABSOLUTE_ERROR,
                                              consts.MEAN_SQUARED_ERROR,
                                              consts.MEAN_SQUARED_LOG_ERROR,
                                              consts.MEDIAN_ABSOLUTE_ERROR,
                                              consts.R2_SCORE,
                                              consts.ROOT_MEAN_SQUARED_ERROR]
        self.save_curve_metric_list = [consts.KS, consts.ROC, consts.LIFT, consts.GAIN, consts.PRECISION, consts.RECALL,
                                       consts.ACCURACY]

        self.metrics = None
        self.round_num = 6

        self.validate_metric = {}
        self.train_metric = {}

    def _init_model(self, model):
        self.model_param = model
        self.eval_type = self.model_param.eval_type
        self.pos_label = self.model_param.pos_label
        self.metrics = model.metrics

    def _run_data(self, data_sets=None, stage=None):
        if not self.need_run:
            return

        data = {}
        for data_key in data_sets:
            if data_sets[data_key].get("data", None):
                data[data_key] = data_sets[data_key]["data"]

        if stage == "fit":
            self.data_output = self.fit(data)
        else:
            LOGGER.warning("Evaluation has not transform, return")

    def split_data_with_type(self, data: list) -> dict:

        split_result = defaultdict(list)
        for value in data:
            mode = value[1][4]
            split_result[mode].append(value)

        return split_result

    def evaluate_metircs(self, mode: str, data: list) -> dict:
        labels = []
        pred_scores = []
        pred_labels = []

        for d in data:
            labels.append(d[1][0])
            pred_labels.append(d[1][1])
            pred_scores.append(d[1][2])

        if self.eval_type == consts.BINARY or self.eval_type == consts.REGRESSION:
            if self.pos_label and self.eval_type == consts.BINARY:
                new_labels = []
                for label in labels:
                    if self.pos_label == label:
                        new_labels.append(1)
                    else:
                        new_labels.append(0)
                labels = new_labels

            pred_results = pred_scores
        else:
            pred_results = pred_labels

        eval_result = defaultdict(list)

        metrics = self.metrics

        for eval_metric in metrics:
            res = getattr(self, eval_metric)(labels, pred_results)
            if res is not None:
                try:
                    if math.isinf(res):
                        res = float(-9999999)
                        LOGGER.info("res is inf, set to {}".format(res))
                except:
                    pass

                eval_result[eval_metric].append(mode)
                eval_result[eval_metric].append(res)

        return eval_result

    def fit(self, data, return_result=False):
        if len(data) <= 0:
            return

        self.eval_results.clear()
        for (key, eval_data) in data.items():
            eval_data_local = list(eval_data.collect())
            split_data_with_label = self.split_data_with_type(eval_data_local)
            for mode, data in split_data_with_label.items():
                eval_result = self.evaluate_metircs(mode, data)
                self.eval_results[key].append(eval_result)

        return self.callback_metric_data(return_single_val_metrics=return_result)

    def __save_single_value(self, result, metric_name, metric_namespace, eval_name):
        self.tracker.saveMetricData(metric_name, metric_namespace, {'metric_type': 'EVALUATION_SUMMARY'},
                                    [(eval_name, np.round(result, self.round_num))])

    def __save_curve_metric(self, x_axis_list, y_axis_list, metric_name, metric_namespace, metric_type,
                            abscissa_name=None, ordinate_name=None,
                            curve_name=None, best=None, pair_type=None, thresholds=None):
        extra_metas = {}
        metric_type = "_".join([metric_type, "EVALUATION"])
        key_list = ["metric_type", "abscissa_name", "ordinate_name", "curve_name", "best", "pair_type", "thresholds"]
        for key in key_list:
            value = locals()[key]
            if value:
                if key == "thresholds":
                    value = np.round(value, self.round_num).tolist()
                extra_metas[key] = value

        points = []
        for i, value in enumerate(x_axis_list):
            if isinstance(value, float):
                value = np.round(value, self.round_num)
            points.append((value, np.round(y_axis_list[i], self.round_num)))
        points.sort(key=lambda x: x[0])

        self.tracker.saveMetricData(metric_name, metric_namespace, metric_meta=extra_metas, kv=points)

    def __filt_override_unit_ordinate_coordinate(self, x_sets, y_sets):
        max_y_dict = {}
        for idx, x_value in enumerate(x_sets):
            if x_value not in max_y_dict:
                max_y_dict[x_value] = {"max_y": y_sets[idx], "idx": idx}
            else:
                max_y = max_y_dict[x_value]["max_y"]
                if max_y < y_sets[idx]:
                    max_y_dict[x_value] = {"max_y": y_sets[idx], "idx": idx}

        x = []
        y = []
        idx_list = []
        for key, value in max_y_dict.items():
            x.append(key)
            y.append(value["max_y"])
            idx_list.append(value["idx"])

        return x, y, idx_list

    def __save_roc(self, data_type, metric_name, metric_namespace, metric_res):
        fpr, tpr, thresholds, _ = metric_res

        # set roc edge value
        fpr.append(1.0)
        tpr.append(1.0)

        fpr, tpr, idx_list = self.__filt_override_unit_ordinate_coordinate(fpr, tpr)
        edge_idx = idx_list[-1]
        if edge_idx == len(thresholds):
            idx_list = idx_list[:-1]
        thresholds = [thresholds[idx] for idx in idx_list]

        self.__save_curve_metric(fpr, tpr, metric_name=metric_name, metric_namespace=metric_namespace,
                                 metric_type="ROC", abscissa_name="fpr", ordinate_name="tpr",
                                 curve_name=data_type, thresholds=thresholds)

    def __save_topn(self, metric_name, metric_namespace, metric_res):
        metric_type = "TOPN"
        metric_type = "_".join([metric_type, "EVALUATION"])
        key_list = ["name", "cut_off", "TP", "total", "acc", "recall"]
        res_list = []
        for mectric in metric_res:
            metric_dict = dict(zip(key_list, mectric))
            res_list.append(metric_dict)

        self.tracker.save_metric_data_to_task_result(metric_name, metric_namespace,
                                                     metric_meta={"metric_type": metric_type}, kv={"topn": res_list},
                                                     need_value=False)

    def callback_metric_data(self, return_single_val_metrics=False):

        """
        Parameters
        ----------
        return_single_val_metrics if True return single_val_metrics

        Returns None or return_result dict
        -------
        """

        collect_dict = {}
        LOGGER.debug('callback metric called')

        for (data_type, eval_res_list) in self.eval_results.items():

            precision_recall = {}
            for eval_res in eval_res_list:
                for (metric, metric_res) in eval_res.items():
                    metric_namespace = metric_res[0]

                    if metric_namespace == 'validate':
                        collect_dict = self.validate_metric
                    elif metric_namespace == 'train':
                        collect_dict = self.train_metric

                    metric_name = '_'.join([data_type, metric])

                    if metric in self.save_single_value_metric_list:
                        self.__save_single_value(metric_res[1], metric_name=data_type, metric_namespace=metric_namespace
                                                 , eval_name=metric)
                        collect_dict[metric] = metric_res[1]

                    elif metric == consts.KS:
                        best_ks, fpr, tpr, thresholds, cuts = metric_res[1]
                        self.__save_single_value(best_ks, metric_name=data_type,
                                                 metric_namespace=metric_namespace,
                                                 eval_name=metric)
                        collect_dict[metric] = best_ks

                        metric_name_fpr = '_'.join([metric_name, "fpr"])
                        curve_name_fpr = "_".join([data_type, "fpr"])
                        self.__save_curve_metric(cuts, fpr, metric_name=metric_name_fpr,
                                                 metric_namespace=metric_namespace,
                                                 metric_type=metric.upper(), abscissa_name="",
                                                 curve_name=curve_name_fpr, pair_type=data_type, thresholds=thresholds)

                        metric_name_tpr = '_'.join([metric_name, "tpr"])
                        curve_name_tpr = "_".join([data_type, "tpr"])
                        self.__save_curve_metric(cuts, tpr, metric_name_tpr, metric_namespace, metric.upper(),
                                                 abscissa_name="",
                                                 curve_name=curve_name_tpr, pair_type=data_type, thresholds=thresholds)

                    elif metric == consts.ROC:
                        self.__save_roc(data_type, metric_name, metric_namespace, metric_res[1])

                    elif metric == consts.TOPN:
                        self.__save_topn(metric_name, metric_namespace, metric_res[1])

                    elif metric in [consts.ACCURACY, consts.LIFT, consts.GAIN]:
                        if self.eval_type == consts.MULTY and metric == consts.ACCURACY:
                            self.__save_single_value(metric_res[1], metric_name=data_type,
                                                     metric_namespace=metric_namespace,
                                                     eval_name=metric)
                            collect_dict[metric] = metric_res[1]
                            continue

                        score, cuts, thresholds = metric_res[1]

                        if metric in [consts.LIFT, consts.GAIN]:
                            score = [float(s[1]) for s in score]
                            cuts = [float(c[1]) for c in cuts]
                            cuts, score, idx_list = self.__filt_override_unit_ordinate_coordinate(cuts, score)
                            thresholds = [thresholds[idx] for idx in idx_list]

                            score.append(1.0)
                            cuts.append(1.0)
                            thresholds.append(0.0)

                        self.__save_curve_metric(cuts, score, metric_name=metric_name,
                                                 metric_namespace=metric_namespace,
                                                 metric_type=metric.upper(), abscissa_name="",
                                                 curve_name=data_type, thresholds=thresholds)

                    elif metric in [consts.PRECISION, consts.RECALL]:
                        precision_recall[metric] = metric_res
                        if len(precision_recall) < 2:
                            continue

                        precision_res = precision_recall.get(consts.PRECISION)
                        recall_res = precision_recall.get(consts.RECALL)

                        if precision_res[0] != recall_res[0]:
                            LOGGER.warning(
                                "precision mode:{} is not equal to recall mode:{}".format(precision_res[0],
                                                                                recall_res[0]))
                            continue

                        metric_namespace = precision_res[0]
                        metric_name_precision = '_'.join([data_type, "precision"])
                        metric_name_recall = '_'.join([data_type, "recall"])

                        pos_precision_score = precision_res[1][0]
                        precision_cuts = precision_res[1][1]
                        if len(precision_res[1]) >= 3:
                            precision_thresholds = precision_res[1][2]
                        else:
                            precision_thresholds = None

                        pos_recall_score = recall_res[1][0]
                        recall_cuts = recall_res[1][1]

                        if len(recall_res[1]) >= 3:
                            recall_thresholds = recall_res[1][2]
                        else:
                            recall_thresholds = None

                        precision_curve_name = data_type
                        recall_curve_name = data_type
                        if self.eval_type == consts.BINARY:
                            pos_precision_score = [score[1] for score in pos_precision_score]
                            pos_recall_score = [score[1] for score in pos_recall_score]

                            pos_recall_score, pos_precision_score, idx_list = self.__filt_override_unit_ordinate_coordinate(
                                pos_recall_score, pos_precision_score)

                            precision_cuts = [precision_cuts[idx] for idx in idx_list]
                            recall_cuts = [recall_cuts[idx] for idx in idx_list]

                            edge_idx = idx_list[-1]
                            if edge_idx == len(precision_thresholds) - 1:
                                idx_list = idx_list[:-1]
                            precision_thresholds = [precision_thresholds[idx] for idx in idx_list]
                            recall_thresholds = [recall_thresholds[idx] for idx in idx_list]

                        elif self.eval_type == consts.MULTY:
                            average_precision = float(np.array(pos_precision_score).mean())
                            average_recall = float(np.array(pos_recall_score).mean())
                            self.__save_single_value(average_precision, metric_name=data_type,
                                                     metric_namespace=metric_namespace,
                                                     eval_name="precision")
                            self.__save_single_value(average_recall, metric_name=data_type,
                                                     metric_namespace=metric_namespace,
                                                     eval_name="recall")
                            collect_dict[consts.PRECISION] = average_precision
                            collect_dict[consts.RECALL] = average_recall

                            precision_curve_name = metric_name_precision
                            recall_curve_name = metric_name_recall

                        self.__save_curve_metric(precision_cuts, pos_precision_score, metric_name_precision,
                                                 metric_namespace,
                                                 "_".join([consts.PRECISION.upper(), self.eval_type.upper()]),
                                                 abscissa_name="", ordinate_name="Precision",
                                                 curve_name=precision_curve_name,
                                                 pair_type=data_type, thresholds=precision_thresholds)

                        self.__save_curve_metric(recall_cuts, pos_recall_score, metric_name_recall, metric_namespace,
                                                 "_".join([consts.RECALL.upper(), self.eval_type.upper()]),
                                                 abscissa_name="", ordinate_name="Recall", curve_name=recall_curve_name,
                                                 pair_type=data_type, thresholds=recall_thresholds)
                    else:
                        LOGGER.warning("Unknown metric:{}".format(metric))

        if return_single_val_metrics:
            if len(self.validate_metric) != 0:
                LOGGER.debug("return validate metric")
                LOGGER.debug('validate metric is {}'.format(self.validate_metric))
                return self.validate_metric
            else:
                LOGGER.debug("validate metric is empty, return train metric")
                LOGGER.debug('train metric is {}'.format(self.train_metric))
                return self.train_metric

    def __filt_threshold(self, thresholds, step):
        cuts = list(map(float, np.arange(0, 1, step)))
        size = len(list(thresholds))
        thresholds.sort(reverse=True)
        index_list = [int(size * cut) for cut in cuts]
        new_thresholds = [thresholds[idx] for idx in index_list]

        return new_thresholds, cuts

    def topn(self, labels, pred_scores):
        '''
        columns=['TOP N', 'CutOff', 'real', 'num', 'accuracy', 'recall rate']
        '''
        if self.eval_type != consts.BINARY:
            logging.warning("topn is just suppose Binary Classification! return None as results")
            return None

        index_map = {
            '95%': 'TOP 5%',
            '90%': 'TOP 10%',
            '85%': 'TOP 15%',
            '80%': 'TOP 20%',
            '70%': 'TOP 30%',
            '50%': 'TOP 50%'
        }
        df = pd.DataFrame()
        df['label'] = labels
        df['score'] = pred_scores
        bad_num = df[df.label == 1].shape[0]
        desc = df.score.describe(percentiles=[0.7, 0.8, 0.85, 0.9, 0.95])
        desc = desc[4:10]
        res = []
        for index in desc.index:
            dd = df[df.score >= desc[index]]
            num = dd.shape[0]
            real = dd[dd.label == 1].shape[0]
            if num <= 0:
                continue
            res.append((
                index_map[index],
                '%.4f' % desc[index],
                # desc[index],
                real,
                num,
                '%.2f%%' % (real * 100 / num),
                '%.2f%%' % (real * 100 / bad_num)))
        res.reverse()
        return res

    def auc(self, labels, pred_scores):
        """
        Compute AUC for binary classification.

        Parameters
        ----------
        labels: value list. The labels of data set.
        pred_scores: value list. The predict results of model. It should be corresponding to labels each data.

        Returns
        ----------
        float
            The AUC
        """
        if self.eval_type == consts.BINARY:
            return roc_auc_score(labels, pred_scores)
        else:
            logging.warning("auc is just suppose Binary Classification! return None as results")
            return None

    def explained_variance(self, labels, pred_scores):
        """
        Compute explain variance
        Parameters
        ----------
        labels: value list. The labels of data set.
        pred_scores: value list. The predict results of model. It should be corresponding to labels each data.

        Returns
        ----------
        float
            The explain variance
        """
        return explained_variance_score(labels, pred_scores)

    def mean_absolute_error(self, labels, pred_scores):
        """
        Compute mean absolute error
        Parameters
        ----------
        labels: value list. The labels of data set.
        pred_scores: value list. The predict results of model. It should be corresponding to labels each data.
        Returns
        ----------
        float
            A non-negative floating point.
        """
        return mean_absolute_error(labels, pred_scores)

    def mean_squared_error(self, labels, pred_scores):
        """
        Compute mean square error
        Parameters
        ----------
        labels: value list. The labels of data set.
        pred_scores: value list. The predict results of model. It should be corresponding to labels each data.
        Returns
        ----------
        float
            A non-negative floating point value
        """
        return mean_squared_error(labels, pred_scores)

    def mean_squared_log_error(self, labels, pred_scores):
        """
        Compute mean squared logarithmic error
        Parameters
        ----------
        labels: value list. The labels of data set.
        pred_scores: value list. The predict results of model. It should be corresponding to labels each data.
        Returns
        ----------
        float
            A non-negative floating point value
        """
        return mean_squared_log_error(labels, pred_scores)

    def median_absolute_error(self, labels, pred_scores):
        """
        Compute median absolute error
        Parameters
        ----------
        labels: value list. The labels of data set.
        pred_scores: value list. The predict results of model. It should be corresponding to labels each data.
        Returns
        ----------
        float
            A positive floating point value
        """
        return median_absolute_error(labels, pred_scores)

    def r2_score(self, labels, pred_scores):
        """
        Compute R^2 (coefficient of determination) score
        Parameters
        ----------
        labels: value list. The labels of data set.
        pred_scores: value list. The predict results of model. It should be corresponding to labels each data.
        Returns
        ----------
        float
            The R^2 score
        """
        return r2_score(labels, pred_scores)

    def root_mean_squared_error(self, labels, pred_scores):
        """
        Compute the root of mean square error
        Parameters
        ----------
        labels: value list. The labels of data set.
        pred_scores: value list. The predict results of model. It should be corresponding to labels each data.
        Return
        ----------
        float
            A positive floating point value
        """
        return np.sqrt(mean_squared_error(labels, pred_scores))

    def roc(self, labels, pred_scores):
        if self.eval_type == consts.BINARY:
            fpr, tpr, thresholds = roc_curve(np.array(labels), np.array(pred_scores), drop_intermediate=1)
            fpr, tpr, thresholds = list(map(float, fpr)), list(map(float, tpr)), list(map(float, thresholds))

            filt_thresholds, cuts = self.__filt_threshold(thresholds=thresholds, step=0.01)
            new_thresholds = []
            new_tpr = []
            new_fpr = []
            for threshold in filt_thresholds:
                index = thresholds.index(threshold)
                new_tpr.append(tpr[index])
                new_fpr.append(fpr[index])
                new_thresholds.append(threshold)

            fpr = new_fpr
            tpr = new_tpr
            thresholds = new_thresholds
            return fpr, tpr, thresholds, cuts
        else:
            logging.warning("roc_curve is just suppose Binary Classification! return None as results")
            fpr, tpr, thresholds, cuts = None, None, None, None

            return fpr, tpr, thresholds, cuts

    def ks(self, labels, pred_scores):
        """
        Compute Kolmogorov-Smirnov
        Parameters
        ----------
        labels: value list. The labels of data set.
        pred_scores: pred_scores: value list. The predict results of model. It should be corresponding to labels each data.
        Returns
        ----------
        max_ks_interval: float max value of each tpr - fpt
        fpr:
        """
        score_label_list = []
        for i, label in enumerate(labels):
            score_label_list.append((pred_scores[i], label))

        score_label_list.sort(key=lambda x: x[0], reverse=True)
        cuts = [c / 100 for c in range(100)]
        data_size = len(pred_scores)
        indexs = [int(data_size * cut) for cut in cuts]
        score_threshold = [score_label_list[idx][0] for idx in indexs]

        fpr = []
        tpr = []
        ks = []
        for i, index in enumerate(indexs):
            positive = 0
            positive_recall = 0
            negative = 0
            false_positive = 0
            for score_label in score_label_list:
                pre_score = score_label[0]
                label = score_label[1]
                if label == self.pos_label:
                    positive += 1

                    if pre_score > score_threshold[i]:
                        positive_recall += 1

                if label == 0:
                    negative += 1
                    if pre_score > score_threshold[i]:
                        false_positive += 1

            if positive == 0 or negative == 0:
                raise ValueError("all labels are positive or negative, please check your data!")

            _tpr = positive_recall / positive
            _fpr = false_positive / negative
            _ks = _tpr - _fpr
            tpr.append(_tpr)
            fpr.append(_fpr)
            ks.append(_ks)

        fpr.append(1.0)
        tpr.append(1.0)
        cuts.append(1.0)

        return max(ks), fpr, tpr, score_threshold, cuts

    def lift(self, labels, pred_scores):
        """
        Compute lift of binary classification.
        Parameters
        ----------
        labels: value list. The labels of data set.
        pred_scores: pred_scores: value list. The predict results of model. It should be corresponding to labels each data.
        Returns
        ----------
        float
            The lift
        """
        if self.eval_type == consts.BINARY:
            thresholds = list(set(pred_scores))
            thresholds, cuts = self.__filt_threshold(thresholds, 0.01)
            lift_operator = Lift()
            lift_y, lift_x = lift_operator.compute(labels, pred_scores, thresholds=thresholds)
            return lift_y, lift_x, thresholds
        else:
            logging.warning("lift is just suppose Binary Classification! return None as results")
            return None

    def gain(self, labels, pred_scores):
        """
        Compute gain of binary classification.
        Parameters
        ----------
        labels: value list. The labels of data set.
        pred_scores: pred_scores: value list. The predict results of model. It should be corresponding to labels each data.
        Returns
        ----------
        float
            The gain
        """

        if self.eval_type == consts.BINARY:
            thresholds = list(set(pred_scores))
            thresholds, cuts = self.__filt_threshold(thresholds, 0.01)
            gain_operator = Gain()
            gain_x, gain_y = gain_operator.compute(labels, pred_scores, thresholds=thresholds)
            return gain_y, gain_x, thresholds
        else:
            logging.warning("gain is just suppose Binary Classification! return None as results")
            return None

    def precision(self, labels, pred_scores):
        """
        Compute the precision
        Parameters
        ----------
        labels: value list. The labels of data set.
        pred_scores: pred_scores: value list. The predict results of model. It should be corresponding to labels each data.
        Returns
        ----------
        dict
            The key is threshold and the value is another dic, which key is label in parameter labels, and value is the label's precision.
        """
        if self.eval_type == consts.BINARY:
            thresholds = list(set(pred_scores))
            thresholds, cuts = self.__filt_threshold(thresholds, 0.01)

            # set for recall edge value
            thresholds.append(min(thresholds) - 0.001)
            cuts.append(1)

            precision_operator = BiClassPrecision()
            precision_res, thresholds = precision_operator.compute(labels, pred_scores, thresholds)
            return precision_res, cuts, thresholds
        elif self.eval_type == consts.MULTY:
            precision_operator = MultiClassPrecision()
            return precision_operator.compute(labels, pred_scores)
        else:
            logging.warning("error:can not find classification type:{}".format(self.eval_type))

    def recall(self, labels, pred_scores):
        """
        Compute the recall
        Parameters
        ----------
        labels: value list. The labels of data set.
        pred_scores: pred_scores: value list. The predict results of model. It should be corresponding to labels each data.
        Returns
        ----------
        dict
            The key is threshold and the value is another dic, which key is label in parameter labels, and value is the label's recall.
        """
        if self.eval_type == consts.BINARY:
            thresholds = list(set(pred_scores))
            thresholds, cuts = self.__filt_threshold(thresholds, 0.01)

            # set for recall edge value
            thresholds.append(min(thresholds) - 0.001)
            cuts.append(1)

            recall_operator = BiClassRecall()
            recall_res, thresholds = recall_operator.compute(labels, pred_scores, thresholds)
            return recall_res, cuts, thresholds
        elif self.eval_type == consts.MULTY:
            recall_operator = MultiClassRecall()
            return recall_operator.compute(labels, pred_scores)
        else:
            logging.warning("error:can not find classification type:{}".format(self.eval_type))

    def accuracy(self, labels, pred_scores, normalize=True):
        """
        Compute the accuracy
        Parameters
        ----------
        labels: value list. The labels of data set.
        pred_scores: pred_scores: value list. The predict results of model. It should be corresponding to labels each data.
        normalize: bool. If true, return the fraction of correctly classified samples, else returns the number of correctly classified samples
        Returns
        ----------
        dict
            the key is threshold and the value is the accuracy of this threshold.
        """

        if self.eval_type == consts.BINARY:
            thresholds = list(set(pred_scores))
            thresholds, cuts = self.__filt_threshold(thresholds, 0.01)
            acc_operator = BiClassAccuracy()
            acc_res, thresholds = acc_operator.compute(labels, pred_scores, thresholds, normalize)
            return acc_res, cuts, thresholds
        elif self.eval_type == consts.MULTY:
            acc_operator = MultiClassAccuracy()
            return acc_operator.compute(labels, pred_scores, normalize)
        else:
            logging.warning("error:can not find classification type:".format(self.eval_type))

    @staticmethod
    def extract_data(data: dict):
        return data

class Lift(object):
    """
    Compute lift
    """

    def __predict_value_to_one_hot(self, pred_value, threshold):
        one_hot = []
        for value in pred_value:
            if value > threshold:
                one_hot.append(1)
            else:
                one_hot.append(0)

        return one_hot

    def __compute_lift(self, labels, pred_scores_one_hot, pos_label="1"):
        tn, fp, fn, tp = confusion_matrix(labels, pred_scores_one_hot).ravel()

        if pos_label == '0':
            tp, tn = tn, tp
            fp, fn = fn, fp

        labels_num = len(labels)
        if labels_num == 0:
            lift_x = 1
            denominator = 1
        else:
            lift_x = (tp + fp) / labels_num
            denominator = (tp + fn) / labels_num

        if tp + fp == 0:
            numerator = 1
        else:
            numerator = tp / (tp + fp)

        if denominator == 0:
            lift_y = sys.float_info.max
        else:
            lift_y = numerator / denominator

        return lift_x, lift_y

    def compute(self, labels, pred_scores, thresholds=None):
        lifts_x = []
        lifts_y = []

        for threshold in thresholds:
            pred_scores_one_hot = self.__predict_value_to_one_hot(pred_scores, threshold)
            label_type = ['0', '1']
            lift_x_type = []
            lift_y_type = []
            for lt in label_type:
                lift_x, lift_y = self.__compute_lift(labels, pred_scores_one_hot, pos_label=lt)
                lift_x_type.append(lift_x)
                lift_y_type.append(lift_y)
            lifts_x.append(lift_x_type)
            lifts_y.append(lift_y_type)

        return lifts_y, lifts_x

class Gain(object):
    """
    Compute Gain
    """

    def __init__(self):
        pass

    def __predict_value_to_one_hot(self, pred_value, threshold):
        one_hot = []
        for value in pred_value:
            if value > threshold:
                one_hot.append(1)
            else:
                one_hot.append(0)

        return one_hot

    def __compute_gain(self, label, pred_scores_one_hot, pos_label="1"):
        tn, fp, fn, tp = confusion_matrix(label, pred_scores_one_hot).ravel()

        if pos_label == '0':
            tp, tn = tn, tp
            fp, fn = fn, fp

        num_label = len(label)
        if num_label == 0:
            gain_x = 1
        else:
            gain_x = float((tp + fp) / num_label)

        num_positives = tp + fn
        if num_positives == 0:
            gain_y = 1
        else:
            gain_y = float(tp / num_positives)

        return gain_x, gain_y

    def compute(self, labels, pred_scores, thresholds=None):
        gains_x = []
        gains_y = []

        for threshold in thresholds:
            pred_scores_one_hot = self.__predict_value_to_one_hot(pred_scores, threshold)
            label_type = ['0', '1']
            gain_x_type = []
            gain_y_type = []
            for lt in label_type:
                gain_x, gain_y = self.__compute_gain(labels, pred_scores_one_hot, pos_label=lt)
                gain_x_type.append(gain_x)
                gain_y_type.append(gain_y)
            gains_x.append(gain_x_type)
            gains_y.append(gain_y_type)

        return gains_x, gains_y

class BiClassPrecision(object):
    """
    Compute binary classification precision
    """

    def __init__(self):
        self.total_positives = 0

    def __predict_value_to_one_hot(self, pred_value, threshold):
        one_hot = []
        self.total_positives = 0
        for value in pred_value:
            if value > threshold:
                one_hot.append(1)
                self.total_positives += 1
            else:
                one_hot.append(0)

        return one_hot

    def compute(self, labels, pred_scores, thresholds):
        scores = []
        for threshold in thresholds:
            pred_scores_one_hot = self.__predict_value_to_one_hot(pred_scores, threshold)
            score = list(map(float, precision_score(labels, pred_scores_one_hot, average=None)))
            if self.total_positives == 0:
                score[1] = 1.0
            scores.append(score)

        return scores, thresholds

class MultiClassPrecision(object):
    """
    Compute multi-classification precision
    """

    def compute(self, labels, pred_scores):
        all_labels = list(set(labels).union(set(pred_scores)))
        all_labels.sort()
        return precision_score(labels, pred_scores, average=None), all_labels

class BiClassRecall(object):
    """
    Compute binary classification recall
    """

    def __predict_value_to_one_hot(self, pred_value, threshold):
        one_hot = []
        for value in pred_value:
            if value > threshold:
                one_hot.append(1)
            else:
                one_hot.append(0)

        return one_hot

    def compute(self, labels, pred_scores, thresholds):
        scores = []

        for threshold in thresholds:
            pred_scores_one_hot = self.__predict_value_to_one_hot(pred_scores, threshold)
            score = list(map(float, recall_score(labels, pred_scores_one_hot, average=None)))
            scores.append(score)

        return scores, thresholds

class MultiClassRecall(object):
    """
    Compute multi-classification recall
    """

    def compute(self, labels, pred_scores):
        all_labels = list(set(labels).union(set(pred_scores)))
        all_labels.sort()
        return recall_score(labels, pred_scores, average=None), all_labels

class BiClassAccuracy(object):
    """
    Compute binary classification accuracy
    """

    def __predict_value_to_one_hot(self, pred_value, threshold):
        one_hot = []
        for value in pred_value:
            if value > threshold:
                one_hot.append(1)
            else:
                one_hot.append(0)

        return one_hot

    def compute(self, labels, pred_scores, thresholds, normalize=True):
        scores = []
        for threshold in thresholds:
            pred_scores_one_hot = self.__predict_value_to_one_hot(pred_scores, threshold)
            score = accuracy_score(labels, pred_scores_one_hot, normalize)
            scores.append(score)

        return scores, thresholds

class MultiClassAccuracy(object):
    """
    Compute multi-classification accuracy
    """

    def compute(self, labels, pred_scores, normalize=True):
        return accuracy_score(labels, pred_scores, normalize)

class IC(object):
    """
    Compute Information Criterion with a given dTable and loss
        When k = 2, result is genuine AIC;
        when k = log(n), results is BIC, also called SBC, SIC, SBIC.
    """

    def compute(self, k, n, dfe, loss):
        aic_score = k * dfe + 2 * n * loss
        return aic_score

class IC_Approx(object):
    """
    Compute Information Criterion value with a given dTable and loss
        When k = 2, result is genuine AIC;
        when k = log(n), results is BIC, also called SBC, SIC, SBIC.
        Note that this formula for linear regression dismisses the constant term n * np.log(2 * np.pi) for sake of simplicity, so the absolute value of result will be small.
    """

    def compute(self, k, n, dfe, loss):
        aic_score = k * dfe + n * np.log(loss * 2)
        return aic_score


import os
import time
import torch

import torchvision.transforms as transforms

from utils import get_device
from data import get_dataset
from losses import ZeroReferenceLoss
from model import ZeroDCE
from config_file import ConfigFile

# ---------------------------------------- DATA --------------------------------------- #

DATA_DIR = 'data\\lol_dataset'
TRAIN_DIR_NAME = 'our485'
VAL_DIR_NAME = 'eval15'

RESIZE_SIZE = (512, 512)
IMAGES_TRANSFORM = transforms.Compose([
    transforms.ToTensor(),
    transforms.Resize(RESIZE_SIZE, antialias=True),
])

CONFIG_FILE_PATH = 'config.yaml'

# ---------------------------------------- TRAIN --------------------------------------- #
def evaluate_data(data, model, loss):
    """ Evaluate the model on a given dataset"""
    model.eval()
    eval_loss = 0.0
    with torch.no_grad():
        print(f'[INFO] VALIDATION START')

        # loop over the validation set
        for batch_num, eval_low_light_images in enumerate(data):
            enhanced, alpha_maps = model(eval_low_light_images)

            loss, losses_dict = loss(enhanced_images=enhanced,
                                     orig_images=eval_low_light_images,
                                     alpha_maps=alpha_maps)
            eval_loss += loss.item()

    # Calculate average loss over an epoch
    eval_loss = round(eval_loss / len(data), 5)
    return eval_loss

def train_epoch(epoch_num, train_data, model, optimizer, loss):
    """ Train the model for one epoch"""
    model.train()
    train_loss = 0.0
    for batch_num, train_low_light in enumerate(train_data):
        print(f'[INFO] EPOCH NUM: {epoch_num + 1}, BATCH NUM: {batch_num + 1}/{len(train_data)}')
        enhanced, alpha_maps = model(train_low_light)

        loss, losses_dict = loss(enhanced_images=enhanced,
                                 orig_images=train_low_light,
                                 alpha_maps=alpha_maps)
        train_loss += loss.item()

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    # Calculate average loss over an epoch
    train_loss = train_loss / len(train_data)
    return train_loss

# ---------------------------------------- MAIN --------------------------------------- #

def train():
    device = get_device()

    # output directory
    output_dir_path = os.path.join('outputs', time.strftime("%Y%m%d-%H%M%S"))
    os.makedirs(output_dir_path)

    # config
    config = ConfigFile.load(config_path=CONFIG_FILE_PATH)
    config_path = os.path.join(output_dir_path, 'config.yaml')
    config.save_config(config_path)

    # model
    model = ZeroDCE(config=config, device=device)

    # Datasets
    batch_size = config.train.batch_size
    train_data = get_dataset(
        dir_path=os.path.join(DATA_DIR, TRAIN_DIR_NAME),
        transform_img=IMAGES_TRANSFORM, batch_size=batch_size, device=device
    )
    eval_data = get_dataset(
        dir_path=os.path.join(DATA_DIR, VAL_DIR_NAME),
        transform_img=IMAGES_TRANSFORM, batch_size=batch_size, device=device
    )
    # loss function and optimizer
    loss_fn = ZeroReferenceLoss()
    optimizer = torch.optim.Adam(model.parameters(),
                                 lr=config.train.learning_rate,
                                 weight_decay=config.train.weight_decay)

    # Train Loop
    min_valid_loss = float('inf')
    for epoch_num in range(config.train.num_epochs):

        # Train the model for one epoch and calculate the train loss (average over epoch).
        train_loss = train_epoch(epoch_num, train_data, model, optimizer, loss_fn)
        print(f'[INFO] EPOCH NUM: {epoch_num + 1}, TRAIN LOSS: {train_loss:.5f}')

        # Evaluate the model on the validation set and calculate the validation loss.
        eval_loss = evaluate_data(eval_data, model, loss_fn)
        print(f'[INFO] EPOCH NUM: {epoch_num + 1}, VALIDATION LOSS: {eval_loss}')

        if min_valid_loss > eval_loss:
            print(f'[INFO] Validation Loss Decreased({min_valid_loss}--->{eval_loss}) '
                  f'\t Saving The Model ({output_dir_path})')
            min_valid_loss = eval_loss

            # Saving State Dict
            model_output_path = os.path.join(output_dir_path, f'model_{eval_loss}.pth')
            torch.save(model.state_dict(), model_output_path)
    print(f'[INFO] Training finished. Output directory: {output_dir_path}')

if __name__ == '__main__':
    train()


import setuptools
#
with open("README.md", "r") as fh:
    long_description = fh.read();
#
setuptools.setup(
    name="pySW",
    packages = ['pySW'],
    version="1.0",
    author="Kalyan Inamdar",
    author_email="kalyaninamdar@protonmail.com",
    url = 'https://github.com/kalyanpi4/pySW',
    download_url='https://github.com/kalyanpi4/pySW/archive/V1.0.tar.gz',
    description="A Python wrapper around Solidworks built-in VBA API for modifying Solidworks assemblies and parts.",
    install_requires=[
          'numpy',
          'pandas',
          'pywin32',
    ],
    classifiers=[
		'License :: OSI Approved :: LGPL-2.0',
        'Programming Language :: Python :: 3',
    ],
    python_requires='>=3.6',
)


import scrapy
from ..CourseItems import CourseItem
#from scrapy.crawler import CrawlerProcess

class DmozSpider(scrapy.Spider):

    name = "dmoz"
    allowed_domains = ["imooc.com"]
    start_urls = [
        "http://www.imooc.com/course/list"
    ]

    def parse(self, response):
        #titles = response.xpath("//input[@type='submit']").extract()

        item = CourseItem()
        for box in response.xpath('//div[@class="course-card-container"]'):
            # 获取每个div中的课程路径
            item['url'] = 'http://www.imooc.com' +  box.xpath('.//@href').extract()[0]
            # 获取div中的课程标题
            item['title'] = box.xpath('.//h3/text()').extract()[0]
            # 获取div中的标题图片url
            item['image_url'] = box.xpath('.//@src').extract()[0]
            # 获取div中的标题图片标签
            item['image_lable'] = box.xpath('.//label/text()').extract()
            # 获取div中的学生人数
            item['student'] = box.xpath(
                './/div[@class="course-card-info"]/span[2]/text()').extract()[0].strip()
            # 获取div中的课程简介
            item['introduction'] = box.xpath('.//p/text()').extract()[0].strip()
            # 返回信息

            yield item
       #url = response.xpath("//a[contains(text(),'下一页')]/@href").extract()
       #if url :
       #    #将信息组合成下一页的url
       #    page = 'http://www.imooc.com' + url[0]
       #    #返回url
       #    yield scrapy.Request(page, callback=self.parse)

#process = CrawlerProcess({
#    'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
#})
#
#process.crawl(DmozSpider)
#process.start()


import json
import pandas as pd
from statsmodels.formula.api import ols
import numpy as np
import matplotlib
matplotlib.use('pdf')
import matplotlib.pyplot as plt

'''
ts = pd.Series(np.random.randn(1000), index=pd.date_range('1/1/2000', periods=1000))
ts = ts.cumsum()
ts.plot()
plt.savefig('test.pdf')
'''
finaldata = {
    'capacity_multiplier': [],
    'number_of_iterations': [],
};
with open('./export.json', 'r') as f:
    data = json.load(f)
for run in data:
    finaldata["capacity_multiplier"].append(data[run]["label"]);
    finaldata["number_of_iterations"].append(data[run]["avgItrs"]);
df = pd.DataFrame(data=finaldata)
df = df.sort_values(by='capacity_multiplier', ascending=True)
test = ols("capacity_multiplier ~ number_of_iterations", data=df).fit()
df.plot.bar(x='capacity_multiplier', y='number_of_iterations', rot=0)
plt.savefig('test.pdf')
print(test.params)
print(test.summary())


from rest_framework import serializers
from models import Country, City, Airport, Artist, Show

class CountrySerializer(serializers.ModelSerializer):
    class Meta:
        model = Country
        fields = ('id', 'iata', 'name', 'cities')

class CitySerializer(serializers.ModelSerializer):
    class Meta:
        model = City
        fields = ('id', 'iata', 'name', 'country')

class AirportSerializer(serializers.ModelSerializer):
    class Meta:
        model = Airport
        fields = ('id', 'iata', 'name', 'city')

class ArtistSerializer(serializers.ModelSerializer):
    genres = serializers.SerializerMethodField()

    def get_genres(self, obj):
        return [genre.name for genre in obj.genres.all()]

    class Meta:
        model = Artist
        fields = ('id', 'spotify_id', 'name', 'popularity', 'genres', 'image')

class ShowSerializer(serializers.ModelSerializer):
    city = CitySerializer(many=False, read_only=True)
    artists = ArtistSerializer(many=True, read_only=True)
    name = serializers.SerializerMethodField()

    def get_name(self, obj):
        return unicode(obj)

    class Meta:
        model = Show
        fields = ('id', 'name', 'description', 'city', 'lat', 'lon', 'datetime', 'artists', 'address', 'price', 'image')


from collections import defaultdict

###http://surprise.readthedocs.io/en/stable/FAQ.html#how-to-get-the-top-n-recommendations-for-each-user
def get_top_n(predictions, n=10):
    '''Return the top-N recommendation for each user from a set of predictions.

    Args:
        predictions(list of Prediction objects): The list of predictions, as
            returned by the test method of an algorithm.
        n(int): The number of recommendation to output for each user. Default
            is 10.

    Returns:
    A dict where keys are user (raw) ids and values are lists of tuples:
        [(raw item id, rating estimation), ...] of size n.
    '''

    # First map the predictions to each user.
    top_n = defaultdict(list)
    for uid, iid, true_r, est, _ in predictions:
        top_n[uid].append((iid, est))

    # Then sort the predictions for each user and retrieve the k highest ones.
    for uid, user_ratings in top_n.items():
        user_ratings.sort(key=lambda x: x[1], reverse=True)
        top_n[uid] = user_ratings[:n]

    return top_n

def get_items_greater_threshold(predictions, threshold=0):
    '''Return the recommendation for each user from a set of predictions.

    Args:
        predictions(list of Prediction objects): The list of predictions, as
            returned by the test method of an algorithm.
        threshold: recommend the item if est > threshold

    Returns:
    A dict where keys are user (raw) ids and values are lists of tuples:
        [(raw item id, rating estimation), ...] of size n.
    '''

    # First map the predictions to each user.
    recommendationlist = defaultdict(list)

    for uid, iid, true_r, est, _ in predictions:
        if est >= threshold:
            recommendationlist[uid].append((iid, est))
        else:
            # print('est < threshold')
            pass
    return recommendationlist

# def precision_recall_at_k(predictions, k=10, threshold=3.5):
#     '''Return precision and recall at k metrics for each user.'''

#     # First map the predictions to each user.
#     user_est_true = defaultdict(list)
#     for uid, _, true_r, est, _ in predictions:
#         user_est_true[uid].append((est, true_r))

#     precisions = dict()
#     recalls = dict()
#     for uid, user_ratings in user_est_true.items():

#         # Sort user ratings by estimated value
#         user_ratings.sort(key=lambda x: x[0], reverse=True)

#         # Number of relevant items
#         n_rel = sum((true_r >= threshold) for (_, true_r) in user_ratings)

#         # Number of recommended items in top k
#         n_rec_k = sum((est >= threshold) for (est, _) in user_ratings[:k])

#         # Number of relevant and recommended items in top k
#         n_rel_and_rec_k = sum(((true_r >= threshold) and (est >= threshold))
#                               for (est, true_r) in user_ratings[:k])

#         # Precision@K: Proportion of recommended items that are relevant
#         precisions[uid] = n_rel_and_rec_k / n_rec_k if n_rec_k != 0 else 1

#         # Recall@K: Proportion of relevant items that are recommended
#         recalls[uid] = n_rel_and_rec_k / n_rel if n_rel != 0 else 1

#     return precisions, recalls

def prediction_for_userids(algo, trainset, testdic, userids):
    user_est_true = defaultdict(list)
    for u in userids:
        u_numid = trainset.to_inner_uid(u)
        user_items = set([j for (j, _) in trainset.ur[u_numid]])
        user_est_true[u] = []
        for i in trainset.all_items():
            if i not in user_items :
                if trainset.to_raw_iid(i) in testdic[u]:
                    user_est_true[u].append((algo.estimate(u_numid,i)-algo.trainset.offset,1))
                else:
                    user_est_true[u].append((algo.estimate(u_numid,i)-algo.trainset.offset,0))
    return user_est_true

def precision_recall_at_k(user_est_true, testdict, k=10, threshold=0):
    '''Return precision and recall at k metrics'''
    precisions = dict()
    recalls = dict()

    for uid, user_ratings in user_est_true.items():

        user_ratings_greater_threshold = [(est, true) for (est,true) in user_ratings if est > threshold]

        n_rel = len(testdict[uid])

        n_greater_threshold = len(user_ratings_greater_threshold)

        # Number of recommended items in top k
        if n_greater_threshold > k:
            n_rec_k = k
        else:
            n_rec_k = n_greater_threshold

        # Number of relevant and recommended items in top k
        if n_greater_threshold > k:
            user_ratings_greater_threshold.sort(key=lambda x: x[0], reverse=True)
            n_rel_and_rec_k = sum((true_r for (est, true_r) in user_ratings_greater_threshold[:k]))
        else:
            try:
                n_rel_and_rec_k = sum((true_r for (est, true_r) in user_ratings_greater_threshold))
            except IndexError:
                n_rel_and_rec_k = 0

        # Precision@K: Proportion of recommended items that are relevant
        precisions[uid] = n_rel_and_rec_k / n_rec_k if n_rec_k != 0 else 1

        # Recall@K: Proportion of relevant items that are recommended
        recalls[uid] = n_rel_and_rec_k / n_rel if n_rel != 0 else 1

    return precisions, recalls


import os
import shutil
import zstandard

import time

def dump_zstd() -> float:
    # Emptying the folder first
    try:
        shutil.rmtree("zstd")
    except FileNotFoundError:
        pass

    compressor = zstandard.ZstdCompressor(level=3)

    os.makedirs("zstd")

    start = time.time()

    for file in os.listdir("raw"):
        with open(f"raw/{file}", "r") as match_file:
            data = bytearray(match_file.read(), "utf-8")

        with open(f"zstd/{file}.zst", "wb+") as file:
            file.write(compressor.compress(data=data))

    end = time.time()

    return end - start

def dump_zstd_dict() -> float:
    # Emptying the folder first
    try:
        shutil.rmtree("zstd_dict")
    except FileNotFoundError:
        pass

    os.makedirs("zstd_dict")

    # # We take some time to create the dictionary first
    # samples = []

    # for file in os.listdir("raw"):
    #     with open(f"raw/{file}", "r") as match_file:
    #         samples.append(bytearray(match_file.read(), "utf-8"))

    # compression_dict = zstandard.train_dictionary(1000 * 1000, samples=samples)

    dict_data = None

    start = time.time()

    for file in os.listdir("raw"):
        with open(f"raw/{file}", "r") as match_file:
            data = bytearray(match_file.read(), "utf-8")

        if not dict_data:
            dict_data = zstandard.ZstdCompressionDict(data)
            compressor = zstandard.ZstdCompressor(level=3, dict_data=dict_data)

        with open(f"zstd_dict/{file}.zst", "wb+") as file:
            file.write(
                compressor.compress(
                    data=data,
                )
            )

    end = time.time()

    return end - start


import pandas as pd               # Para la manipulación y análisis de datos
import numpy as np                # Para crear vectores y matrices n dimensionales
import matplotlib
import matplotlib.pyplot as plt   # Para la generación de gráficas a partir de los datos
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from matplotlib.figure import Figure
import seaborn as sns             # Para la visualización de datos basado en matplotlib
from sklearn.preprocessing import StandardScaler, MinMaxScaler
import scipy.cluster.hierarchy as shc
from sklearn.cluster import AgglomerativeClustering
import secrets
import hashlib
import json
from sklearn.cluster import KMeans
from sklearn.metrics import pairwise_distances_argmin_min
from kneed import KneeLocator

class Clustering():
    @staticmethod
    def generar_hash_aleatorio():
        # Genera una cadena aleatoria
        cadena_aleatoria = secrets.token_hex(16)  # Genera una cadena hexadecimal de 16 bytes (32 caracteres)

        # Calcula el hash SHA256 de la cadena aleatoria
        sha256_hash = hashlib.sha256(cadena_aleatoria.encode()).hexdigest()
        return sha256_hash

    @staticmethod
    def generaMatrizCorrlaciones(file=None, dataset='datos_prueba/clustering/Hipoteca.csv'):
        if file:
            data = pd.read_csv(file)
        else:
            data = pd.read_csv(dataset)

        # eliminando columnas no numericas
        columnas_no_numericas = data.select_dtypes(exclude='number').columns
        data = data.drop(columnas_no_numericas, axis=1)
        columnas = data.columns.values.tolist()

        Correlaciones = data.corr(method='pearson')

        MatrizInf = np.triu(Correlaciones)
        heatmap = sns.heatmap(Correlaciones, cmap='RdBu_r', annot=True, mask=MatrizInf)

        # Ajusta el tamaño de la figura del mapa de calor
        fig = heatmap.get_figure()
        fig.set_size_inches(10, 8)

        # Guarda el mapa de calor en un archivo temporal
        temp_file = f'{Clustering.generar_hash_aleatorio()}.png'
        fig.savefig(f'image_tmp/{temp_file}')

        # Cierra la figura para liberar recursos
        plt.close(fig)

        return {
            "sucess": True,
            "data": {
                "image": f"http://localhost:8000/api/clustering/image/{temp_file}",
                'columnas': columnas
            }
        }

    @staticmethod
    def obtenerArbol(metrica, estandarizacion, columnas, file=None, dataset="datos_prueba/apriori/movies.csv"):
        if file:
            #print('recibiendo archivo')
            data = pd.read_csv(file)
        else:
            #print('default')
            data = pd.read_csv(dataset)

        data = np.array(data[columnas])

        print(columnas)

        if estandarizacion == 'StandardScaler':
            estandarizar = StandardScaler()
        else:
            estandarizar = MinMaxScaler()

        MEstandarizada = estandarizar.fit_transform(data)

        # Crear una figura

        plt.figure(figsize=(8, 7))

        plt.title("Dendograma")
        Arbol = shc.dendrogram(shc.linkage(MEstandarizada, method='complete', metric=metrica))
        temp_file = f'{Clustering.generar_hash_aleatorio()}.png'
        # Guardar la figura en un archivo de imagen
        plt.savefig(f'image_tmp/{temp_file}')
        plt.close()

        return {
            "sucess": True,
            "data": {
                "image": f"http://localhost:8000/api/clustering/image/{temp_file}"
            }
        }

    def obteniendoClusters(numero_clusters, metrica, estandarizacion, columnas, file=None, dataset="datos_prueba/apriori/movies.csv"):
        if file:
            #print('recibiendo archivo')
            data = pd.read_csv(file)
        else:
            #print('default')
            data = pd.read_csv(dataset)

        data_np = np.array(data[columnas])
        new_data = data[columnas]

        if estandarizacion == 'StandardScaler':
            estandarizar = StandardScaler()
        else:
            estandarizar = MinMaxScaler()

        MEstandarizada = estandarizar.fit_transform(data_np)
        MJerarquico = AgglomerativeClustering(n_clusters=int(numero_clusters), linkage='complete', metric=metrica)
        MJerarquico.fit_predict(MEstandarizada)

        new_data['clusterH'] = MJerarquico.labels_
        centroidesH = new_data.groupby('clusterH').mean()

        return {
            "sucess": True,
            "centroides": centroidesH.to_json(orient='records')
        }

    @staticmethod
    def obtenerCodo(estandarizacion, columnas, file=None, dataset="datos_prueba/apriori/movies.csv"):
        if file:
            #print('recibiendo archivo')
            data = pd.read_csv(file)
        else:
            #print('default')
            data = pd.read_csv(dataset)

        # agarramos solo las columnas que seleccionamos
        data = np.array(data[columnas])

        if estandarizacion == 'StandardScaler':
            estandarizar = StandardScaler()
        else:
            estandarizar = MinMaxScaler()

        MEstandarizada = estandarizar.fit_transform(data)

        #Definición de k clusters para K-means
        #Se utiliza random_state para inicializar el generador interno de números aleatorios
        SSE = []
        for i in range(2, 12):
            km = KMeans(n_clusters=i, random_state=0)
            km.fit(MEstandarizada)
            SSE.append(km.inertia_)

        #Se grafica SSE en función de k
        plt.figure(figsize=(10, 7))
        plt.plot(range(2, 12), SSE, marker='o')
        plt.xlabel('Cantidad de clusters *k*')
        plt.ylabel('SSE')
        plt.title('Elbow Method')

        temp_file = f'{Clustering.generar_hash_aleatorio()}.png'
        # Guardar la figura en un archivo de imagen
        plt.savefig(f'image_tmp/{temp_file}')
        plt.close()

        kl = KneeLocator(range(2, 12), SSE, curve="convex", direction="decreasing")
        plt.style.use('ggplot')
        temp_file2 = f'{Clustering.generar_hash_aleatorio()}.png'
        # Guardar la figura en un archivo de imagen
        kl.plot_knee()
        plt.savefig(f'image_tmp/{temp_file2}')
        plt.close()

        return {
            "sucess": True,
            "data": {
                "image": f"http://localhost:8000/api/clustering/image/{temp_file}",
                "knee_point": f"http://localhost:8000/api/clustering/image/{temp_file2}"
            }
        }

    def obteniendoClustersParticional(numero_clusters, estandarizacion, columnas, file=None, dataset="datos_prueba/apriori/movies.csv"):
        if file:
            #print('recibiendo archivo')
            data = pd.read_csv(file)
        else:
            #print('default')
            data = pd.read_csv(dataset)

        data_np = np.array(data[columnas])
        new_data = data[columnas]

        if estandarizacion == 'StandardScaler':
            estandarizar = StandardScaler()
        else:
            estandarizar = MinMaxScaler()

        MEstandarizada = estandarizar.fit_transform(data_np)
        MParticional = KMeans(n_clusters=int(numero_clusters), random_state=0).fit(MEstandarizada)
        MParticional.predict(MEstandarizada)

        new_data['clusterP'] = MParticional.labels_
        centroidesP = new_data.groupby('clusterP').mean()

        return {
            "sucess": True,
            "centroides": centroidesP.to_json(orient='records')
        }


'''
NOTE:

This script provides a demonstration of how the GLM
models in the published paper were fit. This script
is intended ONLY as a demonstration, as there is not
enough training data supplied to produce high-quality
GLM model fits.

There is not much value in using the fits that result
from this script, as they will be massively overfit
to the training data and will not generalize

We use the same hyperparameters here as were used to
fit the models in the paper.
'''

import argparse
import pickle
from typing import Dict, List, Tuple, Union, Optional
import math
import tqdm

import numpy as np
import torch

from encoding_models.poisson_encoder import ScaledPoissonFittedParams, SharedStimMulticellScaleOnlyPoisson

from convex_solver_base.unconstrained_optim import FistaSolverParams, UnconstrainedSolverParams, \
    batch_parallel_unconstrained_solve

from data_util.matched_cells_struct import OrderedMatchedCellsStruct
from data_util.load_data import load_typed_dataset, load_cell_ordering, load_cropped_stas_by_type, \
    compute_stimulus_onset_spikes, get_fit_cell_binned_spikes_lnp

def parallel_no_window_fit_scaled_poissons(all_images: np.ndarray,
                                           cell_response_vector: np.ndarray,
                                           filters_for_cell: np.ndarray,
                                           solver_params: UnconstrainedSolverParams,
                                           device: torch.device,
                                           batch_size=16) -> List[ScaledPoissonFittedParams]:

    n_trials, frame_height, frame_width = all_images.shape
    n_cells = len(sta_list)

    # we have to do this in batches otherwise we run out of GPU space for the midgets
    # also might speed the whole thing up

    full_stimuli = torch.tensor(all_images.reshape(n_trials, -1), dtype=torch.float32, device=device)

    fitted_model_list = []  # type: List[ScaledPoissonFittedParams]

    pbar = tqdm.tqdm(total=int(math.ceil(n_cells / batch_size)))
    for low in range(0, n_cells, batch_size):
        high = min(low + batch_size, n_cells)
        curr_batch_size = high - low

        relevant_filters = filters_for_cell[low:high, ...].reshape((curr_batch_size, -1))
        relevant_spikes = cell_response_vector[:, low:high].T

        relevant_spikes_torch = torch.tensor(relevant_spikes, dtype=torch.float32, device=device)

        # make the model
        multicell_poisson_model = SharedStimMulticellScaleOnlyPoisson(
            relevant_filters,
        ).to(device)  # type: SharedStimMulticellScaleOnlyPoisson

        losses = batch_parallel_unconstrained_solve(
            multicell_poisson_model,
            solver_params,
            verbose=True,
            shared_stimulus=full_stimuli,
            multicell_spikes=relevant_spikes_torch
        ).detach().cpu().numpy()

        # then extract all of the fitted parameters
        trained_filters, trained_biases = multicell_poisson_model.get_filters_and_biases()
        for idx in range(curr_batch_size):
            fitted_model_list.append(ScaledPoissonFittedParams(
                trained_filters[idx, ...].reshape((frame_height, frame_width)),
                trained_biases[idx],
                losses[idx],
            ))

        pbar.update(1)

        del relevant_filters
        del multicell_poisson_model

    del full_stimuli

    pbar.close()

    return fitted_model_list

if __name__ == '__main__':
    parser = argparse.ArgumentParser('Fit scaled single-bin LNP models to flashed trial data')
    parser.add_argument('save_path', type=str, help='path to save model fits')
    parser.add_argument('-gpu', '--gpu', action='store_true', default=False, help='use GPU')
    args = parser.parse_args()

    use_gpu = args.gpu
    device = torch.device('cuda') if use_gpu else torch.device('cpu')

    ################################################################
    # get the data
    # IMPORTANT: THIS IS A TINY SUBSET OF THE ACTUAL DATASET THAT WAS
    # USED TO FIT THE GLM; THESE FITS ARE NOT GOING TO BE VERY GOOD BY
    # DEFINITION
    train_frames, train_spikes_dict = load_typed_dataset()
    train_spikes_lnp_dict = {cell_type: compute_stimulus_onset_spikes(glm_spikes)
                             for cell_type, glm_spikes in train_spikes_dict.items()}

    ################################################################
    # load the STA prior
    all_prior_filters = load_cropped_stas_by_type()

    ################################################################
    # Load the cell types and matching
    cells_ordered = load_cell_ordering()  # type: OrderedMatchedCellsStruct
    ct_order = cells_ordered.get_cell_types()

    #### Train the scaled Poisson for every cell #################################
    typed_model_dict = {}  # type: Dict[str, List[ScaledPoissonFittedParams]]
    for cell_type in ct_order:
        print("Fitting {0}".format(cell_type))

        reference_id_list = cells_ordered.get_reference_cell_order(cell_type)
        cell_response_vector = get_fit_cell_binned_spikes_lnp(train_spikes_lnp_dict,
                                                              cell_type,
                                                              reference_id_list,
                                                              cells_ordered)

        sta_list = all_prior_filters[cell_type]

        all_fits = parallel_no_window_fit_scaled_poissons(
            train_frames,
            cell_response_vector,
            sta_list,
            FistaSolverParams(initial_learning_rate=1.0,
                              max_iter=100,
                              backtracking_beta=0.5,
                              converge_epsilon=1e-6),
            device,
            batch_size=32  # in reality, we use 2 for the full dataset
        )

        typed_model_dict[cell_type] = all_fits

    with open(args.save_path, 'wb') as pfile:
        pickle.dump(typed_model_dict, pfile)


""" URL config for board views """

from django.urls import path
from .views import notice_boards, display_single_board, set_favourite_board, create_notice_board

urlpatterns = [
    path('', notice_boards, name="notice_boards"),
    path('<int:board_pk>/', display_single_board, name="display_single_board"),
    path('<int:board_pk>/set_as_favourite/', set_favourite_board, {'set_unset':True}, name="set_favourite_board"),
    path('<int:board_pk>/unset_as_favourite/', set_favourite_board, name="unset_favourite_board"),
    path('create_board/', create_notice_board, name="create_notice_board"),
]


from argparse import ArgumentParser
from .model_loader import *
from .fad_batch import cache_embedding_files

def main():
    """
    Launcher for caching embeddings of directories using multiple models.
    """
    models = {m.name: m for m in get_all_models()}

    agupa = ArgumentParser()

    # Accept multiple models and directories with distinct prefixes
    agupa.add_argument('-m', '--models', type=str, choices=list(models.keys()), nargs='+', required=True)
    agupa.add_argument('-d', '--dirs', type=str, nargs='+', required=True)

    # Add optional arguments
    agupa.add_argument('-w', '--workers', type=int, default=8)
    agupa.add_argument('-s', '--sox-path', type=str, default='/usr/bin/sox')

    args = agupa.parse_args()

    for model_name in args.models:
        model = models[model_name]
        for d in args.dirs:
            log.info(f"Caching embeddings for {d} using {model.name}")
            cache_embedding_files(d, model, workers=args.workers)

if __name__ == "__main__":
    main()


from node import node
from timer import timer
import statistics
import random
import math

black = (0, 0, 0)

class keeper:
    def __init__(self, bounds, safetyBounds=0.1):
        self.nodes = []
        self.bounds = bounds
        self.c_dis = 0
        self.a_dis = 0
        self.ang_dis = 0
        self.lock_clustering = False
        self.lock_avoidance = False
        self.lock_angleMatching = False
        self.lock_maxAngle = False
        self.safetyBounds = safetyBounds

    #Debugging #################
    def debugHerd(self):
        print("Number of Nodes: " + len(self.nodes))
        print("Clustering: " + str(c_dis > 0))

    #Generation and Updating #################
    def updateHerd(self):
        for nodeNum in range(0, len(self.nodes)):
            node = self.nodes[nodeNum]
            listing = self.sortByDistance(node)
            if(self.c_dis > 0):
                self.clusterNode(node, listing)
            if(self.a_dis > 0):
                self.avoidNode(node, listing)
            if(self.ang_dis > 0):
                self.matchNode(node, listing)

        for nodeNum in range(0, len(self.nodes)):
            node = self.nodes[nodeNum]
            node.updatePosistion()
            node = self.handleEdges(node)
        return(self.nodes)

    def addRandomizedNodes(self, nodeSize, num, speed=10, color=black):
        for i in range(0, num):
            locX = random.randint(0, self.bounds[0])
            locY = random.randint(0, self.bounds[1])
            angle = random.randint(0, 359)
            self.generateNode(nodeSize, [locX, locY], speed=speed, angle=angle,color=color)

    def generateNode(self, nodeSize, pos, speed = 0, angle = 0, color=black):
        newNode = node(nodeSize, pos = pos, speed = speed, angle = angle,  color=color)
        self.nodes.append(newNode)
        return(newNode)

    def handleEdges(self, node):
        #Behavor: when we come to an edge we stop the node
        xBound = self.bounds[0]
        yBound = self.bounds[1]
        xPos = node.pos[0]
        yPos = node.pos[1]
        radius = node.nSize/2

        safteyX = xBound*self.safetyBounds
        safteyY = yBound*self.safetyBounds

        if(xPos < 0):
            node.atEdge = 1
            node.pos[0] = 0
            node.setRestriction(1)
        elif(xPos > xBound - radius):
            node.atEdge = 2
            node.pos[0] = xBound - radius
            node.setRestriction(2)
        elif(yPos < 0):
            node.atEdge = 3
            node.pos[1] = 0
            node.setRestriction(3)
        elif(yPos > yBound - radius):
            node.atEdge = 4
            node.pos[1] = yBound - radius
            node.setRestriction(4)
        else:
            node.atEdge = 0
            rNum = node.restricted
            if((rNum == 1) or (rNum == 2)):
                if((xPos > safteyX) and (xPos < xBound - radius - safteyX)):
                    node.setRestriction(0)
            if((rNum == 3) or (rNum == 4)):
                 if((yPos > safteyY) and (yPos < yBound - radius - safteyY)):
                     node.setRestriction(0)
        return(node)

    ## DISTANCE #################
    def getNumClose(self, distance, listing):
        details = []
        i = 0
        while((i < len(listing)) and (listing[i][0] <= distance)):
            details.append([listing[i][1], listing[i][2], listing[i][3]])
            i+=1
        return(details)

    def sortByDistance(self, iNode):
        listing = []
        initX = iNode.pos[0]
        initY = iNode.pos[1]

        for node in self.nodes:
            if(node is not iNode):
                finalX = node.pos[0]
                finalY = node.pos[1]
                angle = node.angle
                d_calc = math.sqrt((finalX - initX)**2 + (finalY - initY)**2)
                listing.append([d_calc, finalX, finalY, angle])
        listing = sorted(listing)
        return(listing)

    ## CLUSTERING #################
    def enableClustering(self, distance):
        if(self.lock_clustering):
            return
        if(distance < 0):
            distance = 0

        self.c_dis = distance

    def clusterNode(self, node, listing):
        nodeX = node.pos[0]
        nodeY = node.pos[1]
        avgLoc = self.findLocalMean(self.c_dis, listing)
        if(avgLoc is not None):
            node.headToPos(avgLoc)

    def findLocalMean(self, distance, listing):
        listing = self.getNumClose(distance, listing)
        if(len(listing) == 0):
            return None
        xAvg = 0
        yAvg = 0
        for l in listing:
            xAvg += l[0]
            yAvg += l[1]
        xAvg = xAvg/len(listing)
        yAvg = yAvg/len(listing)
        return([xAvg, yAvg])

    ## AVOIDANCE
    def enableAvoidance(self, distance):
        if(self.lock_avoidance):
            return

        if(distance < 0):
            distance = 0

        self.a_dis = distance

    def avoidNode(self, node, listing):
        avgLoc = self.findLocalMean(self.a_dis, listing)
        if(avgLoc is not None):
            node.headToPos(avgLoc, invert=True)

    ## Angle Matching
    def enableAngleMatching(self, distance):
        if(self.lock_angleMatching):
            return

        if(distance < 0):
            distance = 0

        self.ang_dis = distance

    def matchNode(self, node, listing):
        angles = [node.angle]
        listing = self.getNumClose(self.ang_dis,listing)
        if(len(listing) > 1):
            for i in range(0,len(listing)):
                angles.append(listing[i][2])
            node.requestAngleChange(statistics.median(angles))

    #Match Angle
    def enableMaxAngle(self, maxAngle):
        if(self.lock_maxAngle):
            return

        for node in self.nodes:
            node.maxAng = maxAngle


from flask import Flask, request, render_template
app = Flask(__name__)

@app.route('/')
def hello():
    q = request.args.get('q')
    print(q)
    return { "message": "Hello!" }, 201

@app.route('/number/<num>')
def showNumber(num):
    q = request.args.get('q')
    print(q)
    return { "message": num }, 201

@app.route('/showWeb/<name>')
def showWebSite(name=None):
    return render_template('test.html', name=name)

@app.route('/login', methods=['GET', 'POST'])
def login():
    if request.method == "POST":
        return { "message" : 'method post เด้อ'}, 201
    elif request.method == "GET":
        return { "message" : 'method GET เด้อ'}, 201

if __name__=='__main__':
    app.run(debug=True)


#!/usr/bin/env python3
"""
Take the code from wait_n and alter it into
a new function task_wait_n
"""
import asyncio
from typing import List
from basic_async_syntax import wait_random

task_wait_random = __import__('3-tasks').task_wait_random

async def wait_n(n: int, max_delay: int) -> List[float]:
    """
    Asynchronous routine that spawns wait_random n times
    and returns the list of delays in ascending order.

    Args:
        n (int): The number of times to spawn wait_random.
        max_delay (int): The max delay in seconds for each
        call to wait_random.

    Returns:
            List[float]: A list of delays (float values) in
            ascending order
    """
    # Using asyncio.gather to run wait_dom n times
    delays = []
    tasks = []

    for i in range(n):
        task = wait_random(max_delay)
        tasks.append((i, task))

    for i, task in tasks:
        delay = await task
        delays.append((i, delay))

    sorted_delays = sorted(delays, key=lambda x: x[0])
    sorted_values = [delay for _, delay in sorted_delays]

    return sorted_values

async def task_wait_n(n: int, max_delay: int):
    """
    Asynchronous coroutine that spawns task_wait_random
    n times with the specified max_delay and return the
    list of delays in ascending order.

    Args:
        n (int): The number of times to spawn task_wait_randon
        max_delay (int): The maximum delay in secoconds for
        call to task_wait_random

    Returns:
        List[float]: Alist of delays (float value) in ascending
        order.
    """

    delays = [(i, await task_wait_random(max_delay)) for i in range(n)]
    delays.sort(key=lambda x: x[0])
    return [delay for _, delay in delays]

if __name__ == "__main":
    print(asyncio.run(task_wait_n(5, 5)))
    print(asyncio.run(task_wait_n(10, 7)))
    print(asyncio.run(task_wait_n(10, 0)))


"""PyFMD example

This example demonstrate how to build a structure using PyFMD interface for ASE and merge them together.
"""

from ase.visualize import view
import numpy as np
from ase.build import bulk
from ase.build.supercells import make_supercell

from pyfmd import *

# Create a super cell via ASE package
def make_supercell_cu():
    crys = bulk('Cu', 'fcc', a=3.6316, orthorhombic=True)
    P = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) * 10
    return make_supercell(crys, P)

# Create molecular frames 1 (group-id = 0)
mf1 = MolecularFrame('Cu crystal 1').import_from(make_supercell_cu()).set_group_id(0)
# view(mf1.export())

# Create molecular frames 2 (group-id = 1)
mf2 = MolecularFrame('Cu crystal 2').import_from(make_supercell_cu()).set_group_id(1)
# view(mf2.export())

# Clone mf2 and translate atoms along given direction
mf2_moved = mf2.translate_atoms((30,-15,-15), inplace=False)

# Define region
def sphere(x, y, z):
    """sphere region"""
    if np.sqrt(x**2+y**2+z**2) < 10.0:
        return True
    else:
        return False

# Translate and select atoms within predefined region
mf1_sphere = mf1.translate_atoms((-10,-10,-10), inplace=False).select(Select().region(sphere))
# view(mf1_sphere.export())

# Merge two molecular frames
mf = mf1_sphere + mf2_moved
mf.translate_atoms((50, 50, 50))

# Define box and/or state header
box = Box((150, 100, 100))
state = StateHeader(simulation_time=0.0, number_of_atoms=mf.number_of_atoms, box=box, pbc=(False, False, False))
mf.set_sections([box, state])

# print(mf)
mf.write("system.stt")  # This state file can base used FMD simulation.
mf.read("system.stt")
view(mf.export())