# -*- coding: utf-8 -*-
"""
Created on 10-04-2015

@author: Diogo Silva

Evidence accumulation clustering. This module aims to include all
features of the Matlab toolbox plus addressing NxK co-association
matrices.

TODO:
- clustering of non-square co-association matrix
- link everything
- add sanity checks on number of samples of partitions
- robust exception handling
- fix centroid-based prototype creation
- convert dissimilarity matrix to float32 (why?)
"""

import numpy as np
from random import sample
from numba import jit, njit
# from sklearn.neighbors import NearestNeighbors

from scipy.sparse import csr_matrix
from scipy.cluster.hierarchy import linkage,dendrogram
from scipy.sparse.csgraph import connected_components

from scipy_numba.sparse.csgraph import minimum_spanning_tree
from scipy_numba.spatial.distance import squareform

from MyML.cluster.linkage import scipy_numba_slink_wraper as slink
from MyML.cluster.linkage import labels_from_Z
from MyML.cluster.K_Means3 import K_Means

from MyML.EAC.sparse import EAC_CSR, _compute_max_assocs_from_ensemble as biggest_cluster_size
from MyML.EAC.full import EAC_FULL

def sqrt_rule(n):
    n_clusters = [np.sqrt(n)/2, np.sqrt(n)]
    n_clusters = map(np.ceil, n_clusters)
    n_clusters = map(int, n_clusters)
    return n_clusters

def sqrt2_rule(n):
    n_clusters = [np.sqrt(n), np.sqrt(n)*2]
    n_clusters = map(np.ceil, n_clusters)
    n_clusters = map(int, n_clusters)
    return n_clusters

class EAC():

    def __init__(self, n_samples, **kwargs):
        """
        mat_sparse         : stores co-associations in a sparse matrix
        mat_half         : stores co-associations in pdist format, in an (n*(n-1))/2 length array
        """

        self.n_samples = n_samples

        # check if all arguments were passed as a dictionary
        args = kwargs.get("args")
        if args is not None and type(args) == dict:
            kwargs == args

        ## generate ensemble parameters
        self.n_partitions = kwargs.get("n_partitions", 100)
        self.iters = kwargs.get("iters", 3)
        self.n_clusters = kwargs.get("n_clusters", "sqrt")
        self.toFiles = False
        self.toFiles_folder = None

        ## build matrix parameters
        self.condensed = kwargs.get("condensed", True)
        self.kNN = kwargs.get("kNN", False)
        self.assoc_dtype = kwargs.get("assoc_dtype", np.uint8)

        # sparse matrix parameters
        self.sparse = kwargs.get("sparse", False)
        self.sp_sort = kwargs.get("sparse_sort_mode", "surgical")
        self.sp_max_assocs = kwargs.get("sparse_max_assocs", None)
        self.sp_max_assocs_factor = kwargs.get("sparse_max_assocs_factor", 3)
        self.sp_max_assocs_mode = kwargs.get("sparse_max_assocs_mode", "linear")
        self.sp_keep_degree = kwargs.get("sparse_keep_degree", False)

        # if not sparse and not kNN then it is full matrix
        if not self.sparse  and not self.kNN:
            self.full = True
        else:
            self.full = False

        ## final clustering parameters
        self.linkage = kwargs.get("linkage", "SL")

    def _validate_params(self):
        pass

    def generateEnsemble(self):
        pass

    def buildMatrix(self, ensemble):

        if self.sparse:
            if self.sp_max_assocs is None:
                self.sp_max_assocs = biggest_cluster_size(ensemble)
                self.sp_max_assocs *= self.sp_max_assocs_factor

            coassoc = EAC_CSR(self.n_samples, max_assocs=self.sp_max_assocs,
                              condensed=self.condensed,
                              max_assocs_type=self.sp_max_assocs_mode,
                              sort_mode=self.sp_sort,
                              dtype=self.assoc_dtype)

            coassoc.update_ensemble(ensemble)
            coassoc._condense(keep_degree = self.sp_keep_degree)
        elif self.full:
            coassoc = EAC_FULL(self.n_samples, condensed=self.condensed,
                               dtype=self.assoc_dtype)
            coassoc.update_ensemble(ensemble)
        elif self.kNN:
            raise NotImplementedError("kNN matrix building has not been included in this version.")
        else:
            raise ValueError("Build matrix: No sparse, no full, no kNN. No combination possible.")

        self.coassoc = coassoc

        # received names of partition files
        # if files:
        #     for partition_file in ensemble:
        #         partition = self._readPartition(partition_file) # read partition from file
        #         self._update_coassoc_matrix(partition) # update co-association matrix
        # # received partitions
        # else:
        #     for partition in ensemble:
        #         self._update_coassoc_matrix(partition) # update co-association matrix

    def finalClustering(self, n_clusters=0):
        if self.sparse:
            n_fclusts, labels = sp_sl_lifetime(self.coassoc.csr,
                                               max_val=self.n_partitions,
                                               n_clusters=n_clusters)
        elif self.full:
            n_fclusts, labels = full_sl_lifetime(self.coassoc.coassoc,
                                                 self.n_samples,
                                                 max_val=self.n_partitions,
                                                 n_clusters=n_clusters)
        elif self.kNN:
            raise NotImplementedError("kNN not included in this version yet.")
        else:
            raise ValueError("Final clustering: No sparse, no full, no kNN. No combination possible.")

        self.n_fclusts = n_fclusts
        self.labels = labels
        return labels

    def fit(self, ensemble,files=False, assoc_mode="full", prot_mode="none",
            nprot=None, link='single', build_only=False):
        """
        ensemble    : list of partitions; each partition is a list of
                      arrays (clusterings); each array contains the indices
                      of the cluster's data;  if files=True, partitions is
                      a list of file names, each corresponding to a partition
        assoc_mode  : type of association matrix; "full" - NxN, "prot" - NxK prototypes
        prot_mode   : how to build the prototypes; "random" - random selection
                      of K data points, "knn" for K-nearest neighbours, "other"
                      for K centroids/medoids
        nprot       : num. of prototypes to use; default = sqrt(num. of samples)
        """
        # how to build association matrix
        if self._assoc_mode is None:
            self._assoc_mode = assoc_mode
        # how to build prototypes
        if self._prot_mode is None:
            self._prot_mode = prot_mode

        # create co-association matrix
        self._coassoc = self._create_coassoc(assoc_mode, self.n_samples, nprot=nprot)

        if prot_mode is not "none":
            # changing assoc_mode for the matrix updates
            if prot_mode == "knn":
                self._assoc_mode="knn"

            elif assoc_mode == "full" and prot_mode == "random":
                self._assoc_mode = "full_random"

            elif prot_mode == "random":
                self._assoc_mode = "random"

            else:
                self._assoc_mode="other"

            self._build_prototypes(nprot=nprot, mode=prot_mode, data=self.data)

        self.n_partitions = 0

        # delete diagonal
        #self._coassoc[xrange(self.n_samples),xrange(self.n_samples)] = np.zeros(self.n_samples)

        # convert sparse matrix to convenient format, if it is sparse
        if self.mat_sparse:
            self._coassoc = self._coassoc.tocsr()
        # else:
        # 	self._coassoc[np.diag_indices_from(self._coassoc)] = 0

    def _create_coassoc(self, mode, nsamples, nprot=None):
        if self.condensed:
            n = sum(xrange(1, nsamples))
            coassoc = np.zeros(n, dtype=self.assoc_type)
        elif mode == "full":
            if self.mat_sparse:
                coassoc = sparse_type((nsamples, nsamples), dtype=self.assoc_type)
            else:
                coassoc = np.zeros((nsamples, nsamples), dtype=self.assoc_type)
        elif mode =="prot":
            if nprot == None:
                nprot = np.sqrt(nsamples)
            coassoc = np.zeros((nsamples,nprot), dtype=self.assoc_type)
        else:
            validValues=("full", "prot")
            raise ValueError("mode value should be from the list:\t" + str(validValues))

        return coassoc

    def _readPartition(self, filename):
        # list to hold the cluster arrays
        partition = list()

        with open(filename, "r") as pfile:
            # read cluster lines
            for cluster_line in pfile:
                if cluster_line == '\n':
                    continue
                cluster = np.fromstring(cluster_line, sep=',', dtype=np.int32)
                partition.append(cluster)

        return partition

    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                      BUILD PROTOTYPES                       # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

    def _build_prototypes(self, nprot=None, mode="random", data=None):
        if nprot == None:
            nprot = np.sqrt(self.n_samples)

        if mode == "random":
            self.k_labels = self._build_random_prototypes(nprot, self.n_samples)
            self.k_labels.sort()

        elif mode == "knn":
            if data is None:
                raise Exception("Data needs to be set for this method of choosing prototypes.")
            self.k_neighbours = self._build_knn_prototypes(nprot, data)

        elif mode == "other":
            if data is None:
                raise Exception("Data needs to be set for this method of choosing prototypes.")
            self.k_labels = self._build_k_prototypes(nprot, data)

        else:
            validValues=("random","knn","other")
            raise ValueError("Mode value should be from the list:\t" + str(validValues))

    def _build_random_prototypes(self, nprot, nsamples):

        # select nprot unique random samples from the dataset
        return np.array(sample(xrange(nsamples), nprot), dtype=np.int32)

    def _build_knn_prototypes(self, nprot, data):
        """
        K-Nearest Neighbours algorithm
        should return an NxK array with the labels
        """
        #first neighbour is the point itself, it gets discarded afterwards
        nneigh = nprot + 1

        # Minkowski distance is a generalization of Euclidean distance and
        # is equivelent to it for p=2
        neigh = NearestNeighbors(n_neighbors=nneigh, radius=1.0,
                                 algorithm='auto', leaf_size=30,
                                 metric='minkowski', p=2)
        neigh.fit(data)

        k_indices = neigh.kneighbors(X=data, return_distance=False)
        k_indices = k_indices[:,1:] # discard first neighbour

        return k_indices

    def _build_k_prototypes(self, nprot, data):
        # K-Means / K-Medoids algorithm
        # should return a N-length array with he indices of the chosen data
        grouper = K_Means()
        grouper._centroid_mode = "index"
        grouper.fit(data, nprot, iters=300, mode="cuda", cuda_mem='manual',
                    tol=1e-4, max_iters=300)
        centroids = grouper.centroids

        nclusters = centroids.shape[0]

        # TODO - very inefficient
        k_labels = np.zeros(nclusters, dtype=np.int32)

        for k in xrange(nclusters):
            dist = data - centroids[k]
            dist = dist ** 2
            dist = dist.sum(axis=1)

            k_labels[k] = dist.argmin()
        return k_labels

    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #              UPDATE CO-ASSOCIATION MATRIX                   # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

    def _update_coassoc_k(self, assoc_mat, clusters, k_labels):
        """
        Updates an NxK co-association matrix.
        k_labels is an array (List, not np.ndarray) of length K where the k-th
        element is the index of a data point that corresponds to the
        k-th prototype.
        """

        nclusters = len(clusters)
        for i in xrange(nclusters): # for each cluster in ensemble
            # if cluster has more than 1 sample (i.e. not outlier)
            if clusters[i].size > 1:

                # all data points in cluster - rows to select
                n_in_cluster = clusters[i]

                ## select prototypes present in cluster - columns to select
                # in1d checks common values between two 1-D arrays (a,b) and
                # returns boolean array with the shape of a with value True on
                # the indices of common values
                k_in_cluster = np.where(np.in1d(k_labels, n_in_cluster))[0]

                if k_in_cluster.size == 0:
                    continue

                # this indexing selects the rows and columns specified by
                # n_in_cluster and k_in_cluster; np.newaxis is alias for None
                assoc_mat[n_in_cluster[:, np.newaxis], k_in_cluster] += 1

    def _update_coassoc_full_k(self, assoc_mat, clusters, k_labels):
        """
        Updates an NxN co-association matrix with only K prototypes specified
        by k_labels. k_labels is an array (List, not np.ndarray) of length K
        where the k-th element is the index of a data point that corresponds
        to the k-th prototype.
        """

        nclusters = len(clusters)
        for i in xrange(nclusters): # for each cluster in ensemble
            # if cluster has more than 1 sample (i.e. not outlier)
            if clusters[i].size > 1:

                # all data points in cluster - rows to select
                n_in_cluster = clusters[i]

                ## select prototypes present in cluster - columns to select
                # in1d checks common values between two 1-D arrays (a,b) and
                # returns boolean array with the shape of a with value True on
                # the indices of common values
                prots_in_cluster = np.intersect1d(k_labels, n_in_cluster)
                points_in_cluster = np.setdiff1d(n_in_cluster, prots_in_cluster)

                if prots_in_cluster.size == 0:
                    continue

                # this indexing selects the rows and columns specified by
                # n_in_cluster and k_in_cluster; np.newaxis is alias for None
                # select all rows that are not prots and all columns that are
                # prots and increment them
                assoc_mat[points_in_cluster[:, np.newaxis], prots_in_cluster] += 1
                assoc_mat[prots_in_cluster[:, np.newaxis], points_in_cluster] += 1

                # select all rows and columns that are prots and increment them
                assoc_mat[prots_in_cluster[:, np.newaxis], prots_in_cluster] += 1

    def _update_coassoc_knn(self, assoc_mat, clusters, k_neighbours):
        """
        Updates an NxK co-association matrix.
        k_neighbours is an NxK array where the k-th element of the i-th row is
        the index of a data point that corresponds to the k-th nearest neighbour
        of the i-th data point. That neighbour is the k-th prototype of the
        i-th data point.
        """
        nclusters = len(clusters)
        for i in xrange(nclusters):

            if clusters[i].size > 1:

                # all data points in cluster - rows to select
                n_in_cluster = clusters[i]

                # update row j of matrix
                for j in n_in_cluster:
                    # all prototypes in cluster - columns to select
                    k_in_cluster = np.in1d(k_neighbours[j], n_in_cluster)

                    if k_in_cluster.size == 0:
                        continue

                    # this indexing selects the rows and columns specified by
                    # n_in_cluster and k_in_cluster
                    assoc_mat[j, k_in_cluster] += 1 # newaxis is alias for None
        pass
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                      OPERATIONS                             # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    def apply_threshold(self, threshold):
        """
        threshold   : all co-associations whose value is below
                      threshold * max_val are zeroed
        max_val     : usually number of partitions
        assoc_mat   : co-association matrix
        """
        assoc_mat = self._coassoc
        max_val = self.n_partitions
        apply_threshold_to_coassoc(threshold, max_val, assoc_mat)

    def getMaxAssocs(self):
        """\
        Returns the maximum number of co-associations a sample has and the
        index of that sample.\
        """
        # if not hasattr(self, 'degree'):
        #     self._getAssocsDegree()
        # return self.degree.max()

        if not self.mat_sparse:
            max_assocs, max_idx = get_max_assocs_in_sample(self._coassoc)
        else:
            max_assocs, max_idx = get_max_assocs_in_sample_csr(self._coassoc)
        return max_assocs, max_idx

    def _getAssocsDegree(self):
        self.degree = np.zeros(self.n_samples, dtype=np.int32)
        if self.condensed:
            error_str = "Getting degree from condensed matrix. Alternative: " +\
                         "convert to 2d, get degree, multiply by 2."
            raise NotImplementedError(error_str)
        elif not self.mat_sparse:
            full_get_assoc_degree(self._coassoc, self.degree)
        else:
            self.degree = self._coassoc.indptr[1:] - self._coassoc.indptr[:-1]
        self.nnz = self.degree.sum()

    def getNNZAssocs(self):
        """Get total number of associations in co-association matrix."""
        if not self.mat_sparse:
            #return np.count_nonzero(self.._coassoc)
            return numba_array2d_nnz(self._coassoc, self._coassoc.shape[0],
                                     self._coassoc.shape[1])
        else:
            return self._coassoc.getnnz()

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # #                                                             # # # #
# # # #                                                             # # # #
# # # #                                                             # # # #
# # # #                      FINAL CLUSTERING                       # # # #
# # # #                                                             # # # #
# # # #                                                             # # # #
# # # #                                                             # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

def sp_sl_lifetime(mat, max_val=False, n_clusters=0):
    """
    Converts graph weights to dissimilarities if input graph is in
    similarities. Computes MST (Kruskal) of dissimilarity graph.
    Compute number of disconnected clusters (components).
    Sort MST in increasing order to get equivalent of SL clustering.
    Compute lifetimes if number of clusters is not provided.
    Make necessary cuts to have the desired number of clusters.
    Compute connected components (clusters) after the cuts.

    Inputs:
        graph           : dis/similarity matrix in CS form.
        max_val         : maximum value from which dissimilarity will be
                          computed. If False (default) assumes input graph
                          already encodes dissimilarities.
        n_clusters      : number of clusters to compute. If 0 (default),
                          use lifetime criteria.
    Outputs:
        n_fclusts       : final number of clusters
        labels          : final clustering labels
    """

    dtype = mat.dtype

    # converting to diassociations
    if max_val != False:
        mat.data = max_val + 1 - mat.data

    # get minimum spanning tree
    mst = minimum_spanning_tree(mat)

    # compute number of disconnected components
    n_disconnect_clusters = mst.shape[0] - mst.nnz

    # sort associations by weights
    asort = mst.data.argsort()
    sorted_weights = mst.data[asort]

    if n_clusters == 0:
        cont, max_lifetime = lifetime_n_clusters(sorted_weights)

        if n_disconnect_clusters > 1:
            # add 1 to max_val as the maximum weight because I also added
            # 1 when converting to diassoc to avoid having zero weights
            disconnect_lifetime = max_val + 1 - sorted_weights[-1]

            # add disconnected clusters to number of clusters if disconnected
            # lifetime is smaller
            if max_lifetime > disconnect_lifetime:
                cont += n_disconnect_clusters - 1
            else:
                cont = n_disconnect_clusters

        nc_stable = cont
    else:
        nc_stable = n_clusters

    # cut associations if necessary
    if nc_stable > n_disconnect_clusters:
        n_cuts = nc_stable - n_disconnect_clusters

        mst.data[asort[-n_cuts:]] = 0
        mst.eliminate_zeros()

    if nc_stable > 1:
        n_comps, labels = connected_components(mst)
    else:
        labels = np.empty(0, dtype=np.int32)
        n_comps = 1

    return n_comps, labels

def full_sl_lifetime(mat, n_samples, max_val=False, n_clusters=0):

    dtype = mat.dtype

    # convert to diassoc
    if mat.ndim == 2:
        mat = squareform(mat)

    # converting to diassociations
    if max_val != False:
        make_diassoc_1d(mat, max_val + 1)

    #Z = linkage(mat, method="single")
    Z = slink(mat, n_samples)

    if n_clusters == 0:

        cont, max_lifetime = lifetime_n_clusters(Z[:,2])

        nc_stable = cont
    else:
        nc_stable = n_clusters

    if nc_stable > 1:
        labels = labels_from_Z(Z, n_clusters=nc_stable)
        # rename labels
        i=0
        for l in np.unique(labels):
            labels[labels == l] = i
            i += 1
    else:
        labels = np.empty(0, dtype=np.int32)

    return nc_stable, labels

def lifetime_n_clusters(weights):
    # compute lifetimes
    lifetimes = weights[1:] - weights[:-1]

    # maximum lifetime
    m_index = np.argmax(lifetimes)
    th = lifetimes[m_index]

    # get number of clusters from lifetimes
    indices = np.where(weights >weights[m_index])[0]
    if indices.size == 0:
        cont = 1
    else:
        cont = indices.size + 1

    #testing the situation when only 1 cluster is present
    # if maximum lifetime is smaller than 2 times the minimum
    # don't make any cuts (= 1 cluster)
    # max>2*min_interval -> nc=1
    close_to_zero_indices = np.where(np.isclose(lifetimes, 0))
    minimum = np.min(lifetimes[close_to_zero_indices])

    if th < 2 * minimum:
        cont = 1

    return cont, th

# 2d
@njit
def make_diassoc_2d(ary, val):
    for row in range(ary.shape[0]):
        for col in range(ary.shape[1]):
            tmp = ary[row,col]
            ary[row,col] = val - tmp

#1d
@njit
def make_diassoc_1d(ary, val):
    for i in range(ary.size):
        tmp = ary[i]
        ary[i] = val - tmp


#!/usr/bin/python3
# -*- coding: utf-8 -*-
#Written by: Filipe Dezordi (https://dezordi.github.io/)
#At FioCruz/IAM - 2020/07/06

import argparse, subprocess, shlex, csv, re, os
import pandas as pd

parser = argparse.ArgumentParser(description = 'This script returns nucleotide diversy per SNP position and metrics about SNPs by hCoV-19 genome region.')
parser.add_argument("-sn", "--snp_pos", help="SNPs positions in a txt file, onde position per line",  required=True)
parser.add_argument("-bc","--bam_readcount",help="bam-readcount output file", required=True)

args = parser.parse_args()
snp_file = args.snp_pos
bam_rc_file = args.bam_readcount
prefix = snp_file
prefix = re.sub("\..*","",prefix)
prefix = re.sub('.*/','',prefix).rstrip('\n')

"""
Genes position
UTR5 = [1,265]
ORF1AB = [266,21555]
S = [21563,25384]
ORF3A = [25393,26220]
E = [26242,26472]
M = [26523,27191]
ORF6 = [27202,27387]
ORF7A = [27394,27759]
ORF7B = [27756,27887]
ORF8 = [27894,28259]
N = [28274,29533]
ORF10 = [29558,29674]
UTR3 = [29675,29903]
"""

with open(snp_file,'r') as snp_in_file, open(snp_file+'.distr.tsv','w') as snp_file_out:
    reader_snp_in_file = snp_in_file.readlines()
    output_csv_writer = csv.writer(snp_file_out,delimiter='\t')
    output_csv_writer.writerow(['GENOME','POS','REGION','DEPTH','A_DEPTH','A_PLUS','A_MINUS','C_DEPTH','C_PLUS','C_MINUS','G_DEPTH','G_PLUS','G_MINUS','T_DEPTH','T_PLUS','T_MINUS','N'])
    out_list = []
    for snp_line in reader_snp_in_file:
        snp_line = snp_line.rstrip('\n')
        with open(bam_rc_file,'r') as bam_rc_file_in: #, open(bam_rc_file+'.csv','w') as output_csv:
            bam_rc_file_read = csv.reader(bam_rc_file_in, delimiter='\t')
            for line in bam_rc_file_read:
                if snp_line == line[1]:
                    var_pos = line[1].rstrip('\n')
                    var_depth = line[3].rstrip('\n')
                    A_depth_line = line[5].rstrip('\n')
                    A_depth = ":".join(A_depth_line.split(":", 2)[:2])
                    A_depth = re.sub(r".*:","",A_depth)
                    A_plus = ":".join(A_depth_line.split(":", 6)[:6])
                    A_plus = re.sub(r".*:","",A_plus)
                    A_minus = ":".join(A_depth_line.split(":", 7)[:7])
                    A_minus = re.sub(r".*:","",A_minus)
                    C_depth_line = line[6].rstrip('\n')
                    C_depth = ":".join(C_depth_line.split(":", 2)[:2])
                    C_depth = re.sub(r".*:","",C_depth)
                    C_plus = ":".join(C_depth_line.split(":", 6)[:6])
                    C_plus = re.sub(r".*:","",C_plus)
                    C_minus = ":".join(C_depth_line.split(":", 7)[:7])
                    C_minus = re.sub(r".*:","",C_minus)
                    G_depth_line = line[7].rstrip('\n')
                    G_depth = ":".join(G_depth_line.split(":", 2)[:2])
                    G_depth = re.sub(r".*:","",G_depth)
                    G_plus = ":".join(G_depth_line.split(":", 6)[:6])
                    G_plus = re.sub(r".*:","",G_plus)
                    G_minus = ":".join(G_depth_line.split(":", 7)[:7])
                    G_minus = re.sub(r".*:","",G_minus)
                    T_depth_line = line[8].rstrip('\n')
                    T_depth = ":".join(T_depth_line.split(":", 2)[:2])
                    T_depth = re.sub(r".*:","",T_depth)
                    T_plus = ":".join(T_depth_line.split(":", 6)[:6])
                    T_plus = re.sub(r".*:","",T_plus)
                    T_minus = ":".join(T_depth_line.split(":", 7)[:7])
                    T_minus = re.sub(r".*:","",T_minus)
                    N_depth = line[9].rstrip('\n')
                    N_depth = ":".join(N_depth.split(":", 2)[:2])
                    N_depth = re.sub(r".*:","",N_depth)
                    if int(var_pos) <= 265:
                        region = '5UTR'
                    elif int(var_pos) <= 21555:
                        region = 'ORF1AB'
                    elif int(var_pos) <= 25384:
                        region = 'S'
                    elif int(var_pos) <= 26220:
                        region = 'ORF3A'
                    elif int(var_pos) <= 26472:
                        region = 'E'
                    elif int(var_pos) <= 27191:
                        region = 'M'
                    elif int(var_pos) <= 27387:
                        region = 'ORF6'
                    elif int(var_pos) <= 27759:
                        region = 'ORF7A'
                    elif int(var_pos) <= 27887:
                        region = 'ORF7B'
                    elif int(var_pos) <= 28259:
                        region = 'ORF8'
                    elif int(var_pos) <= 29533:
                        region = 'N'
                    elif int(var_pos) <= 29674:
                        region = 'ORF10'
                    else:
                        region = '3UTR'
                    out_list.append([prefix,var_pos,region,var_depth,A_depth,A_plus,A_minus,C_depth,C_plus,C_minus,G_depth,G_plus,G_minus,T_depth,T_plus,T_minus,N_depth])
    output_csv_writer.writerows(out_list)
df = pd.read_csv(snp_file+'.distr.tsv',sep='\t')
df2 = df[['REGION','DEPTH']]
df2 = pd.DataFrame(group.describe().loc[['count','min','max','mean','std']].rename(columns={'DEPTH':name}).squeeze()for name, group in df2.groupby('REGION'))
df2 = df2.round(2)
df2.insert(loc=0,column='GENOME',value=prefix)
df2.to_csv(snp_file+'.metrics.tsv',sep='\t')


python_group = '''
let
  pwp = python{py_version}.buildEnv.override {{
    extraLibs = (with python{py_version}Packages; [
        {packages}
    ]);
    ignoreCollisions = true;
 }};

in
'''

synonyms = {
    'keras': ('Keras', 'package names are case sensitive'),

    # Be nice to users
    'sklearn': ('scikitlearn', None),
    'scikit-learn': ('scikitlearn', None),
}

def generate(data, output, _):
    '''
    Generate a Python group
    '''
    py_version = str(data['version'])
    py_version = {
            '2.7': '2',
            '3': '36',
            '3.4': '34',
            '3.5': '35',
            '3.6': '36',
            '3.7': '37',
            '3.8': '38',
            '3.9': '39',
            '3.10': '310',
            '3.11': '311',
            '3.12': '312',
            }.get(py_version, py_version)
    packages = []
    for p in data['modules']:
        if p in synonyms:
            np,note = synonyms[p]
            if note is not None:
                message = "Replacing package '{}' by '{}' ({}).".format(p, np, note)
            else:
                message = "Replacing package '{}' by its canonical name '{}'.".format(p, np)
            print(message)
            packages.append(np)
        else:
            packages.append(p)
    packages = '\n    '.join(packages)
    output.write(python_group.format(py_version=py_version, packages=packages))
    return ['pwp']


import pandas as pd
import numpy as np
from datetime import datetime
import matplotlib.pyplot as plt

#read CSVs
mandi = pd.read_csv("subset_mandi.csv")
centres = pd.read_csv("subset_centres.csv")
data = pd.read_csv("subset_retail.csv")
ws = pd.read_csv("subset_ws.csv")

ws['date'] =  pd.to_datetime(ws['date'], format='%Y-%m-%d')

a = ws[ws['mid']==11]
b = ws[ws['mid']==35]

idx = pd.date_range('2006-01-01', '2015-06-23')
a.set_index('date', inplace=True)
a = a.reindex(idx, fill_value=0)
a.reset_index(level=0, inplace=True)

b.set_index('date', inplace=True)
b = b.reindex(idx, fill_value=0)
b.reset_index(level=0, inplace=True)

#a = series1
a.reset_index(inplace=True)

#b = series2
b.reset_index(inplace=True)

a['price'] = pd.ewma(a['price'], span=7)
b['price'] = pd.ewma(b['price'], span=7)

df = a.head(len(a)-7)
#df.reset_index('index', inplace=True)

df['slope'] = df.apply(lambda x: 0 if ((b.ix[x['level_0']+7]['price'] - b.ix[x['level_0']]["price"])*x['price'] == 0) else (((a.ix[x['level_0']+7]['price'] - x['price'])*b.ix[x['level_0']]["price"])/((b.ix[x['level_0']+7]['price'] - b.ix[x['level_0']]["price"])*x['price'])), axis=1)

med = df["slope"].median()
tolerance = (med -df["slope"]).median()*1.4826
threshold_P = med + tolerance
threshold_N = med - tolerance
pos_slope = df[df['slope'] > threshold_P]
neg_slope = df[df['slope'] < threshold_N]

#a = pos_slope.reindex(range(0,3454),fill_value=0)
x = pos_slope.index.tolist()
for i in x:
	plt.axvline(x=i)
plt.show()
#plt.plot(a["slope"])


import json
import os
import time

import tvm
from tvm import relay
from tvm import autotvm
from tvm import rpc
from tvm.contrib import util
from tvm.contrib import graph_runtime as runtime
from tvm.contrib.debugger import debug_runtime
import numpy as np
import torch

from .get_model import get_model

def get_input_info(graph, params):
    """
    Get shape and dtype of the input tensors for a compiled model.
    """
    shape_dict = {}
    dtype_dict = {}
    graph = json.loads(graph)
    param_names = params.keys()
    for node_id in graph["arg_nodes"]:
        node = graph["nodes"][node_id]
        name = node["name"]
        if name not in param_names:
            shape_dict[name] = graph["attrs"]["shape"][1][node_id]
            dtype_dict[name] = graph["attrs"]["dltype"][1][node_id]
    return shape_dict, dtype_dict

def generate_tensor_data(shape, dtype):
    """
    Generate data to produce a tensor of given shape and dtype for TVM.
    """
    if "int8" in dtype:
        tensor = np.random.randint(128, size=shape, dtype=dtype)
    else:
        tensor = np.random.uniform(-1, 1, size=shape).astype(dtype)
    return tensor

def generate_pytorch_data(input_shapes, target_string, dtype):
    """
    Generate data to produce a tensor of given shape and dtype for PyTorch.
    """
    data = []
    for input_shape in input_shapes:
        if target_string == "cuda":
            if dtype == "int":
                data.append(torch.randint(0,10, input_shape[1], device="cuda"))
            else:
                data.append(torch.cuda.FloatTensor(input_shape[1]).normal_())
        else:
            data.append(torch.randn(input_shape[1]))
    return data

def make_inputs_dict(shape_dict, dtype_dict):
    """
    Make the inputs dictionary for a graph. Here, random input tensors
    for each input are produced.
    """
    inputs_dict = {}
    for input_name in shape_dict:
        shape = shape_dict[input_name]
        dtype = dtype_dict[input_name]
        data = generate_tensor_data(shape, dtype)
        inputs_dict[input_name] = data
    return inputs_dict

def extract_profile_data(times):
    """
    Provided a series of execution times calculate the mean, std, max and min.
    """
    mean_ts = np.mean(times)
    std_ts = np.std(times)
    max_ts = np.max(times)
    min_ts = np.min(times)

    header = "Execution time summary:\n{0:^10} {1:^10} {2:^10} {3:^10}".format(
        "mean (s)", "max (s)", "min (s)", "std (s)")
    stats = "{0:^10.5f} {1:^10.5f} {2:^10.5f} {3:^10.5f}".format(
        mean_ts, max_ts, min_ts, std_ts)

    return header, stats

def compile_model(mod, params, target_string, tuning_records=None, model_name=""):
    """
    Compile TVM model. Apply tuning records if they exist.
    """
    target = tvm.target.Target(target_string)
    target_host = "llvm"

    if tuning_records:
        if model_name:
            model_str = "_model=" + model_name + ".json"
            tuning_records = tuning_records[:-5]
            tuning_records += model_str
        if os.path.exists(tuning_records):
            print('applying tuning history...')
        with autotvm.apply_history_best(tuning_records):
            with tvm.transform.PassContext(opt_level=3):
                graph_module = relay.build(mod, target, params=params, target_host=target_host)
    else:
        with tvm.transform.PassContext(opt_level=3):
            graph_module = relay.build(mod, target, params=params, target_host=target_host)

    return graph_module.get_json(), graph_module.get_lib(), graph_module.get_params()

def run_model_tvm(graph, lib, params, run_settings, model_name, tuning_records=None):
    """
    Run TVM model. Apply tuning records if they exist.
    """
    profile = run_settings['profile']
    device = run_settings['device']
    repeat = run_settings['repeat']

    session = rpc.LocalSession()
    ctx = session.cpu() if device == "cpu" else session.gpu()
    is_tuned = True if tuning_records else False

    lib_name = "mod.so"
    temp = util.tempdir()
    lib_path = temp.relpath(lib_name)
    lib.export_library(lib_path)
    session.upload(lib_path)
    lib = session.load_module(lib_name)

    if profile:
        module = debug_runtime.create(graph, lib, ctx, dump_root=f"results/prof_{model_name}_tuned={is_tuned}")
    else:
        module = runtime.create(graph, lib, ctx)

    saved_params = relay.save_param_dict(params)
    module.load_params(saved_params)

    shape_dict, dtype_dict = get_input_info(graph, params)
    inputs_dict = make_inputs_dict(shape_dict, dtype_dict)
    module.set_input(**inputs_dict)

    if profile:
        module.run()

    timer = module.module.time_evaluator("run", ctx, 1, repeat=repeat)
    prof_result = timer()
    times = prof_result.results
    header, stats = extract_profile_data(times)

    filename = f'results/stat_table_{model_name}_tuned={is_tuned}'
    with open(filename, 'w') as f:
        print("%s\n%s\n" % (header, stats), filename, file=f)
    print("%s\n%s\n" % (header, stats))

def run_model_pytorch(trace, input_shapes, run_settings, model_name, target_string):
    """
    Run model using the standard pytorch runtime.
    """
    repeat = run_settings['repeat']

    # TODO extract datatype required from PyTorch model itself
    dtype = "int" if model_name == "bert" else "float"
    generated_data = generate_pytorch_data(input_shapes, target_string, dtype)

    # First run is always slower due to on-the-fly optimization, skip it.
    skip_first_run = True
    times = []
    for _ in range(repeat + 1):
        start = time.time()
        # Note: only a single input is currently supported
        trace(*generated_data)
        end = time.time() - start

        if skip_first_run:
            skip_first_run = False
            continue
        times.append(end)

    header, stats = extract_profile_data(times)

    filename = f'results/stat_table_{model_name}_pytorch'
    with open(filename, 'w') as f:
        print("%s\n%s\n" % (header, stats), filename, file=f)
    print("%s\n%s\n" % (header, stats))

def benchmark_models(data):
    """
    Compile and run a TVM model and benchmark it. Then run the same model using PyTorch
    and display statistics comparing the two.
    """
    target_string = data['target']
    tuning_records = data['autotuner_settings'].get('tuning_records') or ""
    run_settings = data['run_settings']

    for model in data['models']:
        for executor in ["tvm", "pytorch"]:
            trace, input_shapes = get_model(model['name'], model['type'])

            if executor == "tvm":
                mod, params = relay.frontend.from_pytorch(trace, input_shapes)

                print(f"Compiling TVM model {model['name']}")
                graph, lib, params = compile_model(mod, params, target_string, tuning_records, model['name'])

                print(f"Running TVM model {model['name']}")
                run_model_tvm(graph, lib, params, run_settings, model['name'], tuning_records)
            elif executor == "pytorch":
                run_model_pytorch(trace, input_shapes, run_settings, model['name'], target_string)
            else:
                raise ValueError(f"executor {executor} not supported.")


import scipy
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
import re
import string
import nltk
from nltk.tokenize import word_tokenize
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from nltk.stem import WordNetLemmatizer
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')

def remove_hyperlink(word):
    word = str(word)
    return re.sub(r"http\S+", "", word)

def to_lower(word):
    result = word.lower()
    return result

def remove_number(word):
    word = str(word)
    result = re.sub(r'\d+', '', word)
    return result

def remove_punctuation(word):
    result = word.translate(str.maketrans(dict.fromkeys(string.punctuation)))
    return result

def remove_whitespace(word):
    result = word.strip()
    return result

def replace_newline(word):
    return word.replace('\n', '')

def clean_up_pipeline(sentence):
    cleaning_utils = [remove_hyperlink,
                      replace_newline,
                      to_lower,
                      remove_number,
                      remove_punctuation, remove_whitespace]
    for o in cleaning_utils:
        sentence = o(sentence)
    return sentence

stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()

def remove_stop_words(words):
    result = [i for i in words if i not in stopwords.words()]
    return result

def word_stemmer(words):
    return [stemmer.stem(o) for o in words]

def word_lemmatizer(words):
    return [lemmatizer.lemmatize(o) for o in words]

def clean_token_pipeline(words):
    cleaning_utils = [remove_stop_words, word_lemmatizer]
    for o in cleaning_utils:
        words = o(words)
    return words

def convert_to_feature(raw_tokenize_data):
    raw_sentences = [' '.join(o) for o in raw_tokenize_data]
    return vectorizer.transform(raw_sentences)

def data_prep(x_test):
    x_test = [word_tokenize(x_test)]
    x_test = [clean_token_pipeline(x_test)]
    x_test = [" ".join(x_test)]
    x_test = [x_test.split(" ")]
    print(x_test)
    vectorizer = TfidfVectorizer()
    raw_sentences = [" ".join(x_test)]
    vectorizer.fit(raw_sentences)

    def convert_to_feature(raw_tokenize_data):
        raw_sentences = [' '.join(raw_tokenize_data)]
        return vectorizer.transform(raw_sentences)

    vectorizer = CountVectorizer()
    raw_sentences = [" ".join(x_test)]
    vectorizer.fit(raw_sentences)
    x_test_features = convert_to_feature(x_test)

    return x_test_features


from copy import copy
import json
import logging

log = logging.getLogger(__name__)

def inference_handler(state, video, inference, stats):
    while state.run:
        state.current_frame = video.get_frame(inference, stats)
        state.new_frame = True

def io_handler(state, stats, socketio):
    previous_stats = copy(stats)
    while state.run:
        if (stats != previous_stats):
            log.info('NEW STATS: %s', json.dumps(stats.__dict__))
            socketio.emit('stats', json.dumps(stats.__dict__))
            previous_stats = copy(stats)


# This is a sample Python script.
import queue
import random
import sys
import threading
import time
import tkinter
from tkinter.messagebox import showinfo

import pyautogui

from voice_command_loader import get_voice_command

text_to_command_queue_ = queue.Queue()
movement_direction = None
movements = {'up': [-1, 0], 'down': [1, 0], 'left': [0, -1], 'right': [0, 1],
             'up left': [-1, -1], 'up right': [-1, 1], 'down left': [1, -1], 'down right': [1, 1],
             'stop': [0, 0]}
clicks = {'click': [1, 'left'], 'double click': [2, 'left'],
          'right click': [1, 'right'], 'right double click': [2, 'right'],
          'middle click': [1, 'middle']}

type_text = None

click_count = None
click_button = None
screen_w, screen_h = pyautogui.size()
lock = threading.Lock()

def do_movement():
    global movement_direction
    while True:
        if movement_direction is not None:
            pyautogui.move(movement_direction[1], movement_direction[0])

def check_stop_typing():
    """
    :return:
    """
    root = tkinter.Tk()
    root.withdraw()
    showinfo(title='confirmation',
             message='Are you sure that you want to stop typing? Say Yes or No')
    return root
    # root.destroy()

def click():
    global click_count
    global click_button
    if click_count is not None and click_button is not None:
        print('i am doing some clicking')
        pyautogui.click(clicks=click_count, button=click_button)
        click_count = None
        click_button = None

def set_variable(text_to_command_queue):
    global movement_direction
    global click_count
    global click_button
    while True:
        if not text_to_command_queue.empty():
            command = text_to_command_queue.get()
            if command in movements:
                print(f'Moving cursor:\n{command}')
                movement_direction = movements[command]
            elif command in clicks:
                click_count = clicks[command][0]
                click_button = clicks[command][1]
                print(f'Click:\n{command}')
                click()
            elif command == 'type':
                print('Typing')
                while True:
                    if not text_to_command_queue.empty():
                        typed_text = text_to_command_queue.get()
                        if typed_text == 'stop typing':
                            break
                        pyautogui.typewrite(text_to_command_queue.get() + ' ')
            else:
                print(f'Unknown command: {command}')
        # time.sleep(5)

if __name__ == '__main__':
    # Create the threads
    thread_set_var = threading.Thread(target=set_variable, daemon=True, args=[text_to_command_queue_])
    thread_do_movement = threading.Thread(target=do_movement, daemon=True)
    # thread_click = threading.Thread(target=click, daemon=True)
    thread_get_voice_command = threading.Thread(target=get_voice_command, daemon=True, args=[text_to_command_queue_])

    # Start the threads
    thread_set_var.start()
    thread_do_movement.start()
    thread_get_voice_command.start()

    # Wait for the threads to finish
    thread_set_var.join()
    thread_do_movement.join()
    thread_get_voice_command.join()


import requests
from bs4 import BeautifulSoup
import re

news_main_page = "https://www.frankfurt-university.de/de/aktuelles/"
url = "https://www.frankfurt-university.de/de/newsmodule/details/mit-projekt-zero-waste-unter-bestplatzierten-beim-data-mining-cup-2023/"

def write_string_to_file(filename, text):
    with open(filename, "w") as file:
        file.write(text)

def remove_special_characters(text):
    return re.sub("[^a-zA-Z0-9 ]", "", text)

def get_all_page_links(url):
    response = requests.get(url)
    content = response.content
    soup = BeautifulSoup(content, "html.parser")

    page2_link = (
        "https://www.frankfurt-university.de"
        + soup.find("a", attrs={"title": "Zu Seite 2"})["href"]
    )
    page3_link = (
        "https://www.frankfurt-university.de"
        + soup.find("a", attrs={"title": "Zu Seite 3"})["href"]
    )

    return [url, page2_link, page3_link]

def get_all_news_links(urls):
    article_links = []

    for url in urls:
        response = requests.get(url)

        if response.status_code == 200:
            content = response.content

            soup = BeautifulSoup(content, "html.parser")

            # get article elementson this page
            article_div = soup.find("div", attrs={"class": "news-simple-list"})
            articles = article_div.find_all("article")

            for article in articles:
                # get article link
                arcticle_link = (
                    article.find("div", attrs={"class": "news-simple-list__text"})
                    .find("div", attrs={"class": "news-simple-list__header"})
                    .find("a", attrs={"class": "news-article-header__link"})
                )

                # add link to article_links list
                article_links.append(
                    "https://www.frankfurt-university.de" + arcticle_link["href"]
                )
        else:
            print("Failed to retrieve webpage.")

    return article_links

def scrape_website_and_save_document_to_file(url):
    response = requests.get(url)

    # Check the status of the request
    if response.status_code == 200:
        content = response.content

        soup = BeautifulSoup(content, "html.parser")

        # 1. headline
        headline = (
            soup.find("div", attrs={"class": "news-article-header"})
            .find_all("h1")[0]
            .text
        )

        # 2. content
        content_div = soup.find("div", attrs={"class": "news-text-wrap"})
        # Get all text within this div
        content_div_text = content_div.get_text()

        # 3. image captions
        try:
            image_captions_text = ""
            image_div = soup.find("div", attrs={"class": "news-img-wrap"})

            for image_caption in image_div.find_all("img"):
                image_captions_text += "\n" + image_caption.get("data-caption")
        except:
            image_captions_text = ""
            print(f"Article has no images: {url}")

        # deprecated
        # h2_elements = content_div.find_all("h2")
        # p_elements = content_div.find_all("p")
        # list_imtems = content_div.find_all("li")
        # text_elements = headline + h2_elements + p_elements
        # for text in text_elements:
        #     if len(document) == 0:
        #         document += text.text
        #     else:
        #         document += "\n" + text.text

        document = headline + "\n" + content_div_text + image_captions_text

        write_string_to_file(
            "documents/"
            + remove_special_characters(document.split("\n", 1)[0])
            + ".txt",
            document,
        )

    else:
        print("Failed to retrieve webpage.")

page_links = get_all_page_links(news_main_page)
news_links = get_all_news_links(page_links)

for news_link in news_links:
    scrape_website_and_save_document_to_file(news_link)


import logging
import traceback

logger = logging.getLogger('django')
ERROR_CODE = {
    0: '成功',
    1: '记录已存在',
    2: '没有查询到指定记录',
    3: '需要登录',
    4: '参数无效',
    5: '用户未创建商店',
    6: '账号或密码无效',
    7: '无操作权限',
    8: '登陆过期',
    9: '名称已存在',
    10: '需指定父分类',
    11: '商品无库存',
    -1: '稍后再试'
}

def base_code_response():
    # msg = ERROR_CODE.get(code)
    # logger.warning(msg)
    return {
        # 'code': code,
        # 'msg': msg,
        'data': {}
    }

def default_response():
    return {
        'code': 0,
        'msg': 'success',
        'data': {}
    }
def invalid_parameters_response(e):
    code = 4
    msg = ERROR_CODE.get(code) + ': ' + str(e.get_codes())
    logger.warning('traceback.format_exc():%s' % traceback.format_exc())

    return {
        'code': code,
        'msg': msg,
        'data': {}
    }
def need_login_response():
    code = 3
    msg = ERROR_CODE.get(code)
    logger.warning(msg)
    return {
        'code': code,
        'msg': msg,
        'data': {}
    }


import cv2
import numpy as np
import datetime
from PIL import ImageFont, ImageDraw, Image
from numpy.core.records import fromrecords
from numpy.lib.type_check import isreal

# 카메라 기본 틀
cap = cv2.VideoCapture(0) # 번호 0부터
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)

# 나눔고딕볼드 로드
font = ImageFont.truetype('./Fonts/NanumGothicBold.ttf', 20)

# 영상 코덱 설정
fourcc = cv2.VideoWriter_fourcc(*'XVID')
is_record = False

# 무한루프 지정 키를 눌렀을 때
while True:
    ret, frame = cap.read() # 카메라 현재 영상 로드, frame에 저장, ret Ture/False
    frame = cv2.flip(frame, 0)

    h, w, _ = frame.shape # channel은 불필요
    now = datetime.datetime.now()
    currDateTime = now.strftime('%Y-%m-%d %H:%M:%S')
    fileDateTime = now.strftime('%Y%m%d_%H%M%S') # 20210720_164725  캡쳐되는 파일 이름

    if ret != True: break # ret이 False면 루프탈출

    frame = Image.fromarray(frame)
    draw = ImageDraw.Draw(frame)
    draw.text(xy=((w-300), 10), text="겸's_TV - {0}".format(currDateTime), font=font, fill=(0, 0, 255))
    frame = np.array(frame) # 웹상태 복귀

    #cv2.imshow('RealTime cam', frame)   # 로드한 영상을 창에 띄움
    cv2.imshow('Gray Result', frame)

    key = cv2.waitKey(1)
    if key == ord('q'): break# q를 입력하면 루프탈출
    elif key == ord('c'): #c는 화면캡쳐
        cv2.imwrite('./capture/img_{0}.png'.format(fileDateTime), frame)
        print('이미지 저장 완료')
    elif key == ord('r'): # 레코드 시작
        is_record = True
        video = cv2.VideoWriter('./capture/record_{0}.avi'.format(fileDateTime), fourcc, 20, (w, h))
        print('녹화 시작')
    elif key == ord('t'): # 레코드 종료
        is_record = False
        video.release() # 객체 해제
        print("녹화 완료")

    if is_record:
        video.write(frame)
        cv2.circle(img=frame, center=(620, 15), radius=5, color=(0, 0, 255), thickness=3)
cap.realse() # 웹캠해제
cv2.destroyAllWindows()


#!/usr/bin/python3
# -*- coding: utf-8 -*-

class Grid(list):
  __sand_start = [500, 0]
  __possible_moves = [[0,1], [-1,1], [1,1]]

  def __init__(self, rocks, has_floor = False):
    self.start, self.end = Grid.__find_dimensions(rocks)
    if has_floor:
      height = self.end[1] - self.start[1] + 2
      self.start = [500 - height, 0]
      self.end = [500 + height, height]

    width = self.end[0] - self.start[0] + 1
    self.grid = [['.' for x in range(width)] for y in range(self.end[1] + 1)]

    for r in rocks:
      p = r[0]

      for c in r[1:]:
        for point in line(p, c):
          self[point] = '#'
        p = c

    if has_floor:
        for point in line([self.start[0], self.end[1]], self.end):
          self[point] = '#'

    self[Grid.__sand_start] = '+'

  def is_in_grid(self, p):
    return p[0] >= self.start[0] and p[0] <= self.end[0] and p[1] <= self.end[1]

  def next_sand(self):
    p = list(Grid.__sand_start)
    if self[p] == 'o':
      return False

    while True:
      can_move = False

      for m in Grid.__possible_moves:
        t = move(p, m)
        if not self.is_in_grid(t):
          return False

        if self[t] == '.':
          p = t
          can_move = True
          break

      if not can_move:
        self[p] = 'o'
        return True

  def __getitem__(self, p):
    return self.grid[p[1]][p[0] - self.start[0]]

  def __setitem__(self, p, v):
    self.grid[p[1]][p[0] - self.start[0]] = v

  @staticmethod
  def __find_dimensions(rocks):
    start = list(Grid.__sand_start)
    end = list(Grid.__sand_start)

    for r in rocks:
      for x,y in r:
        if x < start[0]:
          start[0] = x
        if x > end[0]:
          end[0] = x
        if y < start[1]:
          start[1] = y
        if y > end[1]:
          end[1] = y

    return [start, end]

  def __str__(self):
    return '\n'.join([''.join(x) for x in self.grid])

def read_input() -> str:
  with open('input.txt') as file:
    return file.read()

def parse(data):
  return [[int(x) for x in pair.split(',')] for pair in data.split(' -> ')]

def move(p, o):
  return [p[0] + o[0], p[1] + o[1]]

def line(start, end):
  offset = [0,0]
  if end[0] > start[0]:
    offset[0] = 1
  if end[0] < start[0]:
    offset[0] = -1
  if end[1] > start[1]:
    offset[1] = 1
  if end[1] < start[1]:
    offset[1] = -1

  points = []
  p = start
  while p != end:
    points.append(p)
    p = move(p, offset)
  points.append(p)

  return points

def main(report_result):
  rocks = [parse(x) for x in read_input().split('\n') if x != '']

  grid = Grid(rocks)

  number_of_sand_grains = 0
  while grid.next_sand():
    number_of_sand_grains += 1
  report_result('Number of sand grains:', number_of_sand_grains)

  grid = Grid(rocks, True)

  number_of_sand_grains = 0
  while grid.next_sand():
    number_of_sand_grains += 1
  report_result('Number of sand grains with floor:', number_of_sand_grains)

if __name__ == '__main__':
  main(print)


import collections
import csv
import glob
import os.path
import pickle

import cv2
import numpy as np

def prepare_im(path):
    # loads the image from a path and converts it to greyscale
    img = cv2.imread(path)
    grey = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    grey = cv2.rotate(grey, cv2.ROTATE_90_CLOCKWISE)
    grey = grey.astype(np.float64)
    return grey

def does_have_xandy(pixel_sample):
    has_front = False
    has_side = False

    for angle_name in pixel_sample:
        if "front" in angle_name or "back" in angle_name:
            has_front = True
        elif "left" in angle_name or "right" in angle_name:
            has_side = True

    return has_front and has_side

def get_brightest_front_side(brights):
    front, f_val = None, 0
    side, s_val = None, 0
    for angle_name in brights:
        if "front" in angle_name or "back" in angle_name:
            if brights[angle_name] > f_val:
                f_val = brights[angle_name]
                front = angle_name
        elif "left" in angle_name or "right" in angle_name:
            if brights[angle_name] > s_val:
                s_val = brights[angle_name]
                side = angle_name
    return [front, side]

if __name__ == "__main__":
    # so long as the brightness is this many units greater than the background, we detected a light
    threshold = 50

    pixels_samples = collections.defaultdict(dict)
    pixels_samples_brightness = collections.defaultdict(dict)

    sides = ["front1", "left1", "back1", "right1"]
    for side in sides:
        # load the background image
        background = prepare_im(f"./data/{side}/background.jpg")

        # for each of the images we have in data
        for img_path in glob.glob(f"./data/{side}/*.jpg"):
            light_im = prepare_im(img_path)

            # remove the background so that we are looking at the change in lighting conditions
            diff = np.clip(light_im - background, 0, None)

            # eroding to remove reflections
            kernel = np.ones((6, 6), np.uint8)
            diff = cv2.erode(diff, kernel, cv2.BORDER_REFLECT)

            #  threshold out any change below our defined threshold
            threshed = np.zeros_like(diff, dtype=np.uint8)
            threshed[diff > threshold] = diff[diff > threshold]

            # get the location of the maximum brightness in the image
            (minVal, maxVal, minLoc, maxLoc) = cv2.minMaxLoc(threshed)

            # if we detected a change then
            if maxVal > 0:
                filename = os.path.basename((img_path))
                pixel_id = int(filename.replace(".jpg", ""))

                pixels_samples[pixel_id][side]= [maxLoc[0], maxLoc[1]]
                pixels_samples_brightness[pixel_id][side]= maxVal

            # show the images and the detections
            image = light_im.copy().astype(np.uint8)
            image = np.dstack([image, image, image])
            cv2.circle(image, maxLoc, 10, (255, 0, 0), 2)
            cv2.imshow("im", image)
            cv2.imshow("diff", diff.astype(np.uint8))
            cv2.waitKey(1)

    # for each angle we imaged from, normalize the range of the locations
    for side in sides:
        locations = []
        for pixel_id in pixels_samples:
            xy_loc = pixels_samples[pixel_id].get(side, None)
            if xy_loc:
                locations.append(xy_loc)

        min_pos = np.min(locations, axis=0)
        max_pos = np.max(locations, axis=0)

        for pixel_id in pixels_samples:
            xy_loc = pixels_samples[pixel_id].get(side, None)
            if xy_loc:
                # normalize the coordinate
                xy_loc = (xy_loc - min_pos) / (max_pos - min_pos)

                # flip the coordinates if we are in the back or the right hand side
                if "back" in side or "right" in side:
                    xy_loc[0] = 1 - xy_loc[0]

                #flip vertically because the top of an image is 0, and we want the bottom of the tree to be zero
                xy_loc[1] = 1 - xy_loc[1]

                # rescale the coordinate to the dimensions of the tree
                xy_loc[0] *= 100
                xy_loc[1] *= 300

                pixels_samples[pixel_id][side] = xy_loc

    pixel_coords = []
    for pixel_id in pixels_samples:
        if does_have_xandy(pixels_samples[pixel_id]):
            best_samples = get_brightest_front_side(pixels_samples_brightness[pixel_id])

            # get all X samples
            X_samples = [pixels_samples[pixel_id][s][0] for s in best_samples if "front" in s or "back" in s]
            x = np.array(X_samples).mean()

            Y_samples = [pixels_samples[pixel_id][s][0] for s in best_samples if "left" in s or "right" in s]
            y = np.array(Y_samples).mean()

            Z_samples = [pixels_samples[pixel_id][s][1] for s in [best_samples[0],]]
            z = np.array(Z_samples).mean()

            pixel_coords.append((pixel_id, x, y, z))

    with open("./data/coords.csv", "w", newline="\n") as f:
        writer = csv.writer(f)
        for row in pixel_coords:
            writer.writerow(row)


import credentials
import logging
from tweepy import OAuthHandler, Stream
from tweepy.streaming import StreamListener
from pymongo import MongoClient

def authenticate():
    """Function for handling Twitter Authentication. Please note
       that this script assumes you have a file called credentials.py
       which stores the 4 required authentication tokens:

       1. API_KEY
       2. API_SECRET
       3. ACCESS_TOKEN
       4. ACCESS_TOKEN_SECRET

    See course material for instructions on getting your own Twitter credentials.
    """
    auth = OAuthHandler(credentials.API_KEY, credentials.API_SECRET)
    auth.set_access_token(credentials.ACCESS_TOKEN, credentials.ACCESS_TOKEN_SECRET)
    return auth

class MaxTweetsListener(StreamListener):

    def __init__(self, max_tweets, *args, **kwargs):
        # initialize the StreamListener
        super().__init__(*args, **kwargs)
        # set the instance attributes
        self.max_tweets = max_tweets
        self.counter = 0
        # instanciate mongo database
        self.client = MongoClient('mongodb')
        self.db = self.client['twitter']
        if self.db.list_collection_names() == []:
            self.db.create_collection('tweets')
        # make sure this works
        logging.critical('... created with mongodb connection')

    def on_connect(self):
        logging.critical(f'... listening for incoming {self.max_tweets} tweets:')

    def on_status(self, status):
        # extract information from status
        tweet = {
            'text': status.text,
            'username': status.user.screen_name,
            'followers_count': status.user.followers_count
        }

        # decide what to do here:
        # self.print_tweet(tweet)
        self.add_to_db(tweet)

        # keep track of the counter, eventually terminate
        self.counter += 1
        if self.max_tweets == self.counter:
            self.counter = 0
            logging.critical('... all tweets collected, listener end')
            return False

    def on_error(self, status):
        if status == 420:
            logging.critical('... Rate limit applies. Stop the stream.')
            return False

    def print_tweet(self, tweet):
        print(f'({self.counter}) {tweet["username"]}: {tweet["text"]}')

    def add_to_db(self, tweet):
        self.db.tweets.insert_one(tweet)
        logging.critical(f'{self.counter} tweet -> mongodb')

if __name__ == '__main__':
    auth = authenticate()
    listener = MaxTweetsListener(max_tweets=1000)
    stream = Stream(auth, listener)
    stream.filter(track=['ai'], languages=['en'], is_async=False)


import argschema
import pandas as pd
import numpy as np
from functools import partial
from PIL import Image
from pathlib import Path
from ophys_etl.schemas.fields import H5InputFile
from ophys_etl.utils.array_utils import normalize_array
from ophys_etl.utils.video_utils import downsample_h5_video, encode_video

import matplotlib as mpl
mpl.use('Agg')
from matplotlib import pyplot as plt  # noqa: E402

class RegistrationQCException(Exception):
    pass

class RegistrationQCInputSchema(argschema.ArgSchema):
    movie_frame_rate_hz = argschema.fields.Float(
        required=True,
        description="frame rate of movie, usually 31Hz or 11Hz")
    preview_frame_bin_seconds = argschema.fields.Float(
        required=False,
        default=2.0,
        description=("before creating the webm, the movies will be "
                     "aveaged into bins of this many seconds."))
    preview_playback_factor = argschema.fields.Float(
        required=False,
        default=10.0,
        description=("the preview movie will playback at this factor "
                     "times real-time."))
    uncorrected_path = H5InputFile(
        required=True,
        description=("path to uncorrected original movie."))
    motion_corrected_output = H5InputFile(
        required=True,
        description=("path to motion corrected movie."))
    motion_diagnostics_output = argschema.fields.InputFile(
        required=True,
        description=("Saved path for *.csv file containing motion "
                     "correction offset data")
    )
    max_projection_output = argschema.fields.InputFile(
        required=True,
        description=("Saved path for *.png of the max projection of the "
                     "motion corrected video."))
    avg_projection_output = argschema.fields.InputFile(
        required=True,
        description=("Saved path for *.png of the avg projection of the "
                     "motion corrected video."))
    registration_summary_output = argschema.fields.OutputFile(
        required=True,
        description=("Desired path for *.png summary plot."))
    motion_correction_preview_output = argschema.fields.OutputFile(
        required=True,
        description="Desired path for *.webm motion preview")
    movie_lower_quantile = argschema.fields.Float(
        required=False,
        default=0.1,
        description=("lower quantile threshold for avg projection "
                     "histogram adjustment of movie"))
    movie_upper_quantile = argschema.fields.Float(
        required=False,
        default=0.999,
        description=("upper quantile threshold for avg projection "
                     "histogram adjustment of movie"))

def make_png(max_proj_path: Path, avg_proj_path: Path,
             summary_df: pd.DataFrame, dst_path: Path):
    """
    """
    xo = np.abs(summary_df['x']).max()
    yo = np.abs(summary_df['y']).max()

    fig = plt.figure(figsize=(16, 10))
    gs = fig.add_gridspec(4, 4)
    mx_ax = fig.add_subplot(gs[0:2, 0:2])
    av_ax = fig.add_subplot(gs[0:2, 2:4])
    xyax = fig.add_subplot(gs[2, :])
    corrax = fig.add_subplot(gs[3, :])

    for ax, im_path in zip([mx_ax, av_ax], [max_proj_path, avg_proj_path]):
        with Image.open(im_path) as im:
            ax.imshow(im, cmap='gray')
            sz = im.size
        ax.axvline(xo, color='r', linestyle='--')
        ax.axvline(sz[0] - xo, color='r', linestyle='--')
        ax.axhline(yo, color='g', linestyle='--')
        ax.axhline(sz[1] - yo, color='g', linestyle='--')
        ax.set_title(f"{im_path.parent}\n{im_path.name}", fontsize=8)

    xyax.plot(summary_df['x'], linewidth=0.5, color='r', label='xoff')
    xyax.axhline(xo, color='r', linestyle='--')
    xyax.axhline(-xo, color='r', linestyle='--')
    xyax.plot(summary_df['y'], color='g', linewidth=0.5, alpha=0.5,
              label='yoff')
    xyax.axhline(yo, color='g', linestyle='--')
    xyax.axhline(-yo, color='g', linestyle='--')
    xyax.legend(loc=0)
    xyax.set_ylabel("correction offset [pixels]")

    corrax.plot(summary_df['correlation'], color='k', linewidth=0.5,
                label='corrXY')
    corrax.set_xlabel("frame index")
    corrax.set_ylabel("correlation peak value")
    corrax.legend(loc=0)
    fig.tight_layout()
    fig.savefig(dst_path)

    return dst_path

def downsample_normalize(movie_path: Path, frame_rate: float,
                         bin_size: float, lower_quantile: float,
                         upper_quantile: float) -> np.ndarray:
    """reads in a movie (nframes x nrows x ncols), downsamples,
    creates an average projection, and normalizes according to
    quantiles in that projection.

    Parameters
    ----------
    movie_path: Path
        path to an h5 file, containing an (nframes x nrows x ncol) dataset
        named 'data'
    frame_rate: float
        frame rate of the movie specified by 'movie_path'
    bin_size: float
        desired duration in seconds of a downsampled bin, i.e. the reciprocal
        of the desired downsampled frame rate.
    lower_quantile: float
        arg supplied to `np.quantile()` to determine lower cutoff value from
        avg projection for normalization.
    upper_quantile: float
        arg supplied to `np.quantile()` to determine upper cutoff value from
        avg projection for normalization.

    Returns
    -------
    ds: np.ndarray
        a downsampled and normalized array

    Notes
    -----
    This strategy was satisfactory in the labeling app for maintaining
    consistent visibility.

    """
    ds = downsample_h5_video(
            movie_path,
            input_fps=frame_rate,
            output_fps=1.0 / bin_size)
    avg_projection = ds.mean(axis=0)
    lower_cutoff, upper_cutoff = np.quantile(
                avg_projection.flatten(), (lower_quantile, upper_quantile))
    ds = normalize_array(ds,
                         lower_cutoff=lower_cutoff,
                         upper_cutoff=upper_cutoff)
    return ds

class RegistrationQC(argschema.ArgSchemaParser):
    default_schema = RegistrationQCInputSchema

    def run(self):
        self.logger.name = type(self).__name__
        self.logger.setLevel(self.args['log_level'])

        # create and write the summary png
        motion_offset_df = pd.read_csv(self.args['motion_diagnostics_output'])
        png_out_path = make_png(Path(self.args['max_projection_output']),
                                Path(self.args['avg_projection_output']),
                                motion_offset_df,
                                Path(self.args['registration_summary_output']))
        self.logger.info(f"wrote {png_out_path}")

        # downsample and normalize the input movies
        ds_partial = partial(downsample_normalize,
                             frame_rate=self.args['movie_frame_rate_hz'],
                             bin_size=self.args['preview_frame_bin_seconds'],
                             lower_quantile=self.args['movie_lower_quantile'],
                             upper_quantile=self.args['movie_upper_quantile'])
        processed_vids = [ds_partial(i)
                          for i in [
                              Path(self.args['uncorrected_path']),
                              Path(self.args['motion_corrected_output'])]]
        self.logger.info("finished downsampling motion corrected "
                         "and non-motion corrected movies")

        # tile into 1 movie, raw on left, motion corrected on right
        tiled_vids = np.block(processed_vids)

        # make into a viewable artifact
        playback_fps = (self.args['preview_playback_factor'] /
                        self.args['preview_frame_bin_seconds'])
        encode_video(tiled_vids,
                     self.args['motion_correction_preview_output'],
                     playback_fps)
        self.logger.info("wrote "
                         f"{self.args['motion_correction_preview_output']}")

        return

if __name__ == "__main__":
    rqc = RegistrationQC()
    rqc.run()


class PlugBoard(object):

    board = {
        'a': 'a',
        'b': 'b',
        'c': 'c',
        'd': 'd',
        'e': 'e',
        'f': 'f',
        'g': 'g',
        'h': 'h',
        'i': 'i',
        'j': 'j',
        'k': 'k',
        'l': 'l',
        'm': 'm',
        'n': 'n',
        'o': 'o',
        'p': 'p',
        'q': 'q',
        'r': 'r',
        's': 's',
        't': 't',
        'u': 'u',
        'v': 'v',
        'w': 'w',
        'x': 'x',
        'y': 'y',
        'z': 'z',
    }

    def __init__(self, positions: dict = {}):
        if self.validate(positions):
            self.board = self.buildBoard(self.board, positions)

    def validate(self, positions: dict) -> bool:
        usedPairs = {}

        for k, v in positions.items():
            if k in usedPairs.keys() or v in usedPairs.keys():
                raise Exception('Unable to set {} <-> {}'.format(k, v))

            usedPairs.update({k: v})

        return True

    def buildBoard(self, board: dict, positions: dict) -> dict:
        for k, v in positions.items():
            if k in board.keys() and v in board.keys():
                board.update({k: v})
                board.update({v: k})

        return board

    def parse(self, key: str) -> str:
        return self.board.get(key)


from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware

from Globalizer.predict import k_clustering, radius_clustering, smart_clustering
app = FastAPI()

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],  # Allows all origins
    allow_credentials=True,
    allow_methods=["*"],  # Allows all methods
    allow_headers=["*"],  # Allows all headers
)

@app.get("/")
def index():
    return {"ok": True}

@app.get("/predict")
def predict(country, threshold=None, n_centers=None, radius=None, percent=None):
    '''This function controls whether
    we use the Kmeans or the KmeansMinibatch. By default it's
    the Minibatch with 50km as threshold'''
    country = country.split(",")
    if n_centers != None:
        n_centers= int(n_centers)
        return k_clustering(country, n_centers)
    if threshold != None:
        threshold= float(threshold)
        return smart_clustering(country, threshold)
    if radius != None and percent != None:
        radius = int(radius)
        percent = int(percent)
        return radius_clustering(country, radius, percent)

## Check ##
# to check your code, run:
# python -m api.fast
if __name__=="__main__":
    #print(predict(country = "ABW", n_centers = 5))
    print(predict(country = "AUT", radius=60, percent=80))


from _warnings import warn

from comet_ml import API
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
sns.set(style="darkgrid")

api = API()
reported_metrics = ["Reward Mean", "Episodic Success Rate", "Episode Length Mean "]

def build_plot_dict(orig_env_name, raw_tuples, final_data, log_ts):
    for i in range(len(raw_tuples)):

        env_name = orig_env_name + ' - ' + reported_metrics[i]
        temp_data = np.array([list(x) for x in raw_tuples[i]]).transpose()

        # Preprocess for equal log intervals
        data = np.zeros((temp_data.shape[0], len(log_ts)))
        data[0, :] = log_ts
        for j in range(len(log_ts)):
            index = np.where(temp_data[0] == log_ts[j])[0]
            if index.size == 0:
                continue
            data[:, j] = temp_data[:, index[0]]

        # If same experiment with different seed
        if env_name in final_data:
            final_data[env_name]['n'] += 1
            data_concat = np.vstack((final_data[env_name]['data0'], data[1, :]))
            final_data[env_name]['data0'] = data_concat

        # If first experiment with these hyperparams.
        else:
            final_data[env_name] = {'metric': reported_metrics[i], 'data0': data, 'n': 1}

    return final_data

def running_mean(x, N):
    cumsum = np.cumsum(np.insert(x, 0, 0))
    return (cumsum[N:] - cumsum[:-N]) / float(N)

gps_exp_ids = {
    "SEVN-Test-NoisyGPS-1-v1-s0-10p": "mweiss17/navi-corl-2019/2d3670a8af1e4c9d83f06a889e02862e",
    "SEVN-Test-NoisyGPS-1-v1-s1-10p": "mweiss17/navi-corl-2019/604adedd9fdd4f0bb18fff5d76a4400f",
    "SEVN-Test-NoisyGPS-1-v1-s2-10p": "mweiss17/navi-corl-2019/76078c37a6934a24bb644bda65fcff0e",
    "SEVN-Test-NoisyGPS-5-v1-s0-10p": "mweiss17/navi-corl-2019/60fb0b41644c4ba89949eb03ada53948",
    "SEVN-Test-NoisyGPS-5-v1-s1-10p": "mweiss17/navi-corl-2019/a887d1de91c0463e94985441bbf8b126",
    "SEVN-Test-NoisyGPS-5-v1-s2-10p": "mweiss17/navi-corl-2019/76078c37a6934a24bb644bda65fcff0e",
    "SEVN-Test-NoisyGPS-25-v1-s0-10p": "mweiss17/navi-corl-2019/281db6264d9d46fb8b0ada97ecd19903",
    "SEVN-Test-NoisyGPS-25-v1-s1-10p": "mweiss17/navi-corl-2019/f4f8901bd4944340bd8beb2c203ccca9",
    "SEVN-Test-NoisyGPS-25-v1-s2-10p": "mweiss17/navi-corl-2019/69a88388d483456c9174453956414f97",
    "SEVN-Test-NoisyGPS-100-v1-s0-10p": "mweiss17/navi-corl-2019/5824c0711976465cb1c2363434246961",
    "SEVN-Test-NoisyGPS-100-v1-s1-10p": "mweiss17/navi-corl-2019/79e37ee59d0e41b094eafb5a70f8df23",
    "SEVN-Test-NoisyGPS-100-v1-s2-10p": "mweiss17/navi-corl-2019/cb839ab2d3444f3aaadfb8497c7604ad",
}

plot_info = {
    "SEVN-Test-NoisyGPS-1": {'color': '#404387', 'plot_name': 'NoisyGPS-1'},
    "SEVN-Test-NoisyGPS-5": {'color': '#22a784', 'plot_name': 'NoisyGPS-5'},
    "SEVN-Test-NoisyGPS-25": {'color': '#fde724', 'plot_name': 'NoisyGPS-25'},
    "SEVN-Test-NoisyGPS-100": {'color': '#29788e', 'plot_name': 'NoisyGPS-100'},
}

gps_exp_data = {}
min_numframes = 0000000
max_numframes = 100000000
for name, exp_id in gps_exp_ids.items():
    gps_exp_data[name] = {}
    experiment = api.get(exp_id)

    reward_mean = experiment.metrics_raw[reported_metrics[0]]
    reward_arr = np.array(reward_mean).transpose()
    reward_arr = reward_arr[:, np.where(np.logical_and(reward_arr[0] >= min_numframes,
                                                       reward_arr[0] < max_numframes))[0]]
    gps_exp_data[name][reported_metrics[0]] = {
        'mean': np.mean(reward_arr[1]),
        'std': np.std(reward_arr[1]),
        'raw_data': reward_arr
    }

    episodic_success_rate = experiment.metrics_raw[reported_metrics[1]]
    ep_succes_rate_arr = np.array(episodic_success_rate).transpose()
    ep_succes_rate_arr = ep_succes_rate_arr[:, np.where(np.logical_and(ep_succes_rate_arr[0] >= min_numframes,
                                                                       ep_succes_rate_arr[0] < max_numframes))[0]]
    gps_exp_data[name][reported_metrics[1]] = {
        'mean': np.mean(ep_succes_rate_arr[1]),
        'std': np.std(ep_succes_rate_arr[1]),
        'raw_data': ep_succes_rate_arr
    }

    episode_length_mean = experiment.metrics_raw[reported_metrics[2]]
    ep_length_mean_arr = np.array(episode_length_mean).transpose()
    ep_length_mean_arr = ep_length_mean_arr[:, np.where(np.logical_and(ep_length_mean_arr[0] >= min_numframes,
                                                                       ep_length_mean_arr[0] < max_numframes))[0]]
    gps_exp_data[name][reported_metrics[2]] = {
        'mean': np.mean(ep_length_mean_arr[1]),
        'std': np.std(ep_length_mean_arr[1]),
        'raw_data': ep_length_mean_arr
    }

from collections import defaultdict
exps = defaultdict(list)
for name, exp_id in gps_exp_ids.items():
    exp = "-".join(name.split("-")[:4])
    if exp in name:
        exps[exp].append(gps_exp_data[name])

compiled_data = []
for name, data in exps.items():
    c_data = {'name': name}
    for metric in reported_metrics:
        try:
            trials = np.concatenate([np.expand_dims(data[0][metric]['raw_data'][1], axis=0), np.expand_dims(data[1][metric]['raw_data'][1], axis=0), np.expand_dims(data[2][metric]['raw_data'][1], axis=0)], axis=0)
        except Exception as e:
            continue
        mean = trials.mean(axis=0)
        std = trials.std(axis=0)
        c_data[metric] = np.concatenate([np.expand_dims(data[0][metric]['raw_data'][0], axis=0), np.expand_dims(mean, axis=0), np.expand_dims(std, axis=0)], axis=0)
    compiled_data.append(c_data)

for metric in reported_metrics:
    # fig = plt.figure()
    # plt.title(metric, fontsize=18)
    # plt.xlabel('Timesteps', fontsize=14)
    # plt.ylabel(metric, fontsize=14)
    for data in compiled_data:
        color = plot_info[data['name']]['color']
        label = plot_info[data['name']]['plot_name']
        try:
            import pdb; pdb.set_trace()
            to_plot = pd.DataFrame(data[metric].T, columns=['timestep', 'mean', 'std'])
            sns.lineplot(x="Timesteps", y=metric, data=to_plot)
            # plt.errorbar(data0[metric][0], data0[metric][1], yerr=data0[metric][2], color=color, label=label, alpha=0.5)
        except Exception:
            pass
        # plt.plot(running_mean(data0[0], 100), running_mean(data0[1], 100), color, label=label)

    # plt.ticklabel_format(style='sci', axis='x', scilimits=(0, 0))
    # plt.legend(fontsize=14)
    # plt.savefig('plots/gps_exp'+metric + ".png")
    #