import pygame,random
from pygame.locals import *
pygame.init()
noir=(0,0,0)
blanc=(0xFF, 0xFF, 0xFF)
bleuClair = (18,158,158)
font = pygame.font.SysFont('Calibri', 25, True, False)
text_attaque = font.render("attaque",True,noir)
text_potion = font.render("potion",True,noir)
text_fuite = font.render("fuite",True,noir)

def mainCombat(Fenetre,Joueur,Ennemi):
    """
        Fonction principal d'un combat
        mainCombat(Fenetre = fenetre Pygame , Joueur = Class , Ennemi = Liste de Class)
    """
    statCombat = {'NumerauxTourActuel' : 1}
    initialisationFenetreCombat(Fenetre,Joueur,Ennemi,statCombat)

def initialisationFenetreCombat(Fenetre,Joueur,Ennemi,StatCombat):
    """
        Fonction d'affichage initial du combat (interne)
        mainCombat(Fenetre = fenetre Pygame , Joueur = Class , Ennemi = Liste de Class , StatCombat = Liste de donnée du combat)
    """
    Fenetre.blit( font.render( "tour " + str(StatCombat['NumerauxTourActuel']) , True , noir ) , [10, 10] ) # placement du nombre de tour
    Fenetre.blit( font.render( "Pv : " + str(Joueur.pv) + "/" + str(Joueur.pv_max) , True , noir ) ,  [10, 500] )

    # placement du fond d'ecran
    Fenetre.blit( pygame.image.load("data/fondcombatmap1.jpg").convert() , [0, 0])

    pygame.draw.rect(fenetre, blanc, [0, 590, 640, 50]) # dessin du menu de combat
    Fenetre.blit(Joueur.img_combat, Joueur.img_combat.get_rect(center=(100, 300))) # placement du joueur

    # placement des ennemis

    for i in range(len(Ennemi)):

        text_pv_ennemi_0 = font.render("Pv : "+str(ennemi[0].pv)+"/"+str(ennemi[0].pv_max),True,noir)
        Fenetre.blit(text_pv_ennemi_0, [500, 350])
        Fenetre.blit(ennemi[0].img_combat, ennemi[0].img_combat.get_rect(center=(500, 300)))


from copy import copy
import numpy as np
import os
import pickle as pkl
import networkx as nx
import scipy.sparse as sp
from scipy.sparse.linalg.eigen.arpack import eigsh
from sklearn import preprocessing
import sys

def parse_index_file(filename):
    """Parse index file."""
    index = []
    for line in open(filename):
        index.append(int(line.strip()))
    return index

def sample_mask(idx, l):
    """Create mask."""
    mask = np.zeros(l)
    mask[idx] = 1
    return np.array(mask, dtype=np.bool)

'''
def load_data(dataset_str):
    """
    Loads input data from gcn/data directory

    ind.dataset_str.x => the feature vectors of the training instances as scipy.sparse.csr.csr_matrix object;
    ind.dataset_str.tx => the feature vectors of the test instances as scipy.sparse.csr.csr_matrix object;
    ind.dataset_str.allx => the feature vectors of both labeled and unlabeled training instances
        (a superset of ind.dataset_str.x) as scipy.sparse.csr.csr_matrix object;
    ind.dataset_str.y => the one-hot labels of the labeled training instances as numpy.ndarray object;
    ind.dataset_str.ty => the one-hot labels of the test instances as numpy.ndarray object;
    ind.dataset_str.ally => the labels for instances in ind.dataset_str.allx as numpy.ndarray object;
    ind.dataset_str.graph => a dict in the format {index: [index_of_neighbor_nodes]} as collections.defaultdict
        object;
    ind.dataset_str.test.index => the indices of test instances in graph, for the inductive setting as list object.

    All objects above must be saved using python pickle module.

    :param dataset_str: Dataset name
    :return: All data input files loaded (as well the training/test data).
    """
    names = ['x', 'y', 'tx', 'ty', 'allx', 'ally', 'graph']
    objects = []
    for i in range(len(names)):
        with open("data/ind.{}.{}".format(dataset_str, names[i]), 'rb') as f:
            if sys.version_info > (3, 0):
                objects.append(pkl.load(f, encoding='latin1'))
            else:
                objects.append(pkl.load(f))

    x, y, tx, ty, allx, ally, graph = tuple(objects)
    test_idx_reorder = parse_index_file("data/ind.{}.test.index".format(dataset_str))
    test_idx_range = np.sort(test_idx_reorder)

    if dataset_str == 'citeseer':
        # Fix citeseer dataset (there are some isolated nodes in the graph)
        # Find isolated nodes, add them as zero-vecs into the right position
        test_idx_range_full = range(min(test_idx_reorder), max(test_idx_reorder)+1)
        tx_extended = sp.lil_matrix((len(test_idx_range_full), x.shape[1]))
        tx_extended[test_idx_range-min(test_idx_range), :] = tx
        tx = tx_extended
        ty_extended = np.zeros((len(test_idx_range_full), y.shape[1]))
        ty_extended[test_idx_range-min(test_idx_range), :] = ty
        ty = ty_extended

    features = sp.vstack((allx, tx)).tolil()
    features[test_idx_reorder, :] = features[test_idx_range, :]
    adj = nx.adjacency_matrix(nx.from_dict_of_lists(graph))

    labels = np.vstack((ally, ty))
    labels[test_idx_reorder, :] = labels[test_idx_range, :]

    idx_test = test_idx_range.tolist()
    idx_train = range(len(y))
    idx_val = range(len(y), len(y)+500)

    train_mask = sample_mask(idx_train, labels.shape[0])
    val_mask = sample_mask(idx_val, labels.shape[0])
    test_mask = sample_mask(idx_test, labels.shape[0])

    y_train = np.zeros(labels.shape)
    y_val = np.zeros(labels.shape)
    y_test = np.zeros(labels.shape)
    y_train[train_mask, :] = labels[train_mask, :]
    y_val[val_mask, :] = labels[val_mask, :]
    y_test[test_mask, :] = labels[test_mask, :]

    return adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask
'''
def save_sparse_csr(filename,array):
    np.savez(filename, data=array.data, indices=array.indices,
             indptr=array.indptr, shape=array.shape)

def load_sparse_csr(filename):
    loader = np.load(filename)
    return sp.csr_matrix(
        (loader['data'], loader['indices'], loader['indptr']),
        shape=loader['shape']
    )

def load_data(dataset_str):
    """
    Loads input data from gcn/data directory

    ind.dataset_str.x => the feature vectors of the training instances as scipy.sparse.csr.csr_matrix object;
    ind.dataset_str.tx => the feature vectors of the test instances as scipy.sparse.csr.csr_matrix object;
    ind.dataset_str.allx => the feature vectors of both labeled and unlabeled training instances
        (a superset of ind.dataset_str.x) as scipy.sparse.csr.csr_matrix object;
    ind.dataset_str.y => the one-hot labels of the labeled training instances as numpy.ndarray object;
    ind.dataset_str.ty => the one-hot labels of the test instances as numpy.ndarray object;
    ind.dataset_str.ally => the labels for instances in ind.dataset_str.allx as numpy.ndarray object;
    ind.dataset_str.graph => a dict in the format {index: [index_of_neighbor_nodes]} as collections.defaultdict
        object;
    ind.dataset_str.test.index => the indices of test instances in graph, for the inductive setting as list object.

    All objects above must be saved using python pickle module.

    :param dataset_str: Dataset name
    :return: All data input files loaded (as well the training/test data).
    """
    names = ['x', 'y', 'tx', 'ty', 'allx', 'ally', 'graph']
    objects = []
    for i in range(len(names)):
        with open("data/ind.{}.{}".format(dataset_str, names[i]), 'rb') as f:
            if sys.version_info > (3, 0):
                objects.append(pkl.load(f, encoding='latin1'))
            else:
                objects.append(pkl.load(f))

    x, y, tx, ty, allx, ally, graph = tuple(objects)
    test_idx_reorder = parse_index_file("data/ind.{}.test.index".format(dataset_str))
    test_idx_range = np.sort(test_idx_reorder)

    if dataset_str == 'citeseer':
        # Fix citeseer dataset (there are some isolated nodes in the graph)
        # Find isolated nodes, add them as zero-vecs into the right position
        test_idx_range_full = range(min(test_idx_reorder), max(test_idx_reorder)+1)
        tx_extended = sp.lil_matrix((len(test_idx_range_full), x.shape[1]))
        tx_extended[test_idx_range-min(test_idx_range), :] = tx
        tx = tx_extended
        ty_extended = np.zeros((len(test_idx_range_full), y.shape[1]))
        ty_extended[test_idx_range-min(test_idx_range), :] = ty
        ty = ty_extended

    if 'nell.0' in dataset_str:
        # Find relation nodes, add them as zero-vecs into the right position
        test_idx_range_full = range(allx.shape[0], len(graph))
        isolated_node_idx = np.setdiff1d(test_idx_range_full, test_idx_reorder)
        tx_extended = sp.lil_matrix((len(test_idx_range_full), x.shape[1]))
        tx_extended[test_idx_range - allx.shape[0], :] = tx
        tx = tx_extended
        ty_extended = np.zeros((len(test_idx_range_full), y.shape[1]))
        ty_extended[test_idx_range - allx.shape[0], :] = ty
        ty = ty_extended

        features = sp.vstack((allx, tx)).tolil()
        features[test_idx_reorder, :] = features[test_idx_range, :]

        idx_all = np.setdiff1d(range(len(graph)), isolated_node_idx)

        if not os.path.isfile("data/{}.features.npz".format(dataset_str)):
            print("Creating feature vectors for relations - this might take a while...")
            features_extended = sp.hstack(
                (features, sp.lil_matrix(
                    (features.shape[0], len(isolated_node_idx))
                )), dtype=np.float32).todense()
            features_extended[isolated_node_idx, features.shape[1]:] = np.eye(
                len(isolated_node_idx), dtype=np.float32)
            features = sp.csr_matrix(features_extended)
            print("Done!")
            save_sparse_csr("data/{}.features".format(dataset_str),
                            features)
        else:
            features = load_sparse_csr(
                "data/{}.features.npz".format(dataset_str))

        adj = nx.adjacency_matrix(nx.from_dict_of_lists(graph))
    else:
        features = sp.vstack((allx, tx)).tolil()
        features[test_idx_reorder, :] = features[test_idx_range, :]
        adj = nx.adjacency_matrix(nx.from_dict_of_lists(graph))

    labels = np.vstack((ally, ty))
    labels[test_idx_reorder, :] = labels[test_idx_range, :]

    idx_test = test_idx_range.tolist()
    idx_train = range(len(y))
    idx_val = range(len(y), len(y)+500)

    train_mask = sample_mask(idx_train, labels.shape[0])
    val_mask = sample_mask(idx_val, labels.shape[0])
    test_mask = sample_mask(idx_test, labels.shape[0])

    y_train = np.zeros(labels.shape)
    y_val = np.zeros(labels.shape)
    y_test = np.zeros(labels.shape)
    y_train[train_mask, :] = labels[train_mask, :]
    y_val[val_mask, :] = labels[val_mask, :]
    y_test[test_mask, :] = labels[test_mask, :]

    return adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask

def load_embedding(model_type, dataset_str, all_embedding=False):
    fname = "data/{}.{}.emb".format(model_type, dataset_str)
    if all_embedding:
        fname += 'all'
    print(fname)
    with open(fname, 'rb') as f:
        if sys.version_info > (3, 0):
            saved_embeddings = pkl.load(f, encoding='latin1')
        else:
            saved_embeddings = pkl.load(f)
    print(saved_embeddings.shape)
    return preprocessing.normalize(saved_embeddings, norm='l2')

def sparse_to_tuple(sparse_mx):
    """Convert sparse matrix to tuple representation."""
    def to_tuple(mx):
        if not sp.isspmatrix_coo(mx):
            mx = mx.tocoo()
        coords = np.vstack((mx.row, mx.col)).transpose()
        values = mx.data
        shape = mx.shape
        return coords, values, shape

    if isinstance(sparse_mx, list):
        for i in range(len(sparse_mx)):
            sparse_mx[i] = to_tuple(sparse_mx[i])
    else:
        sparse_mx = to_tuple(sparse_mx)

    return sparse_mx

def preprocess_features(features, sparse=False):
    """Row-normalize feature matrix and convert to tuple representation"""
    rowsum = np.array(features.sum(1))
    r_inv = np.power(rowsum, -1).flatten()
    r_inv[np.isinf(r_inv)] = 0.
    r_mat_inv = sp.diags(r_inv)
    features = r_mat_inv.dot(features)
    if sparse:
        return sparse_to_tuple(features)
    else:
        return sp.spmatrix.toarray(features)

def normalize_adj(adj):
    """Symmetrically normalize adjacency matrix."""
    adj = sp.coo_matrix(adj)
    rowsum = np.array(adj.sum(1))
    d_inv_sqrt = np.power(rowsum, -0.5).flatten()
    d_inv_sqrt[np.isinf(d_inv_sqrt)] = 0.
    d_mat_inv_sqrt = sp.diags(d_inv_sqrt)
    return adj.dot(d_mat_inv_sqrt).transpose().dot(d_mat_inv_sqrt).tocoo()

def preprocess_adj(adj):
    """Preprocessing of adjacency matrix for simple GCN model and conversion to tuple representation."""
    adj_normalized = normalize_adj(adj + sp.eye(adj.shape[0]))
    return sparse_to_tuple(adj_normalized)

def preprocess_adj_dense(adj):
    """Preprocessing of adjacency matrix for simple GCN model and conversion to tuple representation."""
    adj_normalized = normalize_adj(adj + sp.eye(adj.shape[0]))
    # return sparse_to_tuple(adj_normalized)
    return sp.spmatrix.toarray(adj_normalized)

def construct_feed_dict(features, support, labels, labels_mask, placeholders,
                        neighbor_features=None, neighbor_ids=None):
    """Construct feed dictionary."""
    feed_dict = dict()
    feed_dict.update({placeholders['labels']: labels})
    feed_dict.update({placeholders['labels_mask']: labels_mask})
    feed_dict.update({placeholders['features']: features})
    if 'neighbor_features' in placeholders.keys():
        feed_dict.update(
            {placeholders['neighbor_features'][i]: \
                 neighbor_features[i] for i in range(len(neighbor_features))}
        )
    if 'neighbor_ids' in placeholders.keys():
        feed_dict.update(
            {placeholders['neighbor_ids'][i]: \
                 neighbor_ids[i] for i in range(len(neighbor_ids))}
        )
    feed_dict.update({placeholders['support']: support})
    feed_dict.update({placeholders['num_features_nonzero']: features[1].shape})
    return feed_dict

def chebyshev_polynomials(adj, k):
    """Calculate Chebyshev polynomials up to order k. Return a list of sparse matrices (tuple representation)."""
    print("Calculating Chebyshev polynomials up to order {}...".format(k))

    adj_normalized = normalize_adj(adj)
    laplacian = sp.eye(adj.shape[0]) - adj_normalized
    largest_eigval, _ = eigsh(laplacian, 1, which='LM')
    scaled_laplacian = (2. / largest_eigval[0]) * laplacian - sp.eye(adj.shape[0])

    t_k = list()
    t_k.append(sp.eye(adj.shape[0]))
    t_k.append(scaled_laplacian)

    def chebyshev_recurrence(t_k_minus_one, t_k_minus_two, scaled_lap):
        s_lap = sp.csr_matrix(scaled_lap, copy=True)
        return 2 * s_lap.dot(t_k_minus_one) - t_k_minus_two

    for i in range(2, k+1):
        t_k.append(chebyshev_recurrence(t_k[-1], t_k[-2], scaled_laplacian))

    return sparse_to_tuple(t_k)

'''
adj is a csr_matrix
'''
def get_col_indices(adj):
    print(adj.shape)
    col_indices = []
    coo_adj = adj.tocoo()
    row_starts = []
    pre_row = -1
    for i in range(len(coo_adj.row)):
        if not coo_adj.row[i] == pre_row:
            row_starts.append(i)
            pre_row = coo_adj.row[i]
    row_starts.append(len(coo_adj.row))
    assert len(row_starts) == adj.shape[0] + 1
    for i in range(adj.shape[0]):
        col_indices.append(coo_adj.col[row_starts[i]: row_starts[i + 1]])
    return col_indices

def sample_neighbors(col_indices, features, count=1):
    neighbor_features = []
    for i in range(count):
        one_neighbor_features = np.zeros(features.shape, dtype=features.dtype)
        for j in range(features.shape[0]):
            offset = np.random.randint(len(col_indices[j]))
            nei_col = col_indices[j][offset]
            one_neighbor_features[j, :] = copy(features[nei_col, :])
        neighbor_features.append(one_neighbor_features)
    return neighbor_features

def sample_neighbors_id(col_indices, count=1, neighbor_probs=None):
    neighbor_features_id = []
    for i in range(count):
        one_neighbor_features_id = np.zeros(len(col_indices), dtype=np.int32)
        for j in range(len(col_indices)):
            if neighbor_probs is None:
                offset = np.random.randint(len(col_indices[j]))
                one_neighbor_features_id[j] = col_indices[j][offset]
            else:
                one_neighbor_features_id[j] = np.random.choice(
                    col_indices[j], p=neighbor_probs[j]
                )
        neighbor_features_id.append(one_neighbor_features_id)
    return neighbor_features_id

def generate_random_splits(y_train, y_val, y_test, train_mask, val_mask,
                           test_mask, dataset, repeats=5):
    labels = y_train + y_val + y_test

def get_sampling_probability(col_indices, sampling_strategy, dataset):
    if sampling_strategy == 'uniform':
        return None
    elif sampling_strategy == 'degree' or sampling_strategy == 'degree_reverse':
        n = len(col_indices)

        # calculate degree of each node
        degrees = np.zeros(n, dtype=np.float32)
        for i in range(n):
            degrees[i] = len(col_indices[i])

        # generate probability of each node
        sampling_probabilities = []
        for i in range(n):
            neighbor_degrees = copy(degrees[col_indices[i]])
            if sampling_strategy == 'degree_reverse':
                neighbor_degrees = np.reciprocal(neighbor_degrees)
            neighbor_degrees = neighbor_degrees / np.sum(neighbor_degrees)
            sampling_probabilities.append(neighbor_degrees)
        return sampling_probabilities
    elif sampling_strategy == 'pagerank':
        pr_scores = np.load("data/" + dataset + '.prs.npy')
        n = len(col_indices)
        assert n == len(pr_scores), \
            'mismatch lengths of page rank score and graph nodes' + \
            str(n) + ':::' + len(pr_scores)

        # generate probability of each node
        sampling_probabilities = []
        for i in range(n):
            neighbor_scores = copy(pr_scores[col_indices[i]])
            sum_scores = np.sum(neighbor_scores)
            if abs(sum_scores) < 1e-9:
                neighbor_scores = np.ones(neighbor_scores.shape,
                                          dtype=np.float32) / neighbor_scores.shape[0]
            else:
                neighbor_scores = neighbor_scores / np.sum(neighbor_scores)
            sampling_probabilities.append(neighbor_scores)
        return sampling_probabilities
    else:
        raise ValueError('unexpected sampling strategy:' + sampling_strategy)


""" Defines 'types' that various fields can be """

from typing import Tuple, Union

# Literal only available in python 3.8 + so try import otherwise use extensions
try:
    from typing import Literal
except ImportError:
    from typing_extensions import Literal
from typing_extensions import Annotated

import pydantic
import numpy as np
# from matplotlib.axes import Axes
Axes = None
from shapely.geometry.base import BaseGeometry
from ..log import ValidationError

""" Numpy Arrays """

# type tag default name
TYPE_TAG_STR = "type"

def annotate_type(UnionType):  # pylint:disable=invalid-name
    """Annotated union type using TYPE_TAG_STR as discriminator."""
    return Annotated[UnionType, pydantic.Field(discriminator=TYPE_TAG_STR)]

# generic numpy array
Numpy = np.ndarray

class TypedArrayLike(np.ndarray):
    """A numpy array with a type given by cls.inner_type"""

    @classmethod
    def make_tuple(cls, v):
        """Converts a nested list of lists into a list of tuples."""
        return (
            tuple(cls.make_tuple(x) for x in v)
            if isinstance(v, list)
            else cls.inner_type(v)  # pylint:disable=no-member
        )

    @classmethod
    def __get_validators__(cls):
        """boilerplate"""
        yield cls.validate_type

    @classmethod
    def validate_type(cls, val):
        """validator"""
        # need to fix, doesnt work for simulationdata_export and load?

        if isinstance(val, np.ndarray):
            val_ndims = len(val.shape)
            cls_ndims = cls.ndims  # pylint:disable=no-member

            if (cls_ndims is not None) and (cls_ndims != val_ndims):
                raise ValidationError(
                    "wrong number of dimensions given. " f"Given {val_ndims}, expected {cls_ndims}."
                )
            return cls.make_tuple(val.tolist())

        return tuple(val)

    @classmethod
    def __modify_schema__(cls, field_schema):
        """Sets the schema of ArrayLike."""

        field_schema.update(
            dict(
                title="Array Like",
                description="Accepts sequence (tuple, list, numpy array) and converts to tuple.",
                type="tuple",
                properties={},
                required=[],
            )
        )

class ArrayLikeMeta(type):
    """metclass for Array, enables Array[type] -> TypedArray"""

    def __getitem__(cls, type_ndims):
        """Array[type, ndims] -> TypedArrayLike"""
        desired_type, ndims = type_ndims
        return type("Array", (TypedArrayLike,), {"inner_type": desired_type, "ndims": ndims})

class ArrayLike(np.ndarray, metaclass=ArrayLikeMeta):
    """type of numpy array with annotated type (Array[float], Array[complex])"""

""" Complex Values """

class ComplexNumber(pydantic.BaseModel):
    """Complex number with a well defined schema."""

    real: float
    imag: float

    @property
    def as_complex(self):
        """return complex representation of ComplexNumber."""
        return self.real + 1j * self.imag

class tidycomplex(complex):  # pylint: disable=invalid-name
    """complex type that we can use in our models."""

    @classmethod
    def __get_validators__(cls):
        """Defines which validator function to use for ComplexNumber."""
        yield cls.validate

    @classmethod
    def validate(cls, value):
        """What gets called when you construct a tidycomplex."""

        if isinstance(value, ComplexNumber):
            return value.as_complex
        if isinstance(value, dict):
            c = ComplexNumber(**value)
            return c.as_complex
        return cls(value)

    @classmethod
    def __modify_schema__(cls, field_schema):
        """Sets the schema of ComplexNumber."""
        field_schema.update(ComplexNumber.schema())

""" symmetry """

Symmetry = Literal[0, -1, 1]

""" geometric """

Size1D = pydantic.NonNegativeFloat
Size = Tuple[Size1D, Size1D, Size1D]
Coordinate = Tuple[float, float, float]
Coordinate2D = Tuple[float, float]
Bound = Tuple[Coordinate, Coordinate]
GridSize = Union[pydantic.PositiveFloat, Tuple[pydantic.PositiveFloat, ...]]
Axis = Literal[0, 1, 2]
Axis2D = Literal[0, 1]
Shapely = BaseGeometry
Vertices = Union[Tuple[Coordinate2D, ...], ArrayLike[float, 2]]
PlanePosition = Literal["bottom", "middle", "top"]

""" medium """

# custom medium
InterpMethod = Literal["nearest", "linear"]

# Complex = Union[complex, ComplexNumber]
Complex = Union[tidycomplex, ComplexNumber]
PoleAndResidue = Tuple[Complex, Complex]

# PoleAndResidue = Tuple[Tuple[float, float], Tuple[float, float]]
FreqBoundMax = float
FreqBoundMin = float
FreqBound = Tuple[FreqBoundMin, FreqBoundMax]

""" sources """

Polarization = Literal["Ex", "Ey", "Ez", "Hx", "Hy", "Hz"]
Direction = Literal["+", "-"]

""" monitors """

EMField = Literal["Ex", "Ey", "Ez", "Hx", "Hy", "Hz"]
FieldType = Literal["Ex", "Ey", "Ez", "Hx", "Hy", "Hz"]
FreqArray = Union[Tuple[float, ...], ArrayLike[float, 1]]
ObsGridArray = Union[Tuple[float, ...], ArrayLike[float, 1]]

""" plotting """

#Ax = Axes
Ax = None

""" mode tracking """

TrackFreq = Literal["central", "lowest", "highest"]


import json
from random import shuffle, randrange

from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect

# Create your views here.
from django.shortcuts import render_to_response, get_object_or_404
from django.views.generic import UpdateView, DetailView
import requests
from board.models import UserProfile, BoardState
from board.tasks import board_iteration

@login_required()
def secret_page(request, *args, **kwargs):
    return HttpResponse('Secret contents!', status=200)

@login_required()
def start_game(request, *args, **kwargs):
    if not request.user.is_superuser:
        return HttpResponse('You should be admin to display this page')

    BoardState.objects.all().delete()

    empty_board_state = {
        "the_end": False,
        "message": "",
        #"last_player": 0,
        "reset": True,
        "players": []
    }

    players = []
    for user_profile in UserProfile.active_gamers.all():
        players.append({
            "name": user_profile.user.username,
            "url": user_profile.url,
            "avatar": "",
            "dice": [],
            "bid": [],
            "active": True,
        })

    shuffle(players)
    for i, gamer in enumerate(players):
        gamer['id'] = i
    empty_board_state['players'] = players

    empy_state_data = {
        'gameplay': [],
        'players': []
    }
    for user in players:
        empy_state_data['players'].append({'id': user['id'], 'name': user['name'], 'dice': 1})

    empty_board_state['last_player'] = len(empy_state_data['players']) - 1

    BoardState.objects.create(iteration=0, board_data=empty_board_state, state_data=empy_state_data)
    board_iteration.delay(iteration=1)
    return HttpResponseRedirect(reverse('board'))

@login_required()
def board(request, *args, **kwargs):
    return render_to_response('board/board.html')

# @login_required()
def board_json(request, pk):
    object = get_object_or_404(BoardState.objects.filter(iteration=pk))
    return HttpResponse(json.dumps(object.board_data))

class UserProfileUpdate(UpdateView):
    model = UserProfile
    fields = ['url']

    def get_object(self, queryset=None):
        if queryset is None:
            queryset = self.get_queryset()
        return queryset.filter(user=self.request.user)[0]

    def form_valid(self, form):
        url = form.cleaned_data['url']
        url = url.rstrip('/')
        try:
            r = requests.get(url + '/ping', timeout=1)
        except:
            messages.add_message(self.request, messages.ERROR,
                                 'Your URL does not reply "pong" on %s/ping [HTTP %d]' % (url, r.status_code))
            return self.form_invalid(form)

        if r.status_code == 200 and r.text.strip() == 'pong':
            messages.add_message(self.request, messages.SUCCESS,
                                 'Ping-Pong OK! URL successfully saved!')
            return super(UserProfileUpdate, self).form_valid(form)
        else:
            messages.add_message(self.request, messages.ERROR,
                                 'Your URL does not reply "pong" on %s/ping [HTTP %d]' % (url, r.status_code))
            return self.form_invalid(form)

    def get_success_url(self):
        return reverse('profile')


from sklearn.decomposition import PCA
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os

plot = False

df_fname = r"E:\nanopores\2023_02_20\ch1\pore2\data3\f1.pkl"
root = os.path.dirname(df_fname)
name,ext = os.path.splitext(os.path.basename(df_fname))

df = pd.read_pickle(df_fname)

dfInfToNan = df.replace([np.inf, -np.inf], np.nan, inplace=False)
noNan = dfInfToNan.copy().dropna()
conditionedDf = noNan.iloc[:,1:]
array = conditionedDf.to_numpy()

print(f"All values finite: {np.isfinite(array).all()}")

pca = PCA(n_components = 2)
pca.fit(array)
new = pca.transform(array)

if plot:
    fig, ax = plt.subplots()
    ax.scatter(new[:,0],new[:,1], alpha = 0.1)
    plt.show()

noNan['pc_1'] = new[:,0]
noNan['pc_2'] = new[:,1]

noNan.to_pickle(os.path.join(root,f"{name}_PCA.pkl"))

names = list(conditionedDf.columns)
pc1 = dict(zip(names,pca.components_[0]))
pc2 = dict(zip(names,pca.components_[1]))
print(f"PC1: {pc1}")
print(f"PC2: {pc2}")


from keras.layers import Conv2D
from keras.layers import Input
from keras.layers import MaxPooling2D, UpSampling2D
from keras.models import Model
from keras.datasets.mnist import load_data
from numpy import reshape
import matplotlib.pyplot as plt
from keras.callbacks import ModelCheckpoint
import numpy as np

(xtrain0, ytrain0), (xtest0, ytest0) = load_data()

def wheredex(x,y,d=7):
  assert len(x)==len(y),f"{len(x)} {len(y)}"
  i=np.where(y==d)
  k=x[i]
  if len(k)>5000:k=k[:5000]
  return reshape(k.astype('float32') / 255, (len(k), 28, 28, 1))#[:5000]
def wherenotdex(x,y,d=7):
  assert len(x)==len(y),f"{len(x)} {len(y)}"
  i=np.where(y!=d)
  k=x[i]
  if len(k)>5000:k=k[:5000]
  return reshape(k.astype('float32') / 255, (len(k), 28, 28, 1))#[:5000]

xtrain=wheredex(xtrain0,ytrain0,7)
xatrain=wherenotdex(xtrain0,ytrain0,7)
xtest=wheredex(xtest0,ytest0,7)
xatest=wherenotdex(xtest0,ytest0,7)

np.savez_compressed("data",x=xtrain,xa=xatrain,xval=xtest,xaval=xatest)

print(xtrain.shape, xtest.shape)

input_img = Input(shape=(28, 28, 1))

enc_conv1 = Conv2D(12, (3, 3), activation='relu', padding='same')(input_img)
enc_pool1 = MaxPooling2D((2, 2), padding='same')(enc_conv1)
enc_conv2 = Conv2D(8, (4, 4), activation='relu', padding='same')(enc_pool1)
enc_ouput = MaxPooling2D((4, 4), padding='same')(enc_conv2)

dec_conv2 = Conv2D(8, (4, 4), activation='relu', padding='same')(enc_ouput)
dec_upsample2 = UpSampling2D((4, 4))(dec_conv2)
dec_conv3 = Conv2D(12, (3, 3), activation='relu')(dec_upsample2)
dec_upsample3 = UpSampling2D((2, 2))(dec_conv3)
dec_output = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(dec_upsample3)

autoencoder = Model(input_img, dec_output)
autoencoder.compile(optimizer='rmsprop', loss='binary_crossentropy')
autoencoder.summary()

checkpoint = ModelCheckpoint("model.hdf5", monitor='loss', verbose=1,
    save_best_only=True, mode='auto', period=1)

autoencoder.fit(xtrain, xtrain, epochs=20, batch_size=128, shuffle=True,callbacks=[checkpoint])

decoded_imgs = autoencoder.predict(x_test)

n = 10
plt.figure(figsize=(20, 4))
for i in range(n):
  plt.gray()
  ax = plt.subplot(2, n, i+1)
  plt.imshow(x_test[i].reshape(28, 28))
  ax.get_xaxis().set_visible(False)
  ax.get_yaxis().set_visible(False)

  ax = plt.subplot(2, n, i +1+n)
  plt.imshow(decoded_imgs[i].reshape(28, 28))
  ax.get_xaxis().set_visible(False)
  ax.get_yaxis().set_visible(False)
plt.show()


from torch.utils.data import Dataset, DataLoader
from torchvision import transforms
import torch
import matplotlib.pyplot as plt
import numpy as np
from PIL import Image
import os
from os.path import join
from os import listdir
import pandas as pd
from mask_functions import read_dicom, mask2rle, rle2mask
from collections import Counter
import torch.nn as nn
mean=(0.485, 0.456, 0.406)
std=(0.229, 0.224, 0.225)

transform = transforms.Compose([
	transforms.ToPILImage(),
	transforms.Resize((256,256)),
	transforms.ToTensor(),
	transforms.Normalize(mean, std)
	])

label_transform = transforms.Compose([
	transforms.ToPILImage(),
	transforms.Resize((256, 256)),
	transforms.ToTensor()
	])

class siim_dataset(Dataset):

	def __init__(self, img_path, label_csv):

		self.img_path = img_path
		self.label_csv = label_csv
		self.imgs_dir = listdir(join(os.getcwd(), self.img_path))
		###############################################################
		######		 			  [csv format]		  			 ######
		######		ImageId						EncodedPixels	 ######
		###############################################################
		self.label_csv = pd.read_csv(join(os.getcwd(), self.label_csv))
		self.image_id = self.label_csv['ImageId'].values
		self.image_mask = self.label_csv[' EncodedPixels'].values
		self.id2mask = {}
		for x, y in zip(self.image_id, self.image_mask):
			self.id2mask[x]=y

	def __len__(self):
		return len(self.imgs_dir)

	def __getitem__(self, idx):

		dir_name = self.imgs_dir[idx]
		sub_dir_name = [file for file in listdir(join(os.getcwd(), self.img_path, dir_name)) if file != '.DS_Store'][0]
		image_name = [file for file in listdir(join(os.getcwd(), self.img_path, dir_name, sub_dir_name)) if file != '.DS_Store'][0]

		#image_mask = self.label_csv[self.label_csv['ImageId'] == image_name.replace('.dcm', '')][' EncodedPixels'].values[0]
		image_mask = self.id2mask[image_name.replace('.dcm','')]

		image, width, height = read_dicom(join(os.getcwd(), self.img_path, dir_name, sub_dir_name, image_name))

		label = rle2mask(image_mask, width, height).T

		image = transform(image)
		label = label_transform(label)
		label = label.view(1, 256, 256)

		return image, label

if __name__ == '__main__':
	dataset = siim_dataset('dicom-images-train', 'train-rle.csv')
	dataset = DataLoader(dataset, batch_size=1)
	fn = nn.BCELoss()
	loss = 0
	for index, i in enumerate(dataset):
		x, y = i
		out = torch.ones(x.size())
		out /= 2
		c_loss = fn(out, y)
		print(c_loss)
		loss += c_loss.item()
	print(loss/len(dataset))


import json
import logging
import os
import re
from typing import Any, Dict, List, Optional, Tuple, Union

from neural_compressor.ux.utils.exceptions import InternalException
from neural_compressor.ux.utils.json_serializer import JsonSerializer
from neural_compressor.ux.utils.logger import log
from neural_compressor.ux.utils.templates.metric import Metric
from neural_compressor.ux.utils.utils import get_size

logging.basicConfig(level=logging.INFO)

class WorkloadInfo(JsonSerializer):
    """Create template for workload_list entity."""

    def __init__(
        self,
        request_id: str,
        project_name: Optional[str],
        workload_path: Optional[str],
        model_path: Optional[str],
        input_precision: Optional[str],
        model_output_path: Optional[str],
        output_precision: Optional[str],
        mode: Optional[str],
        metric: Optional[Union[Metric, dict]],
        status: Optional[str],
        code_template_path: Optional[str],
        created_at: Optional[str],
        supports_profiling: Optional[bool],
        execution_details: Optional[Dict[str, dict]] = None,
    ) -> None:
        """Initialize configuration WorkloadInfo class."""
        super().__init__()
        self._id = request_id
        self._project_name = project_name
        self._model_path = model_path
        self._input_precision = input_precision
        self._model_output_path = model_output_path
        self._output_precision = output_precision
        self._mode = mode
        self._workload_path = workload_path
        self._status = status
        self._metric = metric
        self._code_template_path = code_template_path
        self._config_path: Optional[str] = None
        self._log_path: Optional[str] = None
        self._execution_details = execution_details
        self._created_at = created_at
        self._supports_profiling = supports_profiling
        if self._workload_path:
            self._config_path = os.path.join(
                self._workload_path,
                "config.yaml",
            )
            self._log_path = os.path.join(self._workload_path, "output.txt")
            if not os.path.isfile(self._log_path):
                os.makedirs(os.path.dirname(self._log_path), exist_ok=True)
                with open(self._log_path, "w") as log_file:
                    log_file.write("Configuration created.\n")
        if self._model_path and self._metric:
            if isinstance(self._metric, dict) and not self._metric.get("size_input_model"):
                self._metric["size_input_model"] = get_size(self._model_path)
            if isinstance(self._metric, Metric) and not self._metric.size_input_model:
                self._metric.insert_data("size_input_model", str(get_size(self._model_path)))

    def insert_data(self, data: dict) -> None:
        """
        Set all available properties from workload_info dict.

        param: data
        type: dict
        """
        for key, value in data:
            attribute = "_" + key
            if attribute in self.__dict__:
                self.__setattr__(attribute, data[key])

    def serialize(
        self,
        serialization_type: str = "default",
    ) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
        """
        Serialize class to dict.

        :param serialization_type: serialization type, defaults to "default"
        :type serialization_type: str, optional
        :return: serialized class
        :rtype: Union[dict, List[dict]]
        """
        result = {}
        for key, value in self.__dict__.items():
            variable_name = re.sub(r"^_", "", key)
            if key in self._skip:
                continue
            elif issubclass(type(value), JsonSerializer):
                # pylint: disable=maybe-no-member
                result[variable_name] = value.serialize(serialization_type)
            else:
                result[variable_name] = self.serialize_item(value)

        if result.get("metric", None):
            result.update(result["metric"])
        return result

class WorkloadsListMigrator:
    """Workloads list migrator."""

    def __init__(self) -> None:
        """Initialize workloads list migrator."""
        self.workloads_json = os.path.join(
            os.environ.get("HOME", ""),
            ".neural_compressor",
            "workloads_list.json",
        )
        self.workloads_data: dict = {}
        self.version_migrators: Dict[int, Any] = {
            2: self._migrate_to_v2,
            3: self._migrate_to_v3,
            4: self._migrate_to_v4,
        }

    @property
    def current_version(self) -> int:
        """Get version of current workloads list."""
        self.ensure_workloads_loaded()
        return int(float(self.workloads_data.get("version", 1)))

    @property
    def require_migration(self) -> bool:
        """Check if workloads list require migration."""
        if not os.path.isfile(self.workloads_json):
            log.debug("Workloads list does not exits.")
            return False
        if self.current_version >= max(self.version_migrators.keys()):
            log.debug("Workloads list already up to date.")
            return False
        return True

    def load_workloads_data(self) -> None:
        """Load workloads data from json."""
        with open(self.workloads_json, encoding="utf-8") as workloads_list:
            self.workloads_data = json.load(workloads_list)

    def ensure_workloads_loaded(self) -> None:
        """Make sure that workloads list is loaded."""
        if not self.workloads_data and os.path.isfile(self.workloads_json):
            self.load_workloads_data()

    def dump(self) -> None:
        """Dump workloads information to json."""
        with open(self.workloads_json, "w") as f:
            json.dump(self.workloads_data, f, indent=4)

    def migrate(self) -> None:
        """Migrate workloads list to latest version."""
        self.ensure_workloads_loaded()
        if not self.require_migration:
            return

        migration_steps = range(self.current_version, max(self.version_migrators.keys()))
        for step in migration_steps:
            migration_version = step + 1
            self._migrate_workloads_list(migration_version)

    def _migrate_workloads_list(self, migration_version: int) -> None:
        """Migrate workloads list one version up."""
        migrate_workloads = self.version_migrators.get(migration_version, None)
        if migrate_workloads is None:
            raise InternalException(
                f"Could not migrate workloads list to version {migration_version}",
            )
        migrate_workloads()

    def _migrate_to_v2(self) -> None:
        """Migrate workloads list from v1 to v2."""
        parsed_workload_data = {
            "active_workspace_path": "",
            "workloads": "",
            "version": 2,
        }
        parsed_workload_data.update(
            {"active_workspace_path": self.workloads_data["active_workspace_path"]},
        )
        parsed_workloads = {}
        for workload_id, workload_data in self.workloads_data["workloads"].items():
            parsed_workloads.update({workload_id: self._parse_workload_to_v2(workload_data)})
        parsed_workload_data.update({"workloads": parsed_workloads})
        self.workloads_data = parsed_workload_data

    def _parse_workload_to_v2(self, workload_data: dict) -> dict:
        """Parse workload from v1 to v2."""
        parsed_workload = {
            "input_precision": "fp32",
            "output_precision": "int8",
            "mode": "tuning",
        }
        for key, value in workload_data.items():
            if key == "metric":
                parsed_key, parsed_value = self._parse_metric_to_v2(value)
            elif key == "execution_details":
                parsed_key, parsed_value = self._parse_exec_details_to_v2(value)
            else:
                parsed_key, parsed_value = self._parse_regular_key_to_v2(key, value)
            parsed_workload.update({parsed_key: parsed_value})

        return parsed_workload

    @staticmethod
    def _parse_metric_to_v2(metric_data: dict) -> Tuple:
        """Parse metric from v1 to v2."""
        metric_mappings = {
            "acc_fp32": "acc_input_model",
            "acc_int8": "acc_optimized_model",
            "tuning_time": "optimization_time",
            "size_fp32": "size_input_model",
            "size_int8": "size_optimized_model",
            "perf_throughput_fp32": "perf_throughput_input_model",
            "perf_throughput_int8": "perf_throughput_optimized_model",
        }
        parsed_metric = {}
        for key, value in metric_data.items():
            parsed_key = metric_mappings.get(key, key)
            parsed_metric.update({parsed_key: value})
        return "metric", parsed_metric

    def _parse_exec_details_to_v2(self, execution_details: Optional[dict]) -> Tuple:
        """Parse metric from v1 to v2."""
        parsed_exec_details = {}
        if execution_details is None:
            return "execution_details", None
        if "tuning" in execution_details:
            parsed_exec_details.update(
                {
                    "optimization": {
                        "input_graph": execution_details["tuning"]["model_path"],
                        "input_precision": "fp32",
                        "output_graph": execution_details["tuning"]["model_output_path"],
                        "output_precision": "int8",
                        "framework": execution_details["tuning"]["framework"],
                        "input_nodes": "N/A",
                        "output_nodes": "N/A",
                        "tune": True,
                        "config_path": execution_details["tuning"]["config_path"],
                        "instances": execution_details["tuning"]["instances"],
                        "cores_per_instance": execution_details["tuning"]["cores_per_instance"],
                        "command": execution_details["tuning"]["command"],
                    },
                },
            )
        if "fp32_benchmark" in execution_details:
            parsed_exec_details.update(
                {
                    "input_model_benchmark": {
                        "performance": self._parse_perf_data_to_v2(
                            execution_details["fp32_benchmark"],
                        ),
                    },
                },
            )
        if "int8_benchmark" in execution_details:
            parsed_exec_details.update(
                {
                    "optimized_model_benchmark": {
                        "performance": self._parse_perf_data_to_v2(
                            execution_details["int8_benchmark"],
                        ),
                    },
                },
            )

        return "execution_details", parsed_exec_details

    @staticmethod
    def _parse_perf_data_to_v2(perf_data: dict) -> dict:
        """Perse performance data from v1 to v2."""
        key_mapping = {"datatype": "precision"}
        parsed_perf_data = {}
        for key, value in perf_data.items():
            parsed_key = key
            if key in key_mapping:
                parsed_key = key_mapping[key]
            parsed_perf_data.update({parsed_key: value})
        return parsed_perf_data

    @staticmethod
    def _parse_regular_key_to_v2(key: str, value: Any) -> Tuple:
        """Parse metric from v1 to v2."""
        key_mapping = {
            "fp32": "input_model",
            "int8": "optimized_model",
            "tuning": "optimization",
            "datatype": "precision",
        }
        for old_key, new_key in key_mapping.items():
            if old_key in key:
                key = key.replace(old_key, new_key)
        return key, value

    def _migrate_to_v3(self) -> None:
        """Migrate workloads list from v2 to v3."""
        for workload_id, workload_data in self.workloads_data["workloads"].items():
            workload_data.update(
                {
                    "project_name": os.path.basename(workload_data.get("model_path", "")),
                    "created_at": "2021-07-15T14:19:18.860579",
                },
            )
        self.workloads_data.update(
            {
                "version": 3,
            },
        )

    def _migrate_to_v4(self) -> None:
        """Migrate workloads list from v3 to v4."""
        for workload_id, workload_data in self.workloads_data["workloads"].items():
            workload_data.update(
                {
                    "supports_profiling": False,
                },
            )
        self.workloads_data.update(
            {
                "version": 4,
            },
        )


import os
from dotenv import load_dotenv
#from datetime import date

from app import APP_ENV
# web_app/routes/pokemon_routess.py

from flask import Blueprint, jsonify, request, render_template, redirect, flash

pokemon_routes = Blueprint("pokemon_routes", __name__)

@pokemon_routess.route("/api/pokemons")
@pokemon_routess.route("/api/pokemons.json")
def list_pokemons():
    print("pokemonS...")
    pokemons = [
        {"id": 1, "title": "pokemon 1", "year": 1957},
        {"id": 2, "title": "pokemon 2", "year": 1990},
        {"id": 3, "title": "pokemon 3", "year": 2031},
    ] # some dummy / placeholder data
    return jsonify(pokemons)

#if URL params are required, we will use this kind of approach:
@pokemon_routess.route("/api/pokemons/<int:pokemon_id>")
@pokemon_routess.route("/api/pokemons/<int:pokemon_id>.json")
def get_pokemon(pokemon_id):
    print("pokemon...", pokemon_id)
    pokemon = {"id": pokemon_id, "title": f"Example pokemon", "year": 2000} # some dummy / placeholder data
    return jsonify(pokemon)


from auth.login import Login

# Criar uma instância da classe Login
login = Login()

# Verificar se a sessão já existe em um arquivo
session_file = 'session.pkl'
try:
    session = login.load_session(session_file)
    print("Sessão carregada do arquivo.")
except FileNotFoundError:
    # Obter uma nova sessão
    status_code, session_id, session = login.get_session()
    login.save_session(session, session_file)
    print("Nova sessão criada e salva no arquivo.")

# Continuar usando a sessão para outras requisições...


# -*- coding: utf-8 -*-
"""
Created on Wed Feb 20 15:29:19 2019

@author: Ham

Task from HackerRank
Given a string , which is the company name in lowercase letters,
your task is to find the top three most common characters in the string.

Print the three most common characters along with their occurrence count.
Sort in descending order of occurrence count.
If the occurrence count is the same, sort the characters in alphabetical order.
For example, according to the conditions described above,

 would have it's logo with the letters .

Input Format

A single line of input containing the string .

Constraints

Output Format

Print the three most common characters along with their occurrence count each on a separate line.
Sort output in descending order of occurrence count.
If the occurrence count is the same, sort the characters in alphabetical order.

Sample Input 0

aabbbccde
Sample Output 0

b 3
a 2
c 2
Explanation 0

Here, b occurs  times. It is printed first.
Both a and c occur  times. So, a is printed in the second line
and c in the third line because a comes before c in the alphabet.

Note: The string  has at least  distinct characters.
"""
import collections

if __name__ == '__main__':
    s = "szrmtbttyyaymadobvwniwmozojggfbtswdiocewnqsjrkimhovimghixqryqgzhgba"\
        "kpncwupcadwvglmupbexijimonxdowqsjinqzytkooacwkchatuwpsoxwvgrrejkukc"\
        "vyzbkfnzfvrthmtfvmbppkdebswfpspxnelhqnjlgntqzsprmhcnuomrvuyolvzlni"
    #s = input()
    #occ = {}
    #[occ.update({c: occ.get(c, 0)+1}) for c in s]
    #for c in s:
    #    occ.update({c: occ.get(c, 0) + 1})
    occ = collections.Counter(s)
    print(s)
    print(occ)
    occ_sorted = sorted(occ.items(), key=lambda x: x[1] * 1000 - ord(x[0]),
                        reverse=True)
    print(occ_sorted)
    [print(x[0], x[1]) for x in occ_sorted[:3]]


import pandas as pd
import numpy as np

from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold

import matplotlib.pyplot as plt
import timeit
import modeldata as md

starting_time = timeit.default_timer()

df = pd.read_csv('./train_set_2k_norm_pt.csv')
df.set_index('gene_id',drop=True, inplace=True)

genes_to_keep = []
with open ('xgbfeaimp_weight_top20.txt', 'r') as file:
    for line in file:
        genes_to_keep.append(line.strip())

df = df[df.index.isin(genes_to_keep)]

# transpose the df
df = df.T

df_y= pd.read_csv('./train_set_2k_metadata.csv')
df['label'] = list(df_y['label'])

x = df.drop(columns='label')
y = df['label']
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.1, random_state=42, stratify=y)

scaler = MinMaxScaler()
x_train = pd.DataFrame(scaler.fit_transform(x_train), columns=x_train.columns)

scaler = MinMaxScaler()
x_test = pd.DataFrame(scaler.fit_transform(x_test), columns=x_test.columns)

# 10fold cv
cv = StratifiedKFold(n_splits=10, random_state=42, shuffle=True)
list_score = []
list_testsetscore = []

testset_y = pd.DataFrame(y_test).reset_index(drop=True)
testset= pd.concat([x_test, testset_y],axis=1)

# Run 10fold cv on each model
modelchoie = ['randomforest','xgboost','lgbm', 'svc']
for model in modelchoie:
    print("Currently running this model: ",model)
    for train_idx, valid_idx in cv.split(x_train, y_train):
        trainset = x_train.iloc[list(train_idx)]
        validset = x_train.iloc[list(valid_idx)]

        trainset_y = y_train.iloc[list(train_idx)]
        validset_y = y_train.iloc[list(valid_idx)]

        trainset.loc[:, 'label'] = list(trainset_y)
        validset.loc[:, 'label'] = list(validset_y)

        validscore = md.callmodel(trainset, model, separatetestset=True, testsetdf=validset, all=True)
        list_score.append(validscore)

        testsetscore = md.callmodel(trainset, model, separatetestset=True, testsetdf=testset, all=True)
        list_testsetscore.append(testsetscore)

outputdf = pd.DataFrame({'list_score': list_score, 'list_testsetscore': list_testsetscore })
outputdf.to_csv('modelselectionresult_2k_norm_pt_weight_gene_top20.csv', index=False)

print("Time taken :", timeit.default_timer() - starting_time)


import logging
import pytest

from mindarmour.utils.logger import LogUtil

@pytest.mark.level0
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_card
@pytest.mark.component_mindarmour
def test_logger():
    logger = LogUtil.get_instance()
    tag = 'Test'
    msg = 'i am %s, work in %s'
    args = ('tom', 'china')
    logger.debug(tag, msg, *args)
    logger.info(tag, msg, *args)
    logger.error(tag, msg, *args)
    logger.warn(tag, msg, *args)

    logger.set_level(logging.DEBUG)

    logger.debug(tag, msg, *args)
    logger.info(tag, msg, *args)
    logger.error(tag, msg, *args)
    logger.warn(tag, msg, *args)

    msg = 'i am tom, work in china'
    logger.info(tag, msg)

    logger.info(tag, 'accuracy is %f.', 0.995)
    logger.debug(tag, 'accuracy is %s.', 0.995)

@pytest.mark.level0
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_card
@pytest.mark.component_mindarmour
def test_value_error():
    with pytest.raises(SyntaxError) as e:
        assert LogUtil()
    assert str(e.value) == 'can not instance, please use get_instance.'
    logger = LogUtil.get_instance()
    handler = 'logging.Handler(level=1)'
    with pytest.raises(ValueError):
        logger.add_handler(handler)


import os
import sys
import numpy as np
import pandas as pd
from datetime import date
from collections import OrderedDict
from dateutil.relativedelta import *
import matplotlib.pyplot as plt
from IPython.core.pylabtools import figsize
from IPython.core.display import display, HTML
from IPython.display import Latex

sys.path = ["./"]+sys.path

import fingenerators as fingen

# ============================================================================
def convertcompounded(n,m,imc,returnstr=False):
    inc = n * ( (1 + imc/m) ** (m/n) -1 )
    if returnstr:
        inc = f'Effective x{n} rate of {imc} compounded x{m} is {inc}'
    return inc

# ============================================================================
def plot_amort_balance(schedules,scenarios):
    """Plot the remaining  balance of all amortisation scenarios
    """
    figsize(12,8)
    fig, ax = plt.subplots(1, 1)
    for scenario in scenarios.keys():
        schedules[scenario].plot(x='Month', y='End Balance', label=f'{scenario}', ax=ax)
    plt.title("Repayment Timelines");
    plt.ylabel("Balance");

# ============================================================================
def plot_amort_annual_interest(schedules,scenarios, stats):
    """Plot the annual interest of all amortisation scenarios
    """
    ys = {}
    labels = {}
    header = {}
    for scenario in scenarios.keys():
        ys[scenario],labels[scenario],header[scenario] = df_annual_amort_interest(schedules[scenario], stats[scenario])

    dfai = pd.concat([ys[scenario] for scenario in ys.keys()], axis=1)
    dfai.columns = header.keys()

    figsize(12,8)
    fig, ax = plt.subplots(1, 1)
    for header in header.keys():
        dfai.plot(y=header, label=f'{header}', ax=ax)
    plt.legend([labels[scenario] for scenario in labels.keys()], loc=1, prop={'size':10})
    plt.title("Interest Payments");

# ============================================================================
def df_annual_amort_interest(schedule, stats):
    """Create a dataframe with annual amortisation interest totals, and a descriptive label
    """
    annint = schedule.set_index('Month')['Interest'].resample("A").sum().reset_index()
    annint["Year"] = annint["Month"].dt.year
    annint.set_index('Year', inplace=True)
    annint.drop('Month', 1, inplace=True)
    label="{} years at {}% with additional payment of {:.0f}".format(stats['BondYears'], stats['Interest Rate']*100, stats['AddPayment'])
    header = f'{stats["AddPayment"]}'
    return annint, label, header

# ============================================================================
def calc_scenarios(scenarios,cyclesPerAnnum=12,paymentSign=1):
    """Given a scenario dictionary calculate bond schedules and statistics
    """
    schedules = {}
    stats = {}

    for scenario in scenarios.keys():
        if 'reqPayment' not in scenarios[scenario].keys():
            scenarios[scenario]['reqPayment'] = \
                paymentSign * round(np.pmt(scenarios[scenario]['intr'] / cyclesPerAnnum,
                              scenarios[scenario]['years'] * cyclesPerAnnum,
                              scenarios[scenario]['princ']), 2);

        schedules[scenario], stats[scenario] = fingen.amortisation_table(
                        scenarios[scenario]['princ'],
                        scenarios[scenario]['intr'],
                        scenarios[scenario]['years'],
                        scenarios[scenario]['reqPayment'],
                        scenarios[scenario]['addPayment'],
                        cyclesPerAnnum=12
                    );
    return schedules,stats

# ============================================================================
# to evaluate the tax benefits
def bondtaxsavingsanalysis(principal,interest_rate,bondyears,taxrate,rentpmonth,
                           increasepyear,cyclesPerAnnum=12,addpayment=0,addpayrate=0):
    """Calculate a monthly schedule and summary of tax benefit on bond loan
    """

    reqpayment = round(np.pmt(interest_rate/cyclesPerAnnum, bondyears*cyclesPerAnnum, principal), 2)

    # calculate the mortgage
    df, stats = fingen.amortisation_table(
        principal=principal,
        interest_rate=interest_rate,
        bondyears=bondyears,
        reqpayment = reqpayment,
        addpayment=addpayment,
        cyclesPerAnnum=cyclesPerAnnum,
        start_date=date(2000, 1,1),
        addpayrate=addpayrate)

    numcycles = stats['Num Payments']
    rischedule = fingen.annIncreaseTable(value=rentpmonth, increasepyear=increasepyear, numcycles=numcycles)
    # Now merge the bond table with the rent income table
    dfc = df.copy().drop(["ReqPayment","AddPayment","Begin Balance"],axis=1)
    dfc = dfc.merge(rischedule.drop(["Month"],axis=1), on='Period')

    # number of payments
    dfc['NumPay'] = stats['Num Payments']

    # save tax for later
    dfc['TaxRate'] = taxrate

    # tax and net income if **NO** bond present
    dfc['TaxNoInter'] = - taxrate * (dfc['Rent'])
    # can't get tax back
    dfc['TaxNoInter'][dfc['TaxNoInter'] > 0] = 0
    dfc['IncomeNoInter'] = dfc['Rent'] + dfc['TaxNoInter']

    # tax and net income if bond present
    dfc['TaxWithInter'] = - taxrate * (dfc['Rent'] + dfc['Interest'])
    # can't get tax back
    dfc['TaxWithInter'][dfc['TaxWithInter'] > 0] = 0
    dfc['IncomeWithInter'] = dfc['Rent'] + dfc['TaxWithInter']

    # net benefit
    dfc['BondBenefit'] = dfc['IncomeWithInter'] - dfc['IncomeNoInter']

    #Create a summary statistics table
    nistats = pd.Series([stats['Principal'],stats['Interest Rate'],
                         stats['ReqPayment'],stats['AddPayment'],dfc["Period"].count(),
                         rentpmonth,  increasepyear,
                         taxrate,
                         dfc["Rent"].sum(),
                         dfc["TaxNoInter"].sum(),
                         dfc["IncomeNoInter"].sum(),
                         dfc["TaxWithInter"].sum(),
                         dfc["IncomeWithInter"].sum(),
                         dfc["BondBenefit"].sum(),
                         100 * dfc["BondBenefit"].sum()/dfc["Rent"].sum(),
                        ],
                       index=["Bond","Interest Rate",
                              "ReqPayment","AddPayment","Num Payments",
                              "InitRent",  "RentIncrease",
                              "TaxRate",
                              "Total Rent",
                              "TaxNoInter",
                              "IncomeNoInter",
                              "TaxWithInter",
                              "IncomeWithInter",
                              "BondBenefit",
                              "Benefit/Rent %",
                             ])

    if False:
        figsize(12,8)
        fig, ax = plt.subplots(1, 1)
        # for scenario in scenarios.keys():
        #     schedules[scenario].plot(x='Month', y='End Balance', label=f'{scenario}', ax=ax)
        dfc.plot(x='Month',y='Rent', label='Rent', ax=ax)
        dfc.plot(x='Month',y='Interest', label='Interest', ax=ax)
        dfc.plot(x='Month',y='TaxNoInter', label='TaxNoInter', ax=ax)
        dfc.plot(x='Month',y='TaxWithInter', label='TaxWithInter', ax=ax)
        plt.title("Repayment Timelines");
        plt.ylabel("Value");
        plt.xlabel("Time");

    return dfc,nistats

# ============================================================================
def dispdfTable(dfi,doLaTeXdisplay,decimals=2,index=None, drops=[]):
    """Creates LaTeX or HTML display string
    """
#     print(type(dfi))
    extraVal = 1 if index==True else 0;
    if isinstance(dfi, pd.Series):
        if 'ID' in dfi.index:
            if len(dfi['ID']) == 0:
                drops = drops + ['ID']
        dfi = dfi.drop(drops)

        df = dfi.to_frame()
        tabstr = f'|l*{{{df.shape[1]}}}{{r}}|'
        index = True if index==None else index
        header = False

        df = df.round(decimals)

    else:
        df = dfi
        tabstr = f'*{{{df.shape[1]+extraVal}}}{{|r}}|'
        index = False if index==None else index
        if 'ID' in df.columns:
            if len(df.iloc[0]['ID']) == 0:
                drops = drops + ['ID']
        df = df.drop(drops,axis=1).round(decimals)
        header = True

    if doLaTeXdisplay:
        display(Latex(df.to_latex(index=index,column_format=tabstr,header=header)))
    else:
        display(HTML(df.to_html(index=index,header=header)))


from os import uname
import sys
import time
import requests
from rich.console import Console
from rich.table import Table
from rich.text import Text
from argparse import ArgumentParser, RawTextHelpFormatter
from collections import OrderedDict
from rdflib import ConjunctiveGraph, Namespace, URIRef
from rdflib.namespace import RDF, RDFS, OWL
import jellyfish

def download_EDAM():
    r = requests.get("http://edamontology.org/EDAM.owl")
    with open("EDAM.owl", "wb") as f:
        f.write(r.content)

# a single function to load EDAM and get the graph object as a result
def load_EDAM():

    g = ConjunctiveGraph()
    try:
        g.load("./EDAM.owl", format="xml")
    except FileNotFoundError:
        download_EDAM()
        g.load("./EDAM.owl", format="xml")
    g.bind("edam", Namespace("http://edamontology.org#"))
    g.bind("oboInOwl", Namespace("http://www.geneontology.org/formats/oboInOwl#"))
    return g

class LimitedSizeDict(OrderedDict):
    def __init__(self, *args, **kwds):
        self.size_limit = kwds.pop("size_limit", None)
        OrderedDict.__init__(self, *args, **kwds)
        self._check_size_limit()

    def __setitem__(self, key, value):
        OrderedDict.__setitem__(self, key, value)
        self._check_size_limit()

    def _check_size_limit(self):
        if self.size_limit is not None:
            while len(self) > self.size_limit:
                self.popitem(last=False)

def index_EDAM(edam_kg):
    index_label_to_uri = {}
    for subject, predicate, obj in edam_kg.triples((None, RDFS.label, None)):
        index_label_to_uri[str(obj)] = str(subject)

    for subject, predicate, obj in edam_kg.triples(
        (
            None,
            URIRef("http://www.geneontology.org/formats/oboInOwl#hasExactSynonym"),
            None,
        )
    ):
        index_label_to_uri[str(obj)] = str(subject)
    return index_label_to_uri

def get_edam_top_10_jaro(l, index):
    max_sim = 0
    min_dist = 100
    top_10 = LimitedSizeDict(size_limit=5)
    for label in index.keys():
        s = jellyfish.jaro_winkler(l, label)
        if s > max_sim:
            max_sim = s
            top_10[label] = s
            top_10.move_to_end(label, last=False)
    return top_10

def get_edam_top_10_jaro_ci(l, index):
    max_sim = 0
    min_dist = 100
    top_10 = LimitedSizeDict(size_limit=5)
    for label in index.keys():
        s = jellyfish.jaro_winkler(l.lower(), label.lower())
        if s > max_sim:
            max_sim = s
            top_10[label] = s
            top_10.move_to_end(label, last=False)
    return top_10

def get_edam_top_10(l, index):
    max_sim = 0
    min_dist = 100
    top = {}
    for label in index.keys():
        top[label] = jellyfish.jaro_winkler(l.lower(), label.lower())

    sorted_hits = dict(sorted(top.items(), key=lambda item: item[1], reverse=True))
    top_10 = {k: sorted_hits[k] for k in list(sorted_hits)[:10]}

    return top_10

parser = ArgumentParser(
    description="""
text2edam recommend EDAM term URIs based on syntactic similarity.

Usage examples :
    python text2edam.py -l

Please report any issue to alban.gaignard@univ-nantes.fr
""",
    formatter_class=RawTextHelpFormatter,
)

parser.add_argument(
    "-u",
    "--update",
    help="download or update the EDAM ontology",
)

parser.add_argument(
    "labels",
    metavar="label",
    type=str,
    nargs="+",
    help="input labels",
)

if __name__ == "__main__":

    if len(sys.argv) == 1:
        parser.print_help(sys.stderr)
        sys.exit(1)

    args = parser.parse_args()
    console = Console()

    table = Table(show_header=True, header_style="bold magenta")
    table.add_column("Label", justify="left")
    table.add_column("Matched term", justify="right")
    table.add_column("Confidence score", justify="right")
    table.add_column("Term URI", justify="right", style="green")

    if args.update:
        download_EDAM()
    elif args.labels:
        s1 = time.time()
        G = load_EDAM()
        ts1 = round((time.time() - s1), 2)
        console.print(f"Loaded {len(G)} triples from EDAM")
        console.print(f"EDAM ontology loaded in {ts1} s")

        s2 = time.time()
        index = index_EDAM(G)
        ts2 = round((time.time() - s2), 2)
        console.print(f"EDAM ontology indexed in {ts2} s")

        s3 = time.time()
        for label in args.labels:
            # top_10 = get_edam_top_10_jaro_ci(label, index=index)
            top_10 = get_edam_top_10(label, index=index)
            for hit in top_10.keys():
                table.add_row(
                    label,
                    hit,
                    str(top_10[hit]),
                    f"[link={index[hit]}]{index[hit]}[/link]",
                )
        ts3 = round((time.time() - s3), 2)
        console.print(f"Approximate term search in {ts3} s")
        console.print(table)


import numpy as np
import matplotlib.pyplot as plt
import pickle
import random as rd
import compute_grad_num as num_grad
import pandas as pd

# Set random seed to get reproducable results
np.random.seed(400)

"Loads the CIFAR-10 data"
# returns:
#	X = image pixel data (d x N)
#	Y = one-hot class representation for each image (K x N)
# 	y = label for each image 0-9 (1 x N)
def read_data(file_name):
	path = 'Datasets/cifar-10/'
	file = path + file_name
	with open(file, 'rb') as fo:
		dict = pickle.load(fo, encoding='bytes')

	X = dict[b'data']
	#transpose X to get d x N matrix
	X = X.T
	# normalize to get pixel values in [0,1]
	X = X / 255

	# zero center the data
	x_mean = np.mean(X, axis=1)
	x_mean = x_mean.reshape((len(x_mean), 1))
	X = X - x_mean

	y = np.array(dict[b'labels'])

	#get one-hot encoding of labels
	Y = np.zeros((10, len(X[0])))
	for i, label in enumerate(y):
		Y[label][i] = 1

	return np.array(X), np.array(Y), np.array(y)

"Init model parameters"
# initialize weights and biases to have Gaussian random values
# with mean = mu and standard deviation = std
# returns:
#	theta = [W_1, b_1, W_2, b_2]
#	W_1 = weight matrices of size m x d
#	W_2 = weight matrices of size K x m
#	b_1 = bias vectors of length m
#	b_2 = bias vectors of length K
def init_model_params(m, d, K, mu = 0, std = 1e-3, xavier = True):

	if(xavier):
		W_1 = np.random.normal(mu, 2/np.sqrt(d), (m,d))
		b_1 = np.random.normal(mu, 2/np.sqrt(m), m)
		W_2 = np.random.normal(mu, 2/np.sqrt(m), (K, m))
		b_2 = np.random.normal(mu, 2/np.sqrt(K), K)
	else:
		W_1 = np.random.normal(mu, std, (m,d))
		b_1 = np.zeros(m)
		W_2 = np.random.normal(mu, std, (K, m))
		b_2 = np.zeros(K)

	theta = [W_1, b_1, W_2, b_2]

	return theta

"Calculate softmax"
# returns:
#	normalized exponential values of vector s
def softmax(s):
	return np.exp(s)/np.sum(np.exp(s), axis=0)

"Perform a forward pass"
# evaluates the network function,
# X = the dataset
# theta = [W_1, b_1, W_2, b_2]
# ----------------------------
# s_1 = W_1X + b_1 (m x N)
# h = max(0, s_1) (ReLu activation function)
# s = W_2h + b2 = ([K x m] x [m x N] => K x N)
# i.e softmax(W_2 x max(0, [WX + b]))
# returns:
#	P - (K x N) matrix of the probabilities for each class
# 	s_1, h for gradient computations
def forward_pass(X, theta):
	b_1 = theta[1].reshape((len(theta[1]), 1))
	b_2 = theta[3].reshape((len(theta[3]), 1))

	s_1 = np.matmul(theta[0], X)
	s_1 = np.add(s_1, b_1)
	h = np.maximum(0, s_1)
	s = np.matmul(theta[2], h)
	s = np.add(s, b_2)
	P = softmax(s)

	return P, h

"Compute the cost function, J (cross entropy loss)"
# returns:
#	J = sum of cross entropy loss for the network
#	+ a l_2-regulizer
# returns:
#	The cost of the model, J
def compute_cost(X, Y, theta, lambda_reg):
	D = len(X[0])
	P, h = forward_pass(X, theta)
	l_cross = 0
	for data_point in range(D):
		l_cross -= np.log(np.dot(Y[:,data_point], P[:,data_point]))

	reg_term_1 = (np.square(theta[0])).sum()
	reg_term_2 = (np.square(theta[2])).sum()
	reg_term = reg_term_1 + reg_term_2

	J = (l_cross/D) + (lambda_reg * reg_term)

	return J

"Compute accuracy"
# calc ratio between correct predictions and total
# number of predictions
# returns:
#	The accuracy of the model, acc (correctly classified/samples)
def compute_accuracy(X, y, theta):
	p, h = forward_pass(X, theta)
	# get columnwise argmax
	p_star = np.argmax(p, axis=0)
	correct = np.sum(p_star == y)
	acc = correct/len(p_star)

	return acc

"Computes the weight and bias gradients"
# returns:
#	The gradients, [grad_W_1, grad_W_2], [grad_b_1, grad_b_2]
def compute_gradients(X, Y, P, theta, h, lambda_reg):

	D = len(X[0])

	grad_W_1 = np.zeros(theta[0].shape)
	grad_b_1 = np.zeros(theta[1].shape)
	grad_W_2 = np.zeros(theta[2].shape)
	grad_b_2 = np.zeros(theta[3].shape)

	for data_point in range(D):
		g = -(Y[:, data_point]-P[:, data_point]).T
		h_i = h[:, data_point]
		h_i = h_i.reshape((1, len(h_i)))
		x_i = X[:, data_point]
		x_i = x_i.reshape((len(x_i), 1))

		grad_b_2 += g

		g = g.reshape((1, len(g)))

		grad_W_2 += np.matmul(g.T, h_i)

		g = np.matmul(g, theta[2])

		h_i[h_i > 0] = 1
		g = np.matmul(g, np.diag(h_i[0]))[0,:]

		grad_b_1 += g
		g = g.reshape((len(g), 1))
		grad_W_1 += np.matmul(g, x_i.T)

	grad_b_1 = (1/D) * grad_b_1
	grad_b_2 = (1/D) * grad_b_2
	grad_W_1 = (1/D) * grad_W_1 + 2*lambda_reg*theta[0]
	grad_W_2 = (1/D) * grad_W_2 + 2*lambda_reg*theta[2]

	return  [grad_W_1, grad_W_2], [grad_b_1, grad_b_2]

"Performs mini-batch gradient decent"
# GD_params = [eta, n_batch, n_epochs]
# X = Y = [train, validation]
# W, b = the initialized weight matrix and bias vector
# lambda_reg = the panalizing factor for l2-regularization
# i = which parameter setting (used for the plotting)
# plot = boolean for plotting
# returns:
#	W_star, b_star = the updated weight matrix and bias vector
def mini_batch_GD(X, Y, GD_params, theta, lambda_reg, i = 1, rho = 0.9, plot = True, momentum = True):
	batches_X, batches_Y = generate_batches(X[0], Y[0], GD_params[1])
	W_star_1 = theta[0]
	b_star_1 = theta[1]
	W_star_2 = theta[2]
	b_star_2 = theta[3]

	decay_rate = 0.95
	eta = GD_params[0]

	mom_W_1 = np.zeros((W_star_1.shape))
	mom_b_1 = np.zeros((b_star_1.shape))
	mom_W_2 = np.zeros((W_star_2.shape))
	mom_b_2 = np.zeros((b_star_2.shape))

	theta_star = [W_star_1, b_star_1, W_star_2, b_star_2]
	if(plot):
		train_cost = np.zeros(GD_params[2] + 1)
		val_cost = np.zeros(GD_params[2] + 1)

	best_theta = theta_star
	min_val = 1000
	check = 0

	for epoch in range(GD_params[2]):
		if(epoch % 10 == 0):
			print("epoch: ", epoch)

		if(plot):
			train_cost[epoch] = compute_cost(X[0], Y[0], theta_star, lambda_reg)
			val_cost[epoch] = compute_cost(X[1], Y[1], theta_star, lambda_reg)

		for batch in range(GD_params[1]):
			X_batch = batches_X[:,:,batch].T
			Y_batch = batches_Y[:,:,batch].T

			P, h = forward_pass(X_batch, theta_star)

			grad_W, grad_b = compute_gradients(X_batch, Y_batch, P, theta_star, h, lambda_reg)

			if(momentum):
				mom_W_1 = (rho * mom_W_1) + (GD_params[0] * grad_W[0])
				mom_b_1 = (rho * mom_b_1) + (GD_params[0] * grad_b[0])
				mom_W_2 = (rho * mom_W_2) + (GD_params[0] * grad_W[1])
				mom_b_2 = (rho * mom_b_2) + (GD_params[0] * grad_b[1])

				W_star_1 = W_star_1 - mom_W_1
				W_star_2 = W_star_2 - mom_W_2
				b_star_1 = b_star_1 - mom_b_1
				b_star_2 = b_star_2 - mom_b_2
			else:
				W_star_1 = W_star_1 - (GD_params[0] * grad_W[0])
				W_star_2 = W_star_2 - (GD_params[0] * grad_W[1])
				b_star_1 = b_star_1 - (GD_params[0] * grad_b[0])
				b_star_2 = b_star_2 - (GD_params[0] * grad_b[1])

			theta_star = [W_star_1, b_star_1, W_star_2, b_star_2]

		v_cost = compute_cost(X[1], Y[1], theta_star, lambda_reg)

		if(v_cost < min_val):
			min_val = v_cost
			best_theta = theta_star
			check = 0
		else:
			check += 1

		if(check == 5):
			eta = eta/2
		elif(check > 10):
			return best_theta

		if(plot):
			t_cost = compute_cost(X[0], Y[0], theta_star, lambda_reg)
			#v_cost = compute_cost(X[1], Y[1], theta_star, lambda_reg)

			# If weights are very small => log(0) in compute cost => inf/NaN
			if(np.isnan(t_cost) or np.isinf(t_cost)):
				t_cost = train_cost[epoch - 1]
			if(np.isnan(v_cost) or np.isinf(v_cost)):
				v_cost = val_cost[epoch - 1]

			train_cost[epoch + 1] = t_cost
			val_cost[epoch + 1] = v_cost

		if(momentum):
			eta = eta * decay_rate

	if(plot):
		plot_cost(train_cost, val_cost, i, GD_params[0])

	return best_theta

"Generates the batches to use for mini-batch GD"
# X, Y = the data and labels (one-hot encoded)
# n_batch = how many batches to use
# returns:
#	batches_X,  batches_Y = arrays containging the batches
def generate_batches(X, Y, n_batch):
	batch_size = int(len(X[0])/n_batch)

	batches_X = np.zeros((batch_size, len(X), n_batch))
	batches_Y = np.zeros((batch_size, len(Y), n_batch))

	for i in range(batch_size):
		start = i*n_batch
		end = (i+1)*n_batch
		batches_X[i] = X[:,start:end]
		batches_Y[i] = Y[:,start:end]

	return batches_X, batches_Y

"Compares the analytical and numerical gradients"
# X, Y = the data and labels (one-hot encoded)
# lambda_reg = the panalizing factor for l2-regularization
# h = the small shift used for numerical gradients
# slow = boolean for using the slow numerical gradient
# check_size = number of data points used for computing the gradients
def check_grad(X, Y, lambda_reg, h_diff = 1e-5, check_size = 10, dim_size = 3072):
	X = X[:dim_size,:check_size]
	Y = Y[:dim_size,:check_size]

	K = len(Y)
	d = len(X)
	m = 50

	theta = init_model_params(m, d, K)

	num_grad_W, num_grad_b = grad_slow(X, Y, theta, lambda_reg, h_diff)

	P, h = forward_pass(X, theta)
	grad_W, grad_b = compute_gradients(X, Y, P, theta, h, lambda_reg)
	epsilon = 1e-10

	comp_W_1 = np.zeros(theta[0].shape)
	comp_b_1 = np.zeros(theta[1].shape)
	comp_W_2 = np.zeros(theta[2].shape)
	comp_b_2 = np.zeros(theta[3].shape)

	abs_W_1 = np.zeros(theta[0].shape)
	abs_b_1 = np.zeros(theta[1].shape)
	abs_W_2 = np.zeros(theta[2].shape)
	abs_b_2 = np.zeros(theta[3].shape)

	comp_W = [comp_W_1, comp_W_2]
	comp_b = [comp_b_1, comp_b_2]

	abs_W = [abs_W_1, abs_W_2]
	abs_b = [abs_b_1, abs_b_2]

	errors = np.zeros((6,2))

	for layer in range(2):
		for i in range(len(comp_W[layer])):
			for j in range(len(comp_W[layer][0])):
				comp_W[layer][i][j] = abs(grad_W[layer][i][j] - \
					num_grad_W[layer][i][j])/max(epsilon, abs(grad_W[layer][i][j]) \
					 + abs(num_grad_W[layer][i][j]))
				abs_W[layer][i][j] = abs(num_grad_W[layer][i][j]) - abs(grad_W[layer][i][j])

		for i in range(len(comp_b[layer])):
			comp_b[layer][i] = abs(num_grad_b[layer][i] - \
				grad_b[layer][i])/max(epsilon, abs(num_grad_b[layer][i]) + abs(grad_b[layer][i]))
			abs_b[layer][i] = abs(num_grad_b[layer][i]) - abs(grad_b[layer][i])

		print("layer: ", layer + 1)
		for i in range(len(num_grad_b[layer])):
			print("n: ", num_grad_b[layer][i], "a: ", grad_b[layer][i])

		tol_error = 1e-6 if layer == 0 else 1e-6

		print("---------------", layer + 1, "---------------")
		print("max relError W", layer + 1, ":", np.max(comp_W[layer]))
		print("min relError W", layer + 1, ":", np.min(comp_W[layer]))
		print("max relError b", layer + 1, ":", np.max(comp_b[layer]))
		print("min relError b", layer + 1, ":", np.min(comp_b[layer]))
		print("Tollerated error: ", tol_error)
		print("# wrong W", layer + 1, ":", np.sum(comp_W[layer] > tol_error), "/", comp_W[layer].size)
		print("# wrong b", layer + 1, ":", np.sum(comp_b[layer] > tol_error), "/", comp_b[layer].size)

"Compute numerical gradients (SLOW)"
def grad_slow(X, Y, theta, lambda_reg, h=1e-6):

	grad_W_1 = np.zeros(theta[0].shape)
	grad_b_1 = np.zeros((len(theta[1]), 1))
	grad_W_2 = np.zeros(theta[2].shape)
	grad_b_2 = np.zeros((len(theta[3]), 1))

	for i in range(len(theta[1])):
		b_try = np.copy(theta[1])
		b_try[i] = b_try[i] - h
		c1 = compute_cost(X, Y, [theta[0], b_try, theta[2], theta[3]], lambda_reg)

		b_try = np.copy(theta[1])
		b_try[i] = b_try[i] + h
		c2 = compute_cost(X, Y, [theta[0], b_try, theta[2], theta[3]], lambda_reg)

		grad_b_1[i] = (c2-c1) / (2*h)

	for i in range(len(theta[3])):
		b_try = np.copy(theta[3])
		b_try[i] = b_try[i] - h
		c1 = compute_cost(X, Y, [theta[0], theta[1], theta[2], b_try], lambda_reg)

		b_try = np.copy(theta[3])
		b_try[i] = b_try[i] + h
		c2 = compute_cost(X, Y, [theta[0], theta[1], theta[2], b_try], lambda_reg)

		grad_b_2[i] = (c2-c1) / (2*h)

	for i in range(len(theta[0])):
		for j in range(len(theta[0][0])):
			W_try = np.copy(theta[0])
			W_try[i][j] = W_try[i][j] - h
			c1 = compute_cost(X, Y, [W_try, theta[1], theta[2], theta[3]], lambda_reg)

			W_try = np.copy(theta[0])
			W_try[i][j] = W_try[i][j] + h;
			c2 = compute_cost(X, Y, [W_try, theta[1], theta[2], theta[3]], lambda_reg);

			grad_W_1[i][j] = (c2-c1) / (2*h);

	for i in range(len(theta[2])):
		for j in range(len(theta[2][0])):
			W_try = np.copy(theta[2])
			W_try[i][j] = W_try[i][j] - h
			c1 = compute_cost(X, Y, [theta[0], theta[1], W_try, theta[3]], lambda_reg)

			W_try = np.copy(theta[2])
			W_try[i][j] = W_try[i][j] + h;
			c2 = compute_cost(X, Y, [theta[0], theta[1], W_try, theta[3]], lambda_reg);

			grad_W_2[i][j] = (c2-c1) / (2*h);

	return [grad_W_1, grad_W_2], [grad_b_1, grad_b_2]

"Plots the training and validation cost as a function of epochs"
def plot_cost(train_cost, val_cost, eta, ind = 1):
	colors = ["green", "red", "yellow", "blue", "black"]
	plt.xlabel("Epochs")
	plt.ylabel("Cost")
	epochs = len(train_cost)
	X = np.linspace(0,epochs,epochs)
	#plt.axis([0, epochs, 1, 3])
	plt.plot(X, train_cost, color = "green", label="Training")
	plt.plot(X, val_cost, color = "red", label="Validation")
	plt.legend()
	plt.savefig("cost_plot_ALL_30E" + str(ind) + ".png")
	plt.close()

"Visualizes the final weight representations"
def plot_weights(W, ind):
	plt.figure(figsize=(8.4, 2))
	for i, w in enumerate(W):
		plt.subplot(1, 10, i + 1)
		im = w.reshape((3, 32, 32))
		im = im.transpose((1,2,0))
		im = ((im - np.min(im))/(np.max(im) - np.min(im)))
		plt.imshow(im)
		plt.xticks(())
		plt.yticks(())
	plt.suptitle("Visualization of the weight matrix", fontsize=16)
	plt.subplots_adjust(0.08, 0.02, 0.92, 0.85, 0.08, 0.23)
	plt.savefig("weight_plot_" + str(ind) + ".png")
	plt.close()
	print("Weight visualization", ind, "completed")

def grad_check():
	X, Y, train_y = read_data('data_batch_1')
	lambda_reg = 0
	check_grad(X, Y, lambda_reg, check_size = 10, dim_size=100)

def gen_rand_etas(e_min, e_max, samples):
	e = e_min + (e_max - e_min) * np.random.uniform(0,1,samples)
	etas = np.power(10, e)

	return np.sort(etas)

def gen_rand_lambdas(e_min, e_max, samples):
	e = e_min + (e_max - e_min) * np.random.uniform(0,1,samples)
	lambdas = np.power(10, e)

	return np.sort(lambdas)

def main():
	# train_X, train_Y, train_y = read_data('data_batch_1')
	# val_X, val_Y, val_y = read_data('data_batch_2')
	# test_X, test_Y, test_y = read_data('test_batch')

	#Use all batches of data
	X_1, Y_1, y_1 = read_data('data_batch_1')
	X_2, Y_2, y_2 = read_data('data_batch_2')
	X_3, Y_3, y_3 = read_data('data_batch_3')
	X_4, Y_4, y_4 = read_data('data_batch_4')
	X_5, Y_5, y_5 = read_data('data_batch_5')
	#val_X, val_Y, val_y = read_data('data_batch_2')
	test_X, test_Y, test_y = read_data('test_batch')

	val_for_train = 0

	train_X = np.array([np.concatenate((X_1, X_2, X_3, X_4, X_5[:,:val_for_train]), axis = 1)]).reshape((3072, (40000 + val_for_train)))
	train_Y = np.array([np.concatenate((Y_1, Y_2, Y_3, Y_4, Y_5[:,:val_for_train]), axis = 1)]).reshape((10, (40000 + val_for_train)))
	val_X = X_5[:,val_for_train:]
	val_Y = Y_5[:,val_for_train:]

	K = len(train_Y)
	d = len(train_X)
	m = 50

	# theta = [W_1, b_1, W_2, b_2]
	theta = init_model_params(m, d, K, xavier = False)

	# Use only a subset of batch_1
	# subset_size = 100
	# train_X = train_X[:, :subset_size]
	# train_Y = train_Y[:, :subset_size]
	# val_X = val_X[:, :subset_size]
	# val_Y = val_Y[:, :subset_size]

	X = [train_X, val_X]
	Y = [train_Y, val_Y]

	n_epochs = 80
	n_batch = 100

	# lambdas = gen_rand_lambdas(-6, -4, 10)
	# etas = gen_rand_etas(-1.7, -1.15, 10)

	lambdas = [1e-5]
	etas = [0.03]

	accuracy = np.zeros((len(lambdas), len(etas)))

	for lamb in range(len(lambdas)):
		print("lambda: ", lamb + 1, "/", len(lambdas))
		for e in range(len(etas)):
			print("eta: ", e + 1, "/", len(etas))
			lambda_reg = lambdas[lamb]
			eta = etas[e]
			GD_params = [eta, n_batch, n_epochs]

			P, h = forward_pass(train_X, theta)

			theta_star = mini_batch_GD(X, Y, GD_params, theta, lambda_reg)
			v_acc = compute_accuracy(test_X, test_y, theta_star)
			print("acc:", v_acc)
			accuracy[lamb][e] = v_acc

	print("Accuracy:")
	for i, lamb in enumerate(lambdas):
		print("------ lambda = ", lamb, "-----------")
		for j, eta in enumerate(etas):
			print("eta = ", eta, " accuracy = ", accuracy[i][j])

	# df_acc = pd.DataFrame(accuracy, index=lambdas, columns=etas)
	# df_acc.to_csv("grid_search.csv", sep = ',')

main()
#grad_check()


from sklearn import datasets
import numpy as np
import collections
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier

irirs = datasets.load_iris()
irirs_data = irirs.data
irirs_label = irirs.target

size = len(irirs_data)

random_irirs_data = np.empty(irirs_data.shape, dtype=irirs_data.dtype)
random_irirs_label = np.empty(irirs_label.shape, dtype=irirs_label.dtype)

index = np.random.permutation(np.arange(size))

# random shuffle the origin dataset to cross-validation
for i in range(len(index)):
    random_irirs_data[i] = irirs_data[index[i]]
    random_irirs_label[i] = irirs_label[index[i]]

k = 5
intervals = int(size/k)
knn_accuracy, svm_accuracy, DecisionTree_accuracy = 0, 0, 0

for i in range(1, k+1):
    # print("now intervals:", (i-1)*intervals, "to",  i*intervals)
    x_test, y_test = random_irirs_data[(i-1)*intervals:i*intervals], random_irirs_label[(i-1)*intervals:i*intervals]
    x_train = np.concatenate((random_irirs_data[0:(i-1)*intervals], random_irirs_data[i*intervals:size]), axis=0)
    y_train = np.concatenate((random_irirs_label[0:(i-1)*intervals], random_irirs_label[i*intervals:size]), axis=0)

    knn = KNeighborsClassifier(n_neighbors=3, weights = 'uniform', algorithm='auto', leaf_size = 30, p=2,
                               metric = 'minkowski', metric_params=None)
    knn.fit(x_train, y_train)

    knn_result = collections.Counter(knn.predict(x_test) - y_test)
    knn_accuracy += knn_result[0]/intervals

    svm = SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0, decision_function_shape='ovr', degree=3, gamma='auto',
              kernel='rbf', max_iter=-1, probability=False, random_state=None, shrinking=True,tol=0.001, verbose=False)
    svm.fit(x_train, y_train)

    svm_result = collections.Counter(svm.predict(x_test) - y_test)
    svm_accuracy += svm_result[0]/intervals

    DecisionTree = DecisionTreeClassifier()
    DecisionTree.fit(x_train, y_train)

    DecisionTree_result = collections.Counter(DecisionTree.predict(x_test) - y_test)
    DecisionTree_accuracy += DecisionTree_result[0]/intervals

print(knn_accuracy/k)
print(svm_accuracy/k)
print(DecisionTree_accuracy/k)


import discord
from discord.ext import commands
from sys import platform
import os
import subprocess
import time
import threading

#import psutil

"""
def find_a(exe: str, return_arr):
    found = false;
    while not found:
        if return_arr[1] < 0:
            return
        for process in psutil.process_iter():
            if process.name() == exe:
                a_proc = process;
                found = True;
                break;
    a_proc_mem = a_proc.memory_full_info().uss;
    return_arr[0] = a_proc_mem;
    return
"""

def write_from_source(language:str, source:str):
    source_file = os.path.join(os.getcwd(), "compiled", "source.{0}".format(language));
    try:
        with open(source_file, 'w', encoding='utf-8') as f:
            f.write(source);
            f.close();
            return 0;
    except:
        return -1;

def compile_from_file(source):
    pass

def run_from_exe(source):
    pass

@commands.command()
async def run(ctx, language: str, *, code=None):
    output = discord.Embed();
    python = ["py", "python"];
    clang = ["c", "c++", "cpp"];
    in_cpp = language in clang;
    in_py = language in python;
    if (not in_cpp and not in_py):
        output.title = "Writing source failed";
        output.color = discord.Color.red();
        output.add_field(name="Reason :", value="Please specify a supported language");
        await ctx.send(embed=output);
        return -1

    # Strip everything other than the source code
    source = ctx.message.content[5:]
    source = source[len(language)+1:];
    first_tilde = source[:3];
    if (first_tilde == "```"):
        source = source[3:-4];
        char_index = 0;
        while (source[char_index] != "\n"):
            char_index = char_index + 1;
        else:
            source = source[char_index:];
    else:
        while source[0] == "\n":
            source = source[1:];

    # Check for sus codes and set language = filetype
    # I (Joe) run the code on termux, and it doesnt support gcc
    # but my pc does, so it is this way, yeah
    if (in_py):
        compiler = "python3"
        language = "py";
        sus_list = ["import os", "from os", "from pathlib", "import path", "import sys", "from sys"];
        for sus in sus_list:
            if sus in source:
                output.title = "Writing source failed";
                output.color = discord.Color.red();
                output.add_field(name="Reason :",value="Risky libraries are not allowed");
                await ctx.send(embed=output);
                return -1;

    elif (in_cpp):
        if (language == "c"):
            language = "c";
            if platform == "win32":
                compiler = "gcc";
            else:
                compiler = "clang";
        else:
            language = "cpp";
            if platform == "win32":
                compiler = "g++";
            else:
                compiler = "clang++";

    write_src_error = write_from_source(language, source);
    if (write_src_error == -1):
        output.title = "Writing source failed";
        output.color = discord.Color.dark_red();
        output.description = "Something unknown went wrong!";
        await ctx.send(embed=output);
        return -1;

    try:
        if platform == "win32":
            file = "a.exe";
        else:
            file = "a.out";
        os.remove(file);
    except:
        pass

    # compilation and running
    try:
        prev_time = time.time();
        source_file = os.path.join(os.getcwd(), "compiled", "source.{0}".format(language));
        compile_process = subprocess.run([compiler, source_file], stderr=subprocess.PIPE, text=True, timeout=15, stdout=subprocess.PIPE);
        compile_time = time.time() - prev_time;
        if (compile_process.returncode != 0):
            error = discord.Embed();
            error.title = "Compilation not successful";
            error.set_footer(text=f"Compile time : {compile_time}s");
            error.color = discord.Color.red();
            error.add_field(name="stderr :", value=f"```{compile_process.stderr} ```");
            stdout = compile_process.stdout;
            if stdout != None:
                error.add_field(name="stdout :", value=f"```{compile_process.stderr} ```");
            await ctx.send(embed=error);
        else:
            # Python is interpreted
            if language == "py":
                output.title = "Running completed";
                output.color = discord.Color.green();
                output.add_field(name="Output:", value=f"```{compile_process.stdout} ```");
                output.set_footer(text=f"Runtime : {compile_time}s\nMemory : KB");
                await ctx.send(embed=output);
                return 0;

            output.title = "Compilation completed";
            if platform == "win32":
                file = "a.exe";
                command = ".\\" + file;
            else:
                file = "a.out";
                command = "./" + file;
            memory = [0, 1];
            #pid = threading.Thread(target=find_a, args=[file, memory],daemon=True);
            try:
                #pid.start();
                prev_time = time.time();
                run_process = subprocess.run(command, stderr=subprocess.PIPE, text=True, timeout=45, stdout=subprocess.PIPE);
                run_time = time.time() - prev_time;
                # thread should stop
                memory[1] = -1;
                if (run_process.returncode != 0):
                    warning = discord.Embed();
                    warning.title = "Compilation completed";
                    warning.color = discord.Color.light_gray();
                    warning.description = f"Error while running : returncode ({run_process.returncode})";
                    warning.add_field(name="stderr :",value=f"```{run_process.stderr} ```");
                    warning.add_field(name="stdout :", value=f"```{run_process.stdout} ```");
                    warning.set_footer(text=f"Compile time : {compile_time}s\nRuntime : {run_time}s");
                    await ctx.send(embed=warning);
                else:
                    output.description = "Running completed";
                    output.color = discord.Color.green();
                    output.add_field(name="Output :",value=f"```{run_process.stdout} ```");
                    if memory[0] == 0:
                        membytes = "-";
                    else:
                        membytes = memory[0];
                    output.set_footer(text=f"Compile time : {compile_time}s\nRuntime : {run_time}s\nMemory : {membytes} bytes");
                    await ctx.send(embed=output);
                return 0;

            except subprocess.TimeoutExpired as e:
                error = discord.Embed();
                error.title = "Running failed!";
                error.color = discord.Color.red();
                error.add_field(name="Reason :", value="Runtime took to long! The limit is 45 seconds!");
                await ctx.send(embed=error);
                return -1;
            except Exception as e:
                error_str = str(e);
                error = discord.Embed();
                error.title = "Running failed!";
                error.add_field(name="Reason :", value=f"```{error_str} ```");
                error.color = discord.Color.red();
                await ctx.send(embed=error);
                return -1;

    except subprocess.TimeoutExpired as e:
        error = discord.Embed();
        error.title = "Compilation failed";
        error.color = discord.Color.red();
        error.add_field(name="Reason :", value="Compilation took to long! The limit is 15 seconds!");
        await ctx.send(embed=error);
    except Exception as e:
        error = discord.Embed();
        error.title = "Compilation failed";
        error_str = str(e);
        error.color = discord.Color.red();
        error.add_field(name="Reason :", value=f"```{error_str} ```");
        await ctx.send(embed=error);
    finally:
        return -1;

@run.error
async def run_error(ctx, error):
    if isinstance(error, commands.BadArgument):
        output = discord.Embed(title="Command Error", description="Wrong arguments, to run code, type `;run <language> <code>`", color=discord.Color.red());
        await ctx.send(embed=output);
        return
    print(error);

def setup(bot):
  bot.add_command(run);


#!/usr/bin/env python3

import sys
import colorsys
from colorz import colorz

WALLPAPER = sys.argv[1]

def normalize(hexv, minv=128, maxv=256):
    hexv = hexv[1]
    r, g, b = (hexv[0] / 256.0, hexv[1] / 256.0, hexv[2] / 256.0,)
    h, s, v = colorsys.rgb_to_hsv(r, g, b)
    minv = minv / 256.0
    maxv = maxv / 256.0
    if v < minv:
        v = minv
    if v > maxv:
        v = maxv
    r, g, b = colorsys.hsv_to_rgb(h, s, v)
    return '#{:02x}{:02x}{:02x}'.format(
        int(r * 256), int(g * 256), int(b * 256))

if __name__ == '__main__':
    if len(sys.argv) == 2:
        n = 16
    else:
        n = int(sys.argv[2])

    i = 0
    cz = colorz(WALLPAPER, n=n)

    def tpl2hex(tpl):
        # print(tpl)
        return '#%02x%02x%02x' % (tpl[0], tpl[1], tpl[2])

    palette = [(tpl2hex(tpl[0]), tpl2hex(tpl[1])) for tpl in cz]

    theme = dict(meta=dict(description='Autogenerated theme'))
    theme['colors'] = {}

    i = 0
    for pair in palette:
        for j in range(2):
            name = "color%d%d" % (i, j)
            theme['colors'][name] = tpl2hex(c[j])
        i += 1

    # hard-coded for n = 5
    use_mapping = dict(urgent='color41', focused='color00',)

    import yaml
    with open('out.yml', 'w') as outfile:
        yaml.dump(theme, outfile, default_flow_style=False)
    # print(output)