import math
from collections import defaultdict
from contextlib import contextmanager
from typing import Optional, Union

import torch
from tabulate import tabulate
from torch.utils.data import DataLoader

from dpattack.libs.luna import (Aggregator, CherryPicker, TrainingStopObserver,
                                as_table, cast_list, create_folder_for_file,
                                fetch_best_ckpt_name, idx_to_msk,
                                log_config, ram_pop, ram_write, show_mean_std,
                                time, time_stamp)
from dpattack.libs.luna.public import auto_create
from dpattack.models import PosTagger, WordParser, WordTagParser
from dpattack.models.tagger import PosTagger
from dpattack.task import ParserTask
from dpattack.utils.corpus import Corpus
from dpattack.utils.data import TextDataset, collate_fn
from dpattack.utils.embedding_searcher import (EmbeddingSearcher, cos_dist,
                                               euc_dist)
from dpattack.utils.metric import Metric, ParserMetric
from dpattack.utils.parser_helper import load_parser
from dpattack.utils.tag_tool import gen_tag_dict, train_gram_tagger
from dpattack.utils.vocab import Vocab
import random
from config import Config
from functools import lru_cache
from nltk import TrigramTagger, CRFTagger
from .hack_util import HACK_TAGS, v
from dpattack.libs.nlpaug.augmenter.word import ContextualWordEmbsAug
import numpy as np

class IHack:
    def __init__(self):
        self.config: Config

        self.train_corpus: Corpus
        self.corpus: Corpus

        self.task: ParserTask

        self.__nn_tagger: PosTagger = None
        self.__trigram_tagger: TrigramTagger = None
        self.__crf_tagger: CRFTagger = None
        self.__tag_dict: dict = None
        self.__bert_aug: ContextualWordEmbsAug = None

        self.embed_searcher: EmbeddingSearcher

        self.loader: DataLoader

    @property
    def nn_tagger(self) -> PosTagger:
        if self.__nn_tagger is None:
            self.__nn_tagger = PosTagger.load(
                fetch_best_ckpt_name(self.config.tagger_model))
        return self.__nn_tagger

    @property
    def trigram_tagger(self) -> TrigramTagger:
        if self.__trigram_tagger is None:
            self.__trigram_tagger = auto_create("trigram_tagger",
                                                lambda: train_gram_tagger(
                                                    self.train_corpus, ngram=3),
                                                cache=True, path=self.config.workspace + '/saved_vars')
        return self.__trigram_tagger

    @property
    def crf_tagger(self) -> CRFTagger:
        if self.__crf_tagger is None:
            self.__crf_tagger = CRFTagger()
            self.__crf_tagger.set_model_file(self.config.crf_tagger_path)
        return self.__crf_tagger

    @property
    def tag_dict(self) -> dict:
        if self.__tag_dict is None:
            self.__tag_dict = auto_create("tagdict3",
                                          lambda: gen_tag_dict(
                                              self.train_corpus, self.vocab, 3, False),
                                          cache=True, path=self.config.workspace + '/saved_vars')
            self.__tag_dict = {k: torch.tensor(v)
                               for k, v in self.tag_dict.items()}
        return self.__tag_dict

    @property
    def bert_aug(self) -> ContextualWordEmbsAug:
        if self.__bert_aug is None:
            self.__bert_aug = ContextualWordEmbsAug(model_path=self.config.path,
                                                    top_k=2048)
        return self.__bert_aug

    @property
    def vocab(self) -> Vocab:
        return self.task.vocab

    @property
    def parser(self) -> Union[WordTagParser, WordParser]:
        return self.task.model

    def init_logger(self, config):
        if config.logf == 'on':
            if config.hk_use_worker == 'on':
                worker_info = "-{}@{}".format(config.hk_num_worker,
                                              config.hk_worker_id)
            else:
                worker_info = ""
            log_config('{}{}'.format(config.mode, worker_info),
                       log_path=config.workspace,
                       default_target='cf')
            from dpattack.libs.luna import log
        else:
            log = print

        log('[General Settings]')
        log(config)
        log('[Hack Settings]')
        for arg in config.kwargs:
            if arg.startswith('hk'):
                log(arg, '\t', config.kwargs[arg])
        log('------------------')

    def setup(self, config):
        self.config = config

        print("Load the models")
        vocab = torch.load(config.vocab)  # type: Vocab
        parser = load_parser(fetch_best_ckpt_name(config.parser_model))

        self.task = ParserTask(vocab, parser)

        print("Load the dataset")

        self.train_corpus = Corpus.load(config.ftrain)

        if config.hk_training_set == 'on':
            self.corpus = self.train_corpus
        else:
            self.corpus = Corpus.load(config.fdata)
        dataset = TextDataset(vocab.numericalize(self.corpus, True))
        # set the data loader
        self.loader = DataLoader(dataset=dataset,
                                 collate_fn=collate_fn)

        def embed_backward_hook(module, grad_in, grad_out):
            ram_write('embed_grad', grad_out[0])

        self.parser.embed.register_backward_hook(embed_backward_hook)
        self.parser.eval()

        self.embed_searcher = EmbeddingSearcher(
            embed=self.parser.embed.weight,
            idx2word=lambda x: self.vocab.words[x],
            word2idx=lambda x: self.vocab.word_dict[x]
        )

        random.seed(self.config.seed)
        np.random.seed(self.config.seed)
        torch.manual_seed(self.config.seed)

    def hack(self, instance, **kwargs):
        raise NotImplementedError

    @lru_cache(maxsize=None)
    def _gen_tag_mask(self, tags: tuple, tsr_device, tsr_size):
        word_idxs = []
        for tag in tags:
            if tag in self.tag_dict:
                word_idxs.extend(self.tag_dict[tag])
        legal_tag_index = torch.tensor(word_idxs, device=tsr_device)
        legal_tag_mask = torch.zeros(tsr_size, device=tsr_device)\
            .index_fill_(0, legal_tag_index, 1.).byte()
        return legal_tag_mask

    @lru_cache(maxsize=10)
    def _gen_bert_mask(self, text, idx, tsr_device, tsr_size):
        bert_sub, _ = self.bert_aug.substitute(text, [idx], n=2000)
        bert_sub_idxs = self.vocab.word2id(bert_sub[0]).to(tsr_device)
        bert_mask = torch.zeros(tsr_size, device=tsr_device)\
            .index_fill_(0, bert_sub_idxs, 1.).byte()
        return bert_mask

    @torch.no_grad()
    def find_replacement(self,
                         changed, must_tags, dist_measure,
                         forbidden_idxs__,
                         repl_method='tagdict',
                         words=None, word_sid=None,  # Only need when using a tagger
                         raw_words=None,
                         ) -> (Optional[torch.Tensor], dict):
        if must_tags is None:
            must_tags = tuple(self.vocab.tags)
        if isinstance(must_tags, str):
            must_tags = (must_tags,)

        if repl_method == 'lstm':
            # Pipeline:
            #    256 minimum dists
            # -> Filtered by a NN tagger
            # -> Smallest one
            words = words.repeat(64, words.size(1))
            dists, idxs = self.embed_searcher.find_neighbours(
                changed, 64, dist_measure, False)
            for i, ele in enumerate(idxs):
                words[i][word_sid] = ele
            self.nn_tagger.eval()
            s_tags = self.nn_tagger(words)
            pred_tags = s_tags.argmax(-1)[:, word_sid]
            pred_tags = pred_tags.cpu().numpy().tolist()
            new_word_vid = None
            for i, ele in enumerate(pred_tags):
                if self.vocab.tags[ele] in must_tags:
                    if idxs[i] not in forbidden_idxs__:
                        new_word_vid = idxs[i]
                        break
            return new_word_vid, {"avgd": dists.mean().item(),
                                  "mind": dists.min().item()}
        elif repl_method in ['3gram', 'crf']:
            # Pipeline:
            #    256 minimum dists
            # -> Filtered by a Statistical tagger
            # -> Smallest one
            tagger = self.trigram_tagger if repl_method == '3gram' else self.crf_tagger
            word_texts = self.vocab.id2word(words)
            word_sid = word_sid.item()

            dists, idxs = self.embed_searcher.find_neighbours(
                changed, 64, dist_measure, False)

            cands = []
            for ele in cast_list(idxs):
                cand = word_texts.copy()
                cand[word_sid] = self.vocab.words[ele]
                cands.append(cand)

            pred_tags = tagger.tag_sents(cands)
            s_tags = [ele[word_sid][1] for ele in pred_tags]

            new_word_vid = None
            for i, ele in enumerate(s_tags):
                if ele in must_tags:
                    if idxs[i] not in forbidden_idxs__:
                        new_word_vid = idxs[i]
                        break
            return new_word_vid, {"avgd": dists.mean().item(),
                                  "mind": dists.min().item()}
        elif repl_method in ['tagdict', 'bertag']:
            # Pipeline:
            #    All dists/Bert filtered dists
            # -> Filtered by a tag dict
            # -> Smallest one
            dist = {'euc': euc_dist, 'cos': cos_dist}[
                dist_measure](changed, self.parser.embed.weight)

            # Mask illegal words by its POS
            if repl_method == 'tagdict':
                msk = self._gen_tag_mask(must_tags, dist.device, dist.size())
            elif repl_method == 'bertag':
                msk = self._gen_tag_mask(must_tags, dist.device, dist.size())
                # Mask illegal words by BERT
                bert_mask = self._gen_bert_mask(
                    " ".join(self.vocab.id2word(raw_words)[1:]),
                    word_sid.item() - 1,
                    dist.device, dist.size())
                # Fuck pytorch.
                msk = msk * bert_mask
            else:
                raise Exception

            dist.masked_fill_(1 - msk, 1000.)
            for ele in forbidden_idxs__:
                dist[ele] = 1000.
            mindist = dist.min()
            if abs(mindist - 1000.) < 0.001:
                new_word_vid = None
            else:
                new_word_vid = dist.argmin()
            return new_word_vid, {}
        else:
            raise NotImplementedError


from typing import List, Set, Tuple

from aoc.utils import get_input

AllergenEntry = Tuple[Set[str], Set[str]]
AllergenMap = List[AllergenEntry]

def parse_food(food: str) -> AllergenEntry:
    ingredients_raw, _, allergens_raw = food.partition(' (contains ')
    ingredients = set(ingredients_raw.split())
    allergens = set(allergens_raw[:-1].split(', '))
    return allergens, ingredients

def find_unsafe_ingredients(allergen_map: AllergenMap) -> Set[str]:
    by_allergen = {}
    for allergens, ingredients in allergen_map:
        for allergen in allergens:
            if allergen not in by_allergen:
                by_allergen[allergen] = set(ingredients)
            else:
                by_allergen[allergen].intersection_update(ingredients)
    return set([ingredient
                for ingredients in by_allergen.values()
                for ingredient in ingredients])

def solve_part_one(ingredients_raw: str) -> int:
    allergen_map = [parse_food(food) for food in ingredients_raw.splitlines()]
    unsafe = find_unsafe_ingredients(allergen_map)
    all_ingredients = [ingredient
                       for _, ingredients in allergen_map
                       for ingredient in ingredients]
    safe = set(all_ingredients).symmetric_difference(unsafe)
    return sum([all_ingredients.count(ingredient) for ingredient in safe])

if __name__ == "__main__":
    ingredients_raw = get_input(2020, 21)
    result = solve_part_one(ingredients_raw)
    print(result)


#read write append log file based on arguments we pass in
#data persistance
#with statments are cool bc they automatically close the file
import sys
#datetime method from datetime module:
from datetime import datetime
from time import time

global file_name
global action

def read_log():
	"""Reads from the log file"""
	with open(file_name, 'r') as log_messages:
		print(log_messages.read())

def log(message):
	"""log a message to the log file"""
	"""Arguments: message--any string in command line after calling file"""
	#a appends message, w overwrites, r reads
	with open(file_name, 'a') as logger:
		timestamp = time()
		logger.write("{0}: {1}\n".format(
			datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S'),
			str(message)
			))

if __name__ == '__main__':
    try:
        file_name = sys.argv[1]
        action = sys.argv[2]
        if action == 'r':
                read_log()
        elif action == 'w' or action == 'a':
                log(" ".join(sys.argv[3:]), action)
                read_log()

    except IndexError:
        pass


from datetime import datetime
import time
import sys
counter_compare = 0
counter_replace = 0

server_file = sys.argv[1]

handle = open(server_file, "r")
Heap_array = handle.read()
handle.close()
print(Heap_array)
Heap_array = Heap_array.split()
print(Heap_array)
for i in range(0, len(Heap_array)):
    Heap_array[i] = int(Heap_array[i])
print(type(Heap_array))
print(Heap_array)
print("Array before shell", Heap_array )

start_time = datetime.now()
s = ''' '''
def heapify(arr, n, i):
    # Find largest among root and children
    largest = i
    l = 2 * i + 1
    r = 2 * i + 2

    if l < n and arr[i] < arr[l]:
        largest = l

    if r < n and arr[largest] < arr[r]:
        largest = r

    # If root is not largest, swap with largest and continue heapifying
    if largest != i:
        arr[i],arr[largest] = arr[largest],arr[i]
        heapify(arr, n, largest)

def heapSort(arr):
    n = len(arr)

    # Build max heap
    for i in range(n, 0, -1):
        heapify(arr, n, i)

    for i in range(n-1, 0, -1):
        # swap
        arr[i], arr[0] = arr[0], arr[i]

        #heapify root element
        heapify(arr, i, 0)

arr = Heap_array
heapSort(arr)
print('Array after shaker', Heap_array )
print(" Lenght:", len(Heap_array))
print("Time:")
print(datetime.now() - start_time)
for i in range(0, len(Heap_array)):
    shell_array[i] = str(Heap_array[i])
result =''.join(Heap_array)
handle = open("server_outcoming.txt", "w")
handle.write(result)
handle.close()


import sys
import time
import logging
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler

import main

PATH = "/home/www/uploads"

def on_created(event):
    main.main(f"{PATH}/config.cfg")

if __name__ == "__main__":
    event_handler = FileSystemEventHandler()

    event_handler.on_created = on_created

    observer = Observer()
    observer.schedule(event_handler, PATH, recursive=True)
    observer.start()

    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        observer.stop()
    observer.join()


from PIL import Image
import pytesseract
# image = Image.open('./破解验证码/2.jpg')
image = Image.open('./破解验证码/asd.gif')
image = image.convert('L')

def binarizing(img, threshold):
    # threshold表示阀值
    pixdata = img.load()
    w, h = img.size
    for y in range(h):
        for x in range(w):
            if pixdata[x, y] < threshold:
                pixdata[x, y] = 0
            else:
                pixdata[x, y] = 255
    return img

def depoint(img):
    # input: gray image
    pixdata = img.load()
    w, h = img.size
    for y in range(1, h - 1):
        for x in range(1, w - 1):
            count = 0
            if pixdata[x, y - 1] > 245:
                count = count + 1
            if pixdata[x, y + 1] > 245:
                count = count + 1
            if pixdata[x - 1, y] > 245:
                count = count + 1
            if pixdata[x + 1, y] > 245:
                count = count + 1
            if count > 2:
                pixdata[x, y] = 255
    return img

image = binarizing(image, 140)
image = depoint(image)
print(image.getbbox())
print(pytesseract.image_to_string(image))
print('chen')
image.show()


import numpy as np
import pytest

from neurobiases import TriangularModel, utils
from neurobiases.em_utils import marginal_log_likelihood_linear_tm_wrapper
from numpy.testing import assert_allclose, assert_raises

def test_random_state():
    """Tests that the random seeds correctly set the coupling and tuning
    parameters."""
    tm1 = TriangularModel(coupling_rng=1234, tuning_rng=4321)
    tm2 = TriangularModel(coupling_rng=1234, tuning_rng=9876)
    tm3 = TriangularModel(coupling_rng=9876, tuning_rng=4321)
    tm4 = TriangularModel(coupling_rng=9876, tuning_rng=9876)

    # Random seed for parameters
    assert_allclose(tm1.a.ravel(), tm2.a.ravel())
    assert_raises(AssertionError, assert_allclose, tm1.b.ravel(), tm2.b.ravel())
    assert_allclose(tm3.a.ravel(), tm4.a.ravel())
    assert_raises(AssertionError, assert_allclose, tm3.b.ravel(), tm4.b.ravel())
    assert_allclose(tm1.b.ravel(), tm3.b.ravel())
    assert_allclose(tm1.B.ravel(), tm3.B.ravel())
    assert_raises(AssertionError, assert_allclose, tm1.a.ravel(), tm3.a.ravel())
    assert_allclose(tm2.b.ravel(), tm4.b.ravel())
    assert_allclose(tm2.B.ravel(), tm4.B.ravel())
    assert_raises(AssertionError, assert_allclose, tm2.a.ravel(), tm4.a.ravel())

    # Random seed for data
    X1, Y1, y1 = tm1.generate_samples(n_samples=50, rng=2332)
    X2, Y2, y2 = tm1.generate_samples(n_samples=50, rng=2332)
    assert_allclose(X1, X2)
    assert_allclose(Y1, Y2)
    assert_allclose(y1, y2)

def test_sparsity():
    """Test that the sparsity inputs correctly set the number of non-zero
    parameters."""
    N = 20
    coupling_sparsity = 0.25
    M = 40
    tuning_sparsity = 0.1
    tm = TriangularModel(N=N, coupling_sparsity=coupling_sparsity,
                         M=M, tuning_sparsity=tuning_sparsity,
                         coupling_rng=1234,
                         tuning_rng=4321)
    assert np.count_nonzero(tm.a.ravel() == 0) == int(N * coupling_sparsity)
    assert np.count_nonzero(tm.b.ravel() == 0) == int(M * tuning_sparsity)

def test_neural_variance():
    """Tests that the analytic expressions for the variance of the neural
    activity are correctly calculated."""
    tm1 = TriangularModel(parameter_design='direct_response', N=10, M=20, K=2)
    tm2 = TriangularModel(parameter_design='basis_functions', N=10, M=20, K=2)

    for tm in [tm1, tm2]:
        # Generate samples
        X, Y, y = tm.generate_samples(n_samples=100000, return_noise=False)

        # Non-target signal variance
        non_target_signal = np.dot(X, tm.B)
        empirical_variance = np.var(non_target_signal, axis=0)
        true_variance = tm.non_target_signal_variance(limits=(0, 1))
        assert_allclose(true_variance, empirical_variance, rtol=0.1)

        # Non-target variance
        empirical_variance = np.var(Y, axis=0)
        true_variance = tm.non_target_variance(limits=(0, 1))
        assert_allclose(true_variance, empirical_variance, rtol=0.1)

        # Target signal variance
        target_signal = np.dot(X, tm.b) + np.dot(Y, tm.a)
        empirical_variance = np.var(target_signal)
        true_variance = tm.target_signal_variance(limits=(0, 1))
        assert_allclose(true_variance, empirical_variance, rtol=0.1)

        # Target neuron variance
        empirical_variance = np.var(y)
        true_variance = tm.target_variance(limits=(0, 1))
        assert_allclose(true_variance, empirical_variance, rtol=0.1)

def test_snr():
    """Tests that the total variance of the neural activity reflects the
    signal-to-noise ratio."""
    tm1 = TriangularModel(parameter_design='direct_response', N=10, M=20, K=2)
    tm2 = TriangularModel(parameter_design='basis_functions', N=10, M=20, K=2)

    for tm in [tm1, tm2]:
        snr = tm.noise_kwargs['snr']
        # Generate samples
        X, Y, y = tm.generate_samples(n_samples=100000, return_noise=False)

        # Non-target neural activity
        true_signal_variance = tm.non_target_signal_variance(limits=(0, 1))
        true_total_variance = (1 + 1. / snr) * true_signal_variance
        empirical_variance = np.var(Y, axis=0)
        assert_allclose(true_total_variance, empirical_variance, rtol=0.1)

        # Target neural activity
        true_signal_variance = tm.target_signal_variance(limits=(0, 1))
        true_noise_variance = true_signal_variance / snr
        noise = y - np.dot(X, tm.b) - np.dot(Y, tm.a)
        empirical_noise_variance = np.var(noise)
        assert_allclose(true_noise_variance, empirical_noise_variance, rtol=0.1)

def test_noise_clusters():
    """Test that clustered noise correlations are generated correctly."""
    corr_cluster = 0.3
    # One latent factor
    tm = TriangularModel(parameter_design='direct_response',
                         K=1,
                         corr_cluster=corr_cluster)
    # Compare true and generated correlation matrices
    corr = utils.cov2corr(tm.get_noise_cov())
    true_corr = corr_cluster * np.ones((tm.N + 1, tm.N + 1))
    np.fill_diagonal(true_corr, np.ones(tm.N + 1))
    assert_allclose(corr, true_corr)

    # Three latent factors
    tm = TriangularModel(parameter_design='direct_response',
                         N=6,
                         K=3,
                         corr_cluster=corr_cluster)

    # compare true and generated correlation matrices
    corr = utils.cov2corr(tm.get_noise_cov())
    true_corr = np.array([
        [1, 0, 0, corr_cluster, corr_cluster, 0, 0],
        [0, 1, corr_cluster, 0, 0, 0, 0],
        [0, corr_cluster, 1, 0, 0, 0, 0],
        [corr_cluster, 0, 0, 1, corr_cluster, 0, 0],
        [corr_cluster, 0, 0, corr_cluster, 1, 0, 0],
        [0, 0, 0, 0, 0, 1, corr_cluster],
        [0, 0, 0, 0, 0, corr_cluster, 1]
    ])
    assert_allclose(corr, true_corr, atol=1e-6)

def test_identifiability():
    """Tests that the identifiability transform leaves the likelihood
    unchanged."""
    tm1 = TriangularModel(parameter_design='direct_response', N=10, M=20, K=2)
    tm2 = TriangularModel(parameter_design='basis_functions', N=10, M=20, K=2)

    for tm in [tm1, tm2]:
        # Generate samples
        X, Y, y = tm.generate_samples(n_samples=10000, return_noise=False)
        # Likelihood with original parameters
        pre_ll = marginal_log_likelihood_linear_tm_wrapper(
            X=X, Y=Y, y=y, tm=tm
        )
        # Identifiability transform
        delta = np.random.normal(loc=0, scale=5.0, size=tm.K)
        tm.identifiability_transform(delta)
        # Likelihood with transfored parameters
        post_ll = marginal_log_likelihood_linear_tm_wrapper(
            X=X, Y=Y, y=y, tm=tm
        )
        assert_allclose(pre_ll, post_ll)

def test_identifiability_warning():
    """Tests that a warning is received if the model is not identifiable."""
    # Check that identifiability warning is raised
    with pytest.warns(RuntimeWarning) as record:
        TriangularModel(coupling_sparsity=0., tuning_sparsity=0., K=3)
    assert len(record) == 1
    # Check that identifiability warning is not raised
    with pytest.warns(None) as record:
        TriangularModel(coupling_sparsity=0.5, tuning_sparsity=0.5, K=3)
    assert len(record) == 0


#!/usr/bin/env python3
# rsyslog produces raw stats in a log file in either @CEE or plain JSON.
# E.g.
# `Mon Jul  1 15:31:19 2019: { "name": "imudp(*:10514)", "origin": "imudp", "submitted": 100000 }`
# `Mon Jul  1 15:31:19 2019: { "name": "resource-usage", "origin": "impstats", "utime": 607857, "stime": 630538, "maxrss": 4504, "minflt": 532, "majflt": 0, "inblock": 0, "oublock": 50768, "nvcsw": 139802, "nivcsw": 16, "openfiles": 14 }``

import argparse
import re
import io
import sys
import warnings
import datetime
import json
import collections
import statistics

def datetimes_to_iso(o):
  """
  convert datetime.datetime instances into string representations
  handle some types of collections by recursively walking keys and/or values in a dict or list
  helps avoid json.dumps() 'TypeError: keys must be str, int, float, bool or None, not datetime.datetime'
  """
  if isinstance(o, datetime.datetime):
    return o.isoformat()
  if isinstance(o, list):
    return [datetimes_to_iso(i) for i in o]
  if isinstance(o, dict):
    return {datetimes_to_iso(k): datetimes_to_iso(v) for (k, v) in o.items()}
  if isinstance(o, tuple):
    return tuple(datetimes_to_iso(i) for i in o)
  return o

def prune_none_and_empty(d):
  """
  Remove keys that have either null values, empty strings or empty arrays
  See: https://stackoverflow.com/a/27974027/5472
  """
  if isinstance(d, list):
    return [v for v in (prune_none_and_empty(v) for v in d) if v]
  if isinstance(d, dict):
    return {k: v for k, v in ((k, prune_none_and_empty(v)) for k, v in d.items()) if v}
  return d

parser = argparse.ArgumentParser(description='Process results from rsyslog impstats in JSON format.')
parser.add_argument('-i', '--input-rsyslog-pstats-file', default='-', help='defaults to \'-\' which is <stdout>')
parser.add_argument('-o', '--output-summary-json-file', default='-', help='defaults to \'-\' which is <stdout>')
parser.add_argument('-a', '--aggregate-stats', action='store_true',  help='show overall statistics grouped per metric in the \'agg_stats\' key')
parser.add_argument('-v', '--show-values', action='store_true',  help='list the metric values within the \'agg_stats\' key')
parser.add_argument('-0', '--show-zero-values', action='store_true', help='Show zero or null values within the \'agg_stats\' key')
parser.add_argument('-s', '--original-stats', action='store_true', help='Output a \'original_stats\' key that includes the original stats. Always shows zero values.')
parser.add_argument('-n', '--no-flatten', action='store_true', help='Group stats per timestamp and don\'t flatten output')
parser.epilog='Summarised output is added at the end. The default is to print a timestamp for every stat (flattend) format.'
args = parser.parse_args()

# input
if args.input_rsyslog_pstats_file == '-':
  input_stream = sys.stdin
else:
  input_stream = open(args.input_rsyslog_pstats_file, 'r')

# output
if args.output_summary_json_file == '-':
  output_stream = sys.stdout
else:
  output_stream = open(args.output_summary_json_file, 'w')

METRICS_WHERE_SUM_HAS_MEANING = {
  # queue
  'msgs.received',
  'called.recvmsg',
  'discarded.nf',
  'discarded.full',
  'full',
  # inputs
  # imptcp & imudp
  'submitted',
  # imptcp
  'sessions.opened',
  'sessions.closed',
  'sessions.openfailed',
  'bytes.received',
  'bytes.decompressed',
  # imdup worker thread
  'called.recvmmsg',
  'called.recvmsg',
  'msgs.received',
  # rsyslog action metrics
  'processed',
  'failed',
  'suspended',
  'resumed',
  # rsyslog resource usage related to http://man7.org/linux/man-pages/man2/getrusage.2.html
  'utime',
  'stime',
  'minflt',
  'majflt',
  'inblock',
  'oublock',
  'nvcsw',
  'nivcsw'
}
# some metric names are generated dynamically and can only determined later
dynamic_metrics_where_sum_has_meaning = set()
JSON_ARRAY_OBJECT_START = '{['
JSON_ARRAY_OBJECT_END = ']}'
# Note:
# - The date header of the messages, e.g. `Mon Jul  1 15:31:19 2019:` is a fixed length of 24 char before `:`.
# - @cee format can optionally be in use
PATTERN = '^(?P<str_timestamp>.{24}): (?:@cee: )?(?P<json_stat>\{.*\})$'
re_extract = re.compile(PATTERN)

# Collections to process, aggregate and report results
stats_ordered = collections.OrderedDict()
results = {}

# Note, used regex in case other content is in the file, e.g. shared syslog file.
line_count = 0
for line in input_stream:
  line_count += 1
  matched = re_extract.match(line)
  # skip if match failed
  if not (matched and 'str_timestamp' in matched.groupdict() and 'json_stat' in matched.groupdict()):
    warnings.warn('Skipped line {} of {} because timestamp and/or JSON pattern did not match.'.format(line_count, input_stream.name))
    continue
  # parse
  stat_timestamp = datetime.datetime.strptime(matched['str_timestamp'], '%a %b %d %H:%M:%S %Y')
  json_stat = json.loads(matched['json_stat'])
  if stat_timestamp not in stats_ordered:
    stats_ordered[stat_timestamp] = []
    stats_ordered[stat_timestamp].append(json_stat)
  else:
    stats_ordered[stat_timestamp].append(json_stat)

if not stats_ordered:
  raise ValueError('Unable to match any rsyslog pstats in JSON or CEE format from {}'.format(input_stream.name))

# Process timestamps for each stat
if args.original_stats:
  if args.no_flatten:
    original_stats = stats_ordered
  else:
    original_stats = []
    for k_timestamp, v_stats in stats_ordered.items():
      #for stat in v_stats:
      #  stat['timestamp'] = k_timestamp
      #  stats.append(stat)
      # Create new dict in order cooerse timestamp as the first key
      for stat in v_stats:
        current_stat = {'timestamp': k_timestamp}
        current_stat.update(stat)
        original_stats.append(current_stat)
  results['original_stats'] = original_stats

# Re-organise and group values per stat name, origin and set of metrics in preperation for overall stats calculations later
grouped_stats = {}
for (k_timestamp, v_stats) in stats_ordered.items():
  for stat in v_stats:
    if stat['name'] not in grouped_stats:
      grouped_stats[stat['name']] = {'origin': stat['origin'], 'metrics': {}}
    for k, v in stat.items():
      # only sum when values are numerics
      if isinstance(v, (int, float)):
        if k not in grouped_stats[stat['name']]['metrics']:
          grouped_stats[stat['name']]['metrics'][k] = {'values': {k_timestamp: v}}
        else:
          grouped_stats[stat['name']]['metrics'][k]['values'].update({k_timestamp: v})
      # process dynstats related items which groups items into values and flatten it into the way other stats are structured
      elif (stat['origin'] == 'dynstats.bucket' or stat['origin'] == 'dynstats.bucket') and k == 'values':
        assert isinstance(v, dict), 'dynstats.bucket object imported from JSON has unexpected structure'
        for dk, dv in v.items():
          if dk not in grouped_stats[stat['name']]['metrics']:
            dynamic_metrics_where_sum_has_meaning.add(dk)
            grouped_stats[stat['name']]['metrics'][dk] = {'values': {k_timestamp: dv}}
          else:
            grouped_stats[stat['name']]['metrics'][dk]['values'].update({k_timestamp: dv})

# Calculate stats from grouped metric values
agg_stats = grouped_stats
for (k_gstat, v_gstat) in grouped_stats.items():
  for (k_metric, v_metric) in v_gstat['metrics'].items():
    values_non_zero = prune_none_and_empty(v_metric['values'])
    agg_stats[k_gstat]['metrics'][k_metric].update({'count': len(v_metric['values'])})
    agg_stats[k_gstat]['metrics'][k_metric].update({'count_non_zero': len(values_non_zero)})
    agg_stats[k_gstat]['metrics'][k_metric].update({'min': min(v_metric['values'].values())})
    agg_stats[k_gstat]['metrics'][k_metric].update({'max': max(v_metric['values'].values())})
    agg_stats[k_gstat]['metrics'][k_metric].update({'mean': statistics.mean(v_metric['values'].values())})
    if args.show_zero_values:
      mean_non_zero, t_min, t_max, t_delta_s = None, None, None, None
    if values_non_zero:
      mean_non_zero = statistics.mean(values_non_zero.values())
      # min / max timestamp values assume values dict is still ordered by time
      t_min = list(values_non_zero.keys())[0]
      t_max = list(values_non_zero.keys())[-1]
      t_delta_s = (t_max - t_min).seconds
      agg_stats[k_gstat]['metrics'][k_metric].update({'mean_non_zero': mean_non_zero})
      agg_stats[k_gstat]['metrics'][k_metric].update({'timestamp_non_zero_min': t_min})
      agg_stats[k_gstat]['metrics'][k_metric].update({'timestamp_non_zero_max': t_max})
      agg_stats[k_gstat]['metrics'][k_metric].update({'time_delta_seconds_non_zero': t_delta_s})
    # summations for stats where summation makes sense
    if k_metric in METRICS_WHERE_SUM_HAS_MEANING or k_metric in dynamic_metrics_where_sum_has_meaning:
      agg_stats[k_gstat]['metrics'][k_metric].update({'sum': sum(v_metric['values'].values())})
    if args.show_values:
      if not args.show_zero_values:
        agg_stats[k_gstat]['metrics'][k_metric].update({'values': values_non_zero})
    else:
      del(agg_stats[k_gstat]['metrics'][k_metric]['values'])

if args.aggregate_stats:
  results['agg_stats'] = agg_stats
  t_start = list(stats_ordered.keys())[0]
  t_end = list(stats_ordered.keys())[-1]
  t_runtime_s = (t_end - t_start).seconds
  results['timestamp_start'] = t_start
  results['timestamp_end'] = t_end
  results['runtime_seconds'] = t_runtime_s

# wrap results in a container so it can be combined with outher json sources
results_with_prefix = {'rsyslog_pstats': results}

if output_stream.name == '<stdout>':
  output_stream.write(json.dumps(datetimes_to_iso(results_with_prefix), indent=2))
  output_stream.write('\n')
else:
  output_stream.write(json.dumps(datetimes_to_iso(results_with_prefix), separators=(',', ':')))

# Cleanup
if input_stream.name != '<stdin>':
  input_stream.close()
if output_stream.name != '<stdout>':
  output_stream.close()


# 텐서보드를 사용해서 Projector를 구현할 때 오류가 있음
# 이 오류를 해결하기 위해서 작성해야 할 것
from torch.utils.tensorboard import SummaryWriter
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
import src.utils.interface_file_io as file_io
import numpy as np
import os
import json

# import tensorflow as tf
import librosa
import tensorboard as tb
# tf.io.gfile = tb.compat.tensorflow_stub.io.gfile
# console: tensorboard --logdir=runs --bind_all
# # nohup tensorboard --logdir=runs --bind_all > /dev/null 2>&1
# nohup tensorboard --logdir=runs --host 0.0.0.0 > /dev/null 2>&1 &

def set_tensorboard_writer(name):
    writer = SummaryWriter(name) # 'runs/fashion_mnist_experiment_1'
    return writer

def inspect_model(writer, model, data):
    writer.add_graph(model, data)

def close_tensorboard_writer(writer):
    writer.close()

def add_dataset_figure(writer, dataloader, desc="Train", epoch=0):
    dataiter = iter(dataloader)
    waveform01, waveform02, _, _ = dataiter.next()
    fig = plt.figure()
    plt.plot(waveform01[0].t().numpy(), alpha=0.5)
    plt.plot(waveform02[0].t().numpy(), alpha=0.5)
    plt.title("sample waveform visualization")
    writer.add_figure('Visualize{}'.format(desc), fig, epoch)
    plt.close()

def visualization_dataset_by_byol(writer, dataloader, desc="Train", epoch=0):
    dataiter = iter(dataloader)
    waveform01, waveform02 = dataiter.next()
    fig = plt.figure()
    plt.plot(waveform01[0].t().numpy(), alpha=0.5)
    plt.plot(waveform02[0].t().numpy(), alpha=0.5)
    plt.title("sample waveform visualization")
    writer.add_figure('Visualize{}'.format(desc), fig, epoch)
    plt.close()
    # writer.add_audio('Visualize{}'.format(desc), waveform01[0], epoch, sample_rate=16000)

def add_dataset_figure_by_byol(writer, dataloader, desc="Train", epoch=0):
    dataiter = iter(dataloader)
    waveform01, waveform02 = dataiter.next()
    fig = plt.figure()
    plt.plot(waveform01[0].t().numpy(), alpha=0.5)
    plt.plot(waveform02[0].t().numpy(), alpha=0.5)
    plt.title("sample waveform visualization")
    writer.add_figure('Visualize{}'.format(desc), fig, epoch)
    plt.close()

def add_latent_heatmap(writer, data, title, desc, epoch):
    fig = plt.figure()
    ax1 = fig.add_subplot(1, 1, 1)
    ax1.matshow(data)
    writer.add_figure('{}/{}'.format(title, desc), fig, epoch)
    plt.close()

def add_latents_heatmap(writer, output, title, desc, step):
    fig, axes = plt.subplots(1, 4)
    for i in range(4):
        axes[i].matshow(output[i], aspect='equal')  # , aspect='auto')
    writer.add_figure('{}/{}'.format(title, desc), fig, step)
    plt.close()

def add_confusion_matrix(writer, title, desc, step, label_num, targets, predicts):
    labels = np.arange(label_num)
    output = confusion_matrix(targets, predicts, labels=labels)
    norm_output = output / output.astype(np.float).sum(axis=1)

    fig = plt.figure()
    ax1 = fig.add_subplot(1, 1, 1)
    ax1.matshow(norm_output)
    xaxis = np.arange(len(labels))
    ax1.set_xticks(xaxis)
    ax1.set_yticks(xaxis)
    ax1.set_xticklabels(labels)
    ax1.set_yticklabels(labels)

    writer.add_figure('{}/{}'.format(title, desc), fig, step)
    plt.close()

def add_classification_matrix(config, epoch, writer, title, desc,step, label_num, targets, predicts):
    labels = list(map(str, list(np.arange(label_num))))
    out = classification_report(targets, predicts, target_names=labels, output_dict=True)
    base_directory = os.path.join(config['checkpoint_save_directory_path'], config['checkpoint_file_name'])
    file_path = os.path.join(base_directory,
                             config['checkpoint_file_name'] + "-model-best-epoch-{}-clf.json".format(epoch))
    if not os.path.exists(os.path.join(config['checkpoint_save_directory_path'], config['checkpoint_file_name'])):
        file_io.make_directory(os.path.join(config['checkpoint_save_directory_path'], config['checkpoint_file_name']))
    with open(file_path, 'w') as f:
        json.dump(out, f)

    x1 = np.arange(1, label_num * 2, 2)
    x2 = x1 + 0.5
    x3 = x1 + 1

    precision_value = []
    recall_value = []
    f1_value = []

    for data in labels:
        precision_value.append(out[data]['precision'])
        recall_value.append(out[data]['recall'])
        f1_value.append(out[data]['f1-score'])

    fig = plt.figure()
    plt.bar(x1, precision_value, color='r', width=0.5, label='precision')
    plt.bar(x2, recall_value, color='g', width=0.5, label='recall')
    plt.bar(x3, f1_value, color='b', width=0.5, label='fl-score')
    plt.xticks(x2, labels)
    plt.legend()

    writer.add_figure('{}/{}'.format(title, desc), fig, step)
    plt.close()

def add_classification_avg_matrix(writer, title, desc, step, label_num, targets, predicts):
    labels = np.arange(label_num)
    term_labels = ['macro avg', 'weighted avg']
    out = classification_report(targets, predicts, target_names=labels, output_dict=True)

    x1 = [1, 3]
    x2 = [1.5, 3.5]
    x3 = [2, 4]

    precision_value = []
    recall_value = []
    f1_value = []
    for data in term_labels:
        precision_value.append(out[data]['precision'])
        recall_value.append(out[data]['recall'])
        f1_value.append(out[data]['f1-score'])

    fig = plt.figure()
    plt.bar(x1, precision_value, color='r', width=0.5, label='precision')
    plt.bar(x2, recall_value, color='g', width=0.5, label='recall')
    plt.bar(x3, f1_value, color='b', width=0.5, label='fl-score')
    plt.xticks(x2, term_labels)
    plt.legend()

    writer.add_figure('{}/{}'.format(title, desc), fig, step)
    plt.close()


# Importing OpenCV library

import cv2 as cv
#Reading images

# img1 = cv.imread('pics/cat1.jpeg')
# img2 = cv.imread('pics/dog2.jpeg')

# cv.imshow('dog', img2)

#Reading videos

capture_vid = cv.VideoCapture('videos/Cute Cat - 3092.mp4')

while True:
    isTrue, frame = capture_vid.read()
    cv.imshow('video', frame)

    if cv.waitKey(20) & 0xFF==ord('d'):
        break

capture.release()
cv.destroyAllWindows()


from django.urls import path
from django.urls.resolvers import URLPattern
from . import views

app_name = "tasks"

urlpatterns = [
    path("taskssearch/", views.TaskFilterView.as_view(), name="task_search"),
    path("projects/", views.ProjectList.as_view(), name="list_projects"),
    path("projects/create", views.ProjectCreateView.as_view(), name="create_project"),
    path("projects/<str:pk>", views.TaskListView.as_view(), name="list_task"),
    path(
        "projects/<str:pk>/update",
        views.ProjectUpdateView.as_view(),
        name="update_project",
    ),
    path(
        "projects/<str:pk>/delete",
        views.ProjectDeleteView.as_view(),
        name="delete_project",
    ),
    path(
        "projects/<str:pk>/tasks/create",
        views.TaskCreateView.as_view(),
        name="create_task",
    ),
    path(
        "projects/<str:pk>/tasks/<str:task_pk>/update",
        views.TaskUpdateView.as_view(),
        name="update_task",
    ),
    path(
        "projects/<str:pk>/tasks/<str:task_pk>/markascomplete",
        views.task_completed_view,
        name="mark_as_complete",
    ),
    path(
        "projects/<str:pk>/tasks/<str:task_pk>/delete",
        views.TaskDeleteView.as_view(),
        name="delete_task",
    ),
    path(
        "overduetasks/",
        views.TaskOverdueListView.as_view(),
        name="list_all_overdue_tasks",
    ),
    path(
        "overduetasks/<str:pk>",
        views.TaskOverdueListView.as_view(),
        name="list_overdue_tasks",
    ),
]


from typing import Dict, List

from django.db.models import Prefetch

from products.models import ProductSale
from products.services import change_product_count, get_total_price_product

def change_total_cost(order) -> None:
    """Calculate and update the total cost of an order."""
    product_sale_prefetch = Prefetch("product__sale", queryset=ProductSale.objects.all())
    order_products = (
        order.products.select_related("product")
        .prefetch_related(product_sale_prefetch)
        .all()
    )
    total_cost = 0

    for order_product in order_products:
        product = order_product.product
        change_product_count(product, order_product.count)
        total_cost += get_total_price_product(product, order_product.count)

    order.totalCost = total_cost
    order.save()

def check_and_update_express_delivery(order_data) -> None:
    """Check and update your order total if express shipping is selected."""
    express_delivery_price = 5
    if order_data.get("deliveryType") == "express":
        order_data["totalCost"] = express_delivery_price + float(order_data["totalCost"])

def get_order_products_data(order_products_data) -> List[Dict]:
    """Converts the list of order products and their quantity"""
    return [
        {
            "product": order_product_data.get("id"),
            "count": order_product_data.get("count"),
        }
        for order_product_data in order_products_data
    ]


from django.db import models
from Register.models import User
from Product.models import Product
# Create your models here.
class Basket(models.Model):
    user_id = models.ForeignKey(User, on_delete=models.CASCADE)
    completed = models.BooleanField(default=False)

    @property
    def getBasketTotal(self):
        itembaskets = self.itembasket_set.all()
        total = sum([item.getTotal for item in itembaskets])
        return total

    @property
    def getBasketCount(self):
        count = self.itembasket_set.all().count()
        return count

    def __str__(self):
        return str(self.id)

class ItemBasket(models.Model):
    basket_id = models.ForeignKey(Basket, on_delete=models.CASCADE, null = True)
    product_id = models.ForeignKey(Product, on_delete=models.CASCADE)
    quantity = models.IntegerField(default=0)

    @property
    def getTotal(self):
        total =self.product_id.price*self.quantity
        return total

    def __str__(self):
        return str(self.product_id.id)


import datetime

from flask import url_for
from mongoengine import StringField, DictField
from mongoengine.document import Document

class Notification(Document):
    icum_face_id = StringField(requred=False)
    msg = StringField(max_length=500, required=True)
    msg_type = StringField(max_length=15, required=False)
    attributes = DictField()

    meta = {
        'collection': 'notifications',
        'ordering': ['-_id'],
        'auto_create_index': True,
        'strict': False
    }

    def to_eventhub_json(self):
        res = {
            'url': 'http://icumistercontrolpanel.azurewebsites.net' + url_for('show_notification', notification_id=str(self.id)),
            'msg': self.msg,
            'type': self.msg_type if self.msg_type else 'unknown',
            'timestamp': str(datetime.datetime.now()),
            'time_elapsed': 5
        }
        return res


import numpy as np
import cv2 as cv
haar_cascade_face = cv.CascadeClassifier('haar_face.xml')
people = ['Ben Afflek','Elton John', 'Jerry Seinfield', 'Madonna', 'Mindy Kaling']
# features = np.load('features.npy',allow_pickle=True)
# labels = np.load('labels.npy')

face_recognizer = cv.face.LBPHFaceRecognizer_create()
face_recognizer.read('face_trained.yml')
img = cv.imread(r'C:\Users\Jack Chau\PycharmProjects\python\python_01\OpenCV\face_Recognition\val\ben_afflek\1.jpg')
gray_img = cv.cvtColor(img,cv.COLOR_BGR2GRAY)
cv.imshow('person',gray_img)

# Detect the face in image
face_rect = haar_cascade_face.detectMultiScale(gray_img,1.1,4)
for (x,y,w,h) in face_rect:
    faces_roi = gray_img[y:y+h,x:x+w]
    label,confidence = face_recognizer.predict(faces_roi)
    print(f'Label = {people[label]}, with a confidence of {confidence}')

    cv.putText(img,str(people[label]),(20,20),cv.FONT_HERSHEY_COMPLEX,1.0,(0,255,0),thickness=2)
    cv.rectangle(img,(x,y),(x+w,y+h),(0,255,0),thickness=2)
cv.imshow('Detected Face',img)
cv.waitKey(0)


from click.testing import CliRunner
import click
import jtlib
import pytest

def test_projects_valid_url_argument(runner, context):
    """Check project group command result for a JIRA server."""
    result = runner.invoke(jtlib.scripts.jt, [ 'https://jira.atlassian.com', 'projects', ], obj = context)
    assert 0 == result.exit_code
    assert 'CLOUD' in result.output

command_list = [
    'project',
]

@pytest.fixture(scope = 'module', params = command_list)
def command(request):
    return request.param

def test_command_with_missing_project(runner, context, server_url, command):
    """Check command without specifying a project."""
    result = runner.invoke(jtlib.scripts.jt, [ server_url, command ], obj = context)
    assert 2 == result.exit_code
    assert 'Usage:' in result.output

bad_project_list = [
    'CLOUDY', # Nonexistant project, match project key regular expression.
    '123', # Malformed project name.
]

@pytest.fixture(scope = 'module', params = bad_project_list)
def bad_project(request):
    return request.param

def test_command_with_nonexistent_project(runner, context, server_url, command, bad_project):
    """Check command specifying a nonexistant project."""
    result = runner.invoke(jtlib.scripts.jt, [ server_url, command, bad_project ], obj = context)
    assert 2 == result.exit_code
    assert 'Usage:' in result.output

def test_command_invalid_url_argument(runner, context, command):
    """Check project group command result for a server."""
    result = runner.invoke(jtlib.scripts.jt, [ 'https://www.example.com', command ], obj = context)
    assert 2 == result.exit_code
    assert 'Usage:' in result.output


# ---API list---
# top_n
# search
# recommend_item_cf
# recommend_user_cf

import flask
from flask import render_template, jsonify, request, make_response
from flask_cors import CORS
import pandas as pd
import numpy as np
import re
import jieba as jb
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.neighbors import NearestNeighbors
from scipy.sparse import csr_matrix
import json
import time

app = flask.Flask(__name__)
app.config["DEBUG"] = True
CORS(app, supports_credentials=True)

def readCSV():
    products = pd.read_csv('./products_join_categories.csv')
    products = products.loc[:, ~products.columns.str.contains('^Unnamed')]
    products.set_index( ['productId'] , inplace=True, drop=True )
    products = products.rename({'name': 'product_name'}, axis='columns')
    categories = pd.read_csv( './categories.csv')
    rating = pd.read_csv('./ratings_drop.csv' , usecols=[ 'userId' , 'productId' , 'rating'])
    rating.dropna(subset=["userId"], inplace = True)
    rating.dropna(subset=["rating"], inplace = True)
    products.dropna(subset=["product_name"], inplace = True)
    return products, categories , rating

def readCSVWord():
    product_lists = pd.read_csv( './product_lists_with_cut.csv').fillna('')
    top_20_words1 = pd.read_csv( './top_20_word1.csv', names = ['cut_name' , 'count'])
    top_20_words2 = pd.read_csv( './top_20_word2.csv', names = ['cut_name' , 'count'])
    top_20_words3 = pd.read_csv( './top_20_word3.csv', names = ['cut_name' , 'count'])
    product_lists.dropna(subset=["name"], inplace = True)
    return product_lists, top_20_words1, top_20_words2, top_20_words3

def processText(text, join_str = " ", array = False):
    text = str(text)
    if text.strip()=='':
        return ''
    r = '[’!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~]+'
    text = re.sub(r, '', text)
    # cut word, GitBub: https://github.com/fxsjy/jieba
    text = join_str.join([w for w in list(jb.cut(text)) if w !=' '])
    return text

def recommendations(text,num = 20):
    res = []
    ss = [processText(text, ",")]
    ss_vec = tfidf.transform(ss)
    cos_sim = cosine_similarity(ss_vec, tfidf_vec)
    arr =cos_sim[0]
    idxs=list(np.argsort(-arr)[:num])
    for idx in idxs:
        row = product_lists[product_lists.index==idx]
        productId = row.productId.values[0]
        name = row.cut_name.values[0]

        res.append({
            'productId' : str(productId),
            'product_name' : name.replace(" ", ""),
            'cat1' : str(products.loc[idx].cat1),
            'cat2' : str(products.loc[idx].cat2),
            'cat3' : str(products.loc[idx].cat3),
            'cat1_name' : products.loc[idx].cat1_name,
            'cat2_name' : products.loc[idx].cat2_name,
            'cat3_name' : products.loc[idx].cat2_name,
            'cos' : str(arr[idx])
        })
    return res

def filterRatingArray():
    global rating
    useRatingCount = rating['userId'].value_counts()
    ratingFiltered = rating[rating['userId'].isin(useRatingCount[useRatingCount >= 50 ].index)]
    productRatingCount = rating['productId'].value_counts()
    ratingFiltered = ratingFiltered[ratingFiltered['userId'].isin(productRatingCount[productRatingCount >= 100 ].index)]
    del useRatingCount
    del productRatingCount
    return ratingFiltered , rating

def findByKItemByTopKUsers(ratingFilteredPivotByUser,userId,distances,indices,k):
    global products
    prediction=0
    similarities = 1-distances.flatten()
    sum_wt = np.sum(similarities)
    wtd_sum = 0

    # all - user rated!
    notRatedItems = np.setdiff1d(ratingFilteredPivotByUser.columns.to_numpy() , ratingFilteredPivotByUser[ (ratingFilteredPivotByUser.index == userId) ].iloc[0].nonzero()[0] )
    simArray = np.array([similarities]).T * ratingFilteredPivotByUser.ix[indices.flatten()].filter(notRatedItems.tolist()).to_numpy()
    sortedIndex = (simArray.sum(axis=0)/sum_wt).argsort()[::-1][:10]
    result = []
    for index , productId in enumerate(sortedIndex.tolist()):
        result.append({
            'productId' : str(notRatedItems[productId]),
            'product_name' : products.loc[notRatedItems[productId]].product_name,
            'cat1' : str(products.loc[notRatedItems[productId]].cat1),
            'cat2' : str(products.loc[notRatedItems[productId]].cat2),
            'cat3' : str(products.loc[notRatedItems[productId]].cat3),
            'cat1_name' : products.loc[notRatedItems[productId]].cat1_name,
            'cat2_name' : products.loc[notRatedItems[productId]].cat2_name,
            'cat3_name' : products.loc[notRatedItems[productId]].cat2_name,
        })
    return result

def findTopKByItemCf(ratingFilteredPivot,itemId, k):
    #user based:
    ratingFilteredMatrix = csr_matrix(ratingFilteredPivot.values)
    model_knn = NearestNeighbors(metric = 'cosine', algorithm = 'brute')
    model_knn.fit(ratingFilteredMatrix)
    NearestNeighbors(algorithm='brute', leaf_size=30, metric='cosine', metric_params=None, n_jobs=1, n_neighbors=k, p=2, radius=1.0)

    #489000->productId
    queryIndex = ratingFilteredPivot.index.get_loc(itemId)
    # queryIndex = np.random.choice(ratingFilteredPivot.shape[0])
    distances, indices = model_knn.kneighbors(ratingFilteredPivot.iloc[queryIndex, :].values.reshape(1, -1), n_neighbors=k+1)
    return distances, indices

def findTopKByUserCf(ratingFilteredPivotByUser,userId):
    raitingFilteredMatrix = csr_matrix(ratingFilteredPivotByUser.values)
    model_knn = NearestNeighbors(metric = 'cosine', algorithm = 'brute')
    model_knn.fit(raitingFilteredMatrix)
    NearestNeighbors(algorithm='brute', leaf_size=30, metric='cosine', metric_params=None, n_jobs=1, n_neighbors=5, p=2, radius=1.0)
    #119->userId
    #114416 , 3400 , 17750 ,83601 => 0.7
    # 127020.0 => 0.3
    # 127579.0 => 0.3
    queryIndex = ratingFilteredPivotByUser.index.get_loc(userId)
    distances, indices = model_knn.kneighbors(ratingFilteredPivotByUser.iloc[queryIndex, :].values.reshape(1, -1), n_neighbors=6)
    return distances, indices

start_time = time.time()
print("Starting....")
products, categories, rating = readCSV()
product_lists, top_20_words1, top_20_words2, top_20_words3 = readCSVWord()

tfidf  = TfidfVectorizer(analyzer='word', ngram_range=(1, 1), min_df=0).fit(product_lists['cut_name'])
tfidf_vec = tfidf.transform(product_lists['cut_name'])

rating , ratingOrigin = filterRatingArray()
rating.drop_duplicates(subset=['userId', 'productId'], keep='first' , inplace =True )
ratingFilteredPivot = rating.pivot( index='productId' , columns='userId' , values="rating" ).fillna(0)
ratingFilteredPivotByUser = rating.pivot( index='userId' , columns='productId' , values="rating" ).fillna(0)
print("--- %s seconds ---" % (time.time() - start_time))

def makeResponse(result_text):
    response = make_response(jsonify(data=result_text))
    response.headers['Access-Control-Allow-Origin'] = '*'
    response.headers['Access-Control-Allow-Methods'] = 'OPTIONS,HEAD,GET,POST'
    response.headers['Access-Control-Allow-Headers'] = 'x-requested-with'
    return response

# tf-idf
@app.route('/top_n', methods=['GET'])
def top_n_word():
    n = request.args.get('n') or "1"
    if int(n) > 3:
        n = "1"
    str_n = "top_20_words" + n
    exec("global %s" % str_n)
    words = globals()[str_n].to_dict('index')
    res = [(v) for k, v in words.items()]
    return(makeResponse(res))
    #return render_template("aaa.html")

@app.route('/search', methods=['GET'])
def search():
    query_str = request.args.get('query_str')
    res = recommendations(query_str)
    return(makeResponse(res))

# knn
@app.route('/recommend_item_cf', methods=['GET'])
def recommendByItemCf():
    global ratingFilteredPivot
    global ratingOrigin
    global products
    itemId = int(request.args.get('itemid'))
    k = int(request.args.get('k') or '10')
    #489000

    topK = []
    if itemId in ratingFilteredPivot.index:
        distances, indices = findTopKByItemCf(ratingFilteredPivot,itemId,k)
        for i in range(0, len(distances.flatten())):
            topK.append({
                'product_name' : products.loc[ratingFilteredPivot.index[indices.flatten()[i]]].product_name,
                'cat1_name' : products.loc[ratingFilteredPivot.index[indices.flatten()[i]]].cat1_name
            })
    else:
        if products.loc[itemId].cat3 is None:
            tempDataSet = (products.cat2 == products.loc[itemId].cat2)
        else:
            tempDataSet = (products.cat3 == products.loc[itemId].cat3)
        temp = ratingOrigin[ratingOrigin.productId.isin(products.loc[tempDataSet].index) ].sort_values(by='rating', ascending=False)[:10].productId.tolist()
        for i in temp:
          topK.append({
              'product_name' : products.loc[i].product_name,
              'cat1_name' : products.loc[i].cat1_name
          })
    return makeResponse(topK)

@app.route('/recommend_user_cf', methods=['GET'])
def recommendByUserCf():
    global ratingFilteredPivotByUser
    global products
    userId = int(request.args.get('userid'))
    k = int(request.args.get('k') or '10')
    #7488
    distances, indices = findTopKByUserCf(ratingFilteredPivotByUser,userId)
    topK = findByKItemByTopKUsers(ratingFilteredPivotByUser,userId,distances,indices,k)
    return makeResponse(topK)

@app.route('/find_product', methods=['GET'])
def findProductById():
    global products
    global ratingOrigin
    productId = int(request.args.get('productId'))
    product = products.loc[productId]
    avgRating = ratingOrigin.loc

    return makeResponse(
    {
        'productId' : str(productId),
        'product_name' : products.loc[productId].product_name,
        'cat1' : str(products.loc[productId].cat1),
        'cat2' : str(products.loc[productId].cat2),
        'cat3' : str(products.loc[productId].cat3),
        'cat2_name' : products.loc[productId].cat2_name,
        'cat3_name' : products.loc[productId].cat2_name,
        'cat1_name' : products.loc[productId].cat1_name,
        'rating' : ratingOrigin[ratingOrigin.productId == productId].mean().rating
    })

app.run(host='0.0.0.0', port=5000)


from datetime import datetime, timedelta
from airflow.decorators import dag
from tasks.ktc import (
    ktc_web_scraper,
    data_validation,
    ktc_player_load,
    surrogate_key_formatting,
)

dag_owner = "dynasty_superflex_db"

@dag(
    default_args={
        "owner": dag_owner,
        "depends_on_past": False,
        "email": ["grayson.stream@gmail.com"],
        "email_on_failure": False,
        "email_on_retry": False,
        "retries": 1,
        "retry_delay": timedelta(minutes=5),
    },
    description="Web Scaper pulling in KTC Player data to build Superflex Power Rankings",
    schedule_interval="30 * * * *",
    start_date=datetime(2022, 6, 8),
    catchup=False,
    tags=["scraper", "database"],
)
def ktc_player_task_api():

    player_data = ktc_web_scraper()
    player_validation = data_validation(player_data)
    player_load = ktc_player_load(player_validation)
    surrogate_key_formatting(player_load)

ktc_player_dag = ktc_player_task_api()


import json

#data_pizza = {
#        'margarita' : 400,
#        'carbonara' : 500,
#        }
#with open('pizza.json', 'w') as f:
#    json.dump(data_pizza, f)
'''
Инициация файла. Код выше - запускается 1 раз
'''

def add_pizza(name, price):
    with open('pizza.json', 'r') as f:
        data_pizza = json.load(f)
    if name not in data_pizza.keys():
        data_pizza[name] = price
        with open('pizza.json', 'w') as f:
            json.dump(data_pizza, f)
    else:
        print('Pizza already exist')

def del_pizza(name):
    with open('pizza.json', 'r') as f:
        data_pizza = json.load(f)
    if name in data_pizza.keys():
        del data_pizza[name]
        with open('pizza.json', 'w') as f:
            json.dump(data_pizza, f)
    else:
        print('Pizza not exist')

def order_pizza():
    with open('pizza.json', 'r') as f:
        data_pizza = json.load(f)
    order = []
    cost = 0
    while True:
        q1 = input('Continue? (yes | no)')
        if q1.lower() == 'no':
            break
        pizza_name = input('Pizza name?').lower()
        if pizza_name in data_pizza.keys():
            order.append(pizza_name)
            cost += data_pizza[pizza_name]
            print('Pizza added')
    return {'order' : order, 'cost': cost}

while True:
    break_point = input('Stop?').lower()
    if break_point == 'yes':
        break
    chose_role = input('Chose role:')
    if chose_role.lower() == 'user':
        print(order_pizza())
    else:
        q2 = input('Add or delete?').lower()
        if q2 == 'delete':
            del_pizza(input('Pizza name?'))
        elif q2 == 'add':
            add_pizza(input('Pizza name?').lower(), int(input('Price')))


import cv2
import numpy as np
import streamlit as st
from PIL import Image
from streamlit_toggle import st_toggle_switch

def update_slider_vals(kl, kh, new_l, new_h):
    st.session_state[kl] = new_l
    st.session_state[kh] = new_h

fnames = st.file_uploader("Upload an RGB image", type=["png", "jpg"],
                        accept_multiple_files=True)
file_num = st.number_input("Image to display", min_value=1, max_value=len(fnames))
grid = st_toggle_switch(
    label="Grid layout?",
    default_value=False,
)
tabnames = ("R", "G", "B", "H", "S", "V")

if len(fnames) != 0:
    # load channels
    fname = fnames[file_num - 1]
    img = Image.open(fname).convert("RGB") # BGR format
    w_, h_ = img.size

    cola, colb, colc = st.columns(3)
    resize = cola.checkbox('Resize?')
    width = colb.number_input('New width in px', min_value=250, max_value=w_)
    height = colc.number_input('New height in px', min_value=250, max_value=h_)

    if resize:
        img = img.resize((width, height))

    img = np.array(img).astype(np.uint8)

    r = img[:, :, 0]
    g = img[:, :, 1]
    b = img[:, :, 2]
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    h = hsv[:, :, 0]
    s = hsv[:, :, 1]
    v = hsv[:, :, 2]

    # iterate channels and show with masks
    if grid:
        with st.container():
            c1, c2, c3 = st.columns(3)
            for i,im in enumerate((r, g, b, h, s, v)):
                index = i - 3 if i >= 3 else i
                col = (c1, c2, c3)[index]
                col.header(tabnames[i])
                k_low = "low_" + tabnames[i]
                k_high = "high_" + tabnames[i]
                if k_low in st.session_state:
                    low = st.session_state[k_low]
                    high = st.session_state[k_high]
                else:
                    low, high = 0, 255

                k = f"{tabnames[i]}_{i}"
                l, h = col.slider("Select a values for thresholding",
                                0, 255, (low, high), step=5, key=k,
                                on_change=update_slider_vals, kwargs={"kl" : k_low,
                                "kh" : k_high, "new_l" : low, "new_h" : high}
                                )
                col.image(cv2.inRange(im, l, h))

    else:
        # create and display tabs
        with st.container():
            tabs = st.tabs(tabnames)
            for i,im in enumerate((r, g, b, h, s, v)):
                with tabs[i]:
                    st.header(tabnames[i])
                    k_low = "low_" + tabnames[i]
                    k_high = "high_" + tabnames[i]
                    if k_low in st.session_state:
                        low = st.session_state[k_low]
                        high = st.session_state[k_high]
                    else:
                        low, high = 0, 255

                    k = f"{tabnames[i]}_{i}"
                    l, h = st.slider("Select a values for thresholding",
                                    0, 255, (low, high), step=5, key=k,
                                    on_change=update_slider_vals, kwargs={"kl" : k_low,
                                    "kh" : k_high, "new_l" : low, "new_h" : high}
                                    )
                    c1, c2 = st.columns(2)
                    c1.image(im)
                    c2.image(cv2.inRange(im, l, h))


from itertools import combinations

T = int(input())
tmp = -1
result = 0
for tc in range(1,T+1):
    arr = list(map(int,input().split()))
    t = -1
    for a in list(combinations(arr,3)):
        if sum(a)%10>t:
            t = sum(a)%10
    if tmp<=t:
        tmp = t
        result = tc
print(result)


# -*- coding: utf-8 -*-
"""
Created on Wed Nov 28 21:22:47 2018

@author: Inna
"""

import numpy as np
import torch
from tqdm import tqdm
from itertools import zip_longest
from operator import itemgetter
import math

from Record import Mode, RecordType, Record, save_to_file
import Config as cfg
import maskfactory as mf

CQ_DEBUG = False

class ChannelQuantizier():
    def __init__(self, rec, init_acc, max_acc_loss, patch_size, out_rec=None, default_in_pattern=None):
        self.patch_size = patch_size
        self.input_patterns = rec.all_patterns
        self.input = rec.gen_pattern_lists(init_acc - max_acc_loss)

        input_new = []
        self.no_of_patterns = 0
        for l in range(len(self.input)):
            input_new.append([self.input[l][c][0] for c in range(rec.layers_layout[l][0])])
            self.no_of_patterns += max([len(self.input[l][c][0]) for c in range(rec.layers_layout[l][0])])
        self.input = input_new

        if default_in_pattern is not None:
            self.default_in_pattern = default_in_pattern
        elif rec.mode == Mode.MAX_GRANULARITY:
            self.default_in_pattern = np.ones((self.patch_size, self.patch_size), dtype=self.input_patterns[0][0][0].dtype)
        else:
            self.default_in_pattern = np.ones((self.input_patterns.shape[0], self.input_patterns.shape[0]),
                                              dtype=self.input_patterns.dtype)

        if out_rec is None:
            self._generate_patterns(rec.mode, rec.layers_layout, rec.gran_thresh, rec.filename, max_acc_loss, init_acc)
        else:
            self.output_rec = out_rec

    def number_of_iters(self):
        return self.no_of_patterns

    def simulate(self, nn, test_gen):
        st_point = self.output_rec.find_resume_point()
        print('==> starting ChannelQuantizier simulation.')

        save_counter = 0
        for layer in tqdm(range(st_point[0], len(self.input))):
            for p_idx in range(st_point[3], len(self.output_rec.all_patterns[layer])):
                st_point[3] = 0
                nn.net.strict_mask_update(update_ids=[layer],
                                              masks=[torch.from_numpy(self.output_rec.all_patterns[layer][p_idx])])
                if CQ_DEBUG:
                    test_acc = 100
                    ops_saved = 100
                    ops_total = 100
                else:
                    _, test_acc, _ = nn.test(test_gen)
                    ops_saved, ops_total = nn.net.num_ops()
                    nn.net.reset_ops()
                self.output_rec.addRecord(ops_saved, ops_total, test_acc, layer, 0, 0, p_idx)

                save_counter += 1
                if save_counter > cfg.SAVE_INTERVAL:
                    self.save_state()
                    save_counter = 0

        self.save_state()
        print('==> finised ChannelQuantizier simulation.')

    def is_finised(self):
        return self.output_rec.find_resume_point() is None

    def save_state(self):
        save_to_file(self.output_rec, True, cfg.RESULTS_DIR)

    def _generate_patterns(self, mode, layers_layout, gran_thresh, rec_in_filename, max_acc_loss, init_acc):
        self.output_rec = Record(layers_layout, gran_thresh, False, mode, init_acc)
        self.output_rec.set_results_dimensions(no_of_layers=len(self.input),
                                               no_of_channels=[1] * len(self.input),
                                               no_of_patches=[1] * len(self.input))
        if cfg.CHANNELQ_UPDATE_RATIO == 1:
            no_of_patterns_gen, all_patterns = self._gen_patterns_zip_longest(mode, layers_layout)
        else:
            no_of_patterns_gen, all_patterns = self._gen_patterns_zip_ratio(mode, layers_layout)

        self.output_rec.set_all_patterns(all_patterns, RecordType.cQ_REC)
        self.output_rec.set_results_dimensions(no_of_patterns=no_of_patterns_gen)

        fn = f'ChannelQ{cfg.CQ_OPTION.value}r{cfg.CHANNELQ_UPDATE_RATIO}_ma{max_acc_loss}_' + rec_in_filename
        if CQ_DEBUG:
            fn = 'DEBUG_' + fn
        self.output_rec.set_filename(fn)

    def _build_layer(self, layer_dims, layer_opt, l, mode):
        layer = np.ones(mf.expend_dims(layer_dims, self.patch_size), dtype=self.default_in_pattern.dtype)
        for idx, opt in enumerate(layer_opt):
            if type(opt) is int:
                opt = self.input[l][idx][opt]
            if opt[0] == -1:
                tmp = mf.tile_opt((layer_dims[1], layer_dims[2]),
                                               self.default_in_pattern, False)
            elif mode == Mode.MAX_GRANULARITY:
                tmp = mf.tile_opt((layer_dims[1], layer_dims[2]),
                                               self.input_patterns[l][idx][opt[0]], False)
            else:
                tmp = mf.tile_opt((layer_dims[1], layer_dims[2]),
                                               self.input_patterns[:, :, opt[0]], False)
            layer[idx, :, :] = mf.crop(layer_dims,tmp, self.patch_size)
        return layer

    def _gen_patterns_zip_ratio(self, mode, layers_layout):
        all_patterns = []
        no_of_patterns = [None] * len(self.input)
        for l in range(len(self.input)):
            channels_to_update = math.ceil(cfg.CHANNELQ_UPDATE_RATIO*layers_layout[l][0])
            layers = []
            curr_layer_opt = [0]*layers_layout[l][0]
            while curr_layer_opt is not None:
                layers.append(self._build_layer(layers_layout[l], curr_layer_opt, l, mode))
                curr_layer_opt = self._zip_ratio(curr_layer_opt, channels_to_update, self.input[l])
            no_of_patterns[l] = len(layers)
            all_patterns.append(layers)
        return no_of_patterns, all_patterns

    def _zip_ratio(self, curr_layer_opt, channels_to_update, layer_input):
        possible_channels = []
        last_stage_channels = []
        for channel, opt_idx in enumerate(curr_layer_opt):
            if cfg.TWO_STAGE and opt_idx+1 < (len(layer_input[channel])-1):
                possible_channels.append(channel)
            elif cfg.TWO_STAGE and opt_idx+1 == (len(layer_input[channel])-1):
                last_stage_channels.append(channel)
            elif not cfg.TWO_STAGE and opt_idx+1 < (len(layer_input[channel])):
                possible_channels.append(channel)

        if channels_to_update < len(possible_channels):
            if cfg.CQ_OPTION == cfg.CQ_modes.DEFAULT:
                sorted_channels = [x for x,_ in sorted(zip(possible_channels, itemgetter(*possible_channels)(curr_layer_opt)), key=lambda pair: pair[1])]
                if channels_to_update == 1:
                    channels_to_inc = [sorted_channels[0]]
                else:
                    channels_to_inc = sorted_channels[:channels_to_update]
        elif len(possible_channels) > 0:
            channels_to_inc = possible_channels
        else:
            channels_to_inc = []

        additional_channels = channels_to_update - len(channels_to_inc)
        if additional_channels > 0:
            if additional_channels < len(last_stage_channels):
                if cfg.CQ_OPTION == cfg.CQ_modes.DEFAULT:
                    if additional_channels == 1:
                        channels_to_inc.append(last_stage_channels[0])
                    else:
                        channels_to_inc += last_stage_channels[:additional_channels]
            elif len(last_stage_channels) > 0:
                channels_to_inc += last_stage_channels

        if len(channels_to_inc) == 0:
            return None
        else:
            for channel in channels_to_inc:
                curr_layer_opt[channel] += 1
            return curr_layer_opt

    def _gen_patterns_zip_longest(self, mode, layers_layout):
        all_patterns = []
        no_of_patterns = [None] * len(self.input)
        for l in range(len(self.input)):
            layers = []
            for layer_opt in zip_longest(*self.input[l], fillvalue=(-1, -1, -1)):
                layers.append(self._build_layer(layers_layout[l], layer_opt, l, mode))
            no_of_patterns[l] = len(layers)
            all_patterns.append(layers)
        return no_of_patterns, all_patterns

    def _clean_input(self):
        for l in range(len(self.input)):
            for c in range(len(self.input[l])):
                self.input[l][c][:] = [tup for tup in self.input[l][c] if self._determine(tup)]

    def _determine(self, tup):
        p_idx, ops_saved, acc, tot_ops = tup
        if ops_saved == 0 and p_idx != -1:
            return False
        return True


from tinydb import TinyDB

player_database = TinyDB('models_players.json')

class Player:
    def __init__(self,first_name = None,
                       last_name = None,
                       birthdate = None,
                          gender = None,
                         ranking = None,
                           score = 0,
                       player_id = 0
                 ):
        self.first_name = first_name
        self.last_name = last_name
        self.birthdate = birthdate
        self.gender = gender
        self.ranking = ranking
        self.score = score
        self.player_id = player_id

class Game: #Non ici il faut une méthode game
    def __init__(self, game_wins=1,game_loses=0, game_draw=0.5):
        self.game_wins = game_wins
        self.game_loses = game_loses
        self.game_draw = game_draw

#Voici des instances de la classe Player
Jona = Player('Jonathan', 'BUISSON', '09.11.1985', 'Man' ,'700','90','1')
print(Jona.last_name)
Micha = Player('Michael','DWIGHT','01.01.1980','Man','700', '50','2')
print(Micha.score)
Maria = Player('Maria','PEREZ','12.08.1982','Woman','700','20','3')
print(Maria.gender)
Ivana = Player('Ivana','SIDOROV','15.12.1995','Woman','800','40','4')
print(Ivana.ranking)
Clark = Player('Clark','KENT','26.04.1977','Man','1000','100','5')
print(Clark.birthdate)
Zata = Player('Zatana','COOPER','20.03.1990','Woman','900','80','6')
print(Zata.first_name)
Martin = Player('Martin','DUPOND','22.09.1975','Man','990','90','7')
print(Martin.ranking)
Xena = Player('Xena','GABRIEL','29.03.1970','Woman','985','70','8')
print(Xena.player_id)

"""Lorsqu'on remplit un formulaire ou qu'on se présente formellement en anglais,
on donne toujours son prénom puis son nom à l'inverse du français"""
def serialized(self):
    player_infos = {}
    player_infos['Prenom'] = self.first_name
    player_infos['Nom'] = self.last_name
    player_infos['Date_de_naissance'] = self.birthdate
    player_infos['Sexe_ou_Genre'] = self.gender
    player_infos['Classement'] = self.ranking
    player_infos['Score'] = self.score
    player_infos['Id du joueur'] = self.player_id
    return player_infos

def unserialized(self, serialized_player):
    first_name = serialized_player['Prenom']
    last_name = serialized_player['Nom']
    birthdate = serialized_player['Date_de_naissance']
    gender = serialized_player['Sexe_ou_Genre']
    ranking = serialized_player['Classement']
    tournament_score = serialized_player['Score']
    player_id = serialized_player['Id_du_joueur']
    return Player(first_name,
                   last_name,
                   birthdate,
                     gender,
                    ranking,
                      score,
                  player_id
                 )

def __str__(self):
    return f"first_name:{self.first_name} last_name:{self.last_name}, classement: {self.ranking}"

#def __repr__(self):
   # return f"first_name:{self.first_name} last_name:{self.last_name}, classement: {self.ranking}"


import torch
import pytorch_lightning as pl
from misc.utils import compute_loss, decode_text, decode_graph, compute_scores
import torch.nn.functional as F
import torch.nn as nn
import os
from model.grapher import Grapher

def sorted_ls(path):
    mtime = lambda f: os.stat(os.path.join(path, f)).st_mtime
    return list(sorted(os.listdir(path), key=mtime))

class FocalLoss(nn.modules.loss._WeightedLoss):

    def __init__(self, gamma, weight=None):
        super(FocalLoss, self).__init__(weight)
        self.gamma = gamma
        self.weight = weight

    def forward(self, input, target):

        # input: batch_size x vocab_size x d1 x ... x dn x seq_len
        # target: batch_size x d1 x ... x dn x seq_len
        ce_loss = F.cross_entropy(input, target, weight=self.weight, reduction='none')
        p = torch.exp(-ce_loss)
        focal_loss = ((1 - p) ** self.gamma * ce_loss)

        return focal_loss

class LitGrapher(pl.LightningModule):
    def __init__(self,
                 transformer_class,
                 transformer_name,
                 tokenizer,
                 cache_dir,
                 max_nodes,
                 max_edges,
                 edges_as_classes,
                 default_seq_len_edge,
                 num_classes,
                 dropout_rate,
                 num_layers,
                 vocab_size,
                 bos_token_id,
                 eos_token_id,
                 nonode_id,
                 noedge_id,
                 node_sep_id,
                 noedge_cl,
                 edge_classes,
                 focal_loss_gamma,
                 eval_dir,
                 lr,
                 ):
        super().__init__()
        self.save_hyperparameters()

        model = Grapher(transformer_class=transformer_class,
                        transformer_name=transformer_name,
                        cache_dir=cache_dir,
                        max_nodes=max_nodes,
                        edges_as_classes=edges_as_classes,
                        node_sep_id=node_sep_id,
                        default_seq_len_edge=default_seq_len_edge,
                        num_classes=num_classes,
                        dropout_rate=dropout_rate,
                        num_layers=num_layers,
                        vocab_size=vocab_size,
                        bos_token_id=bos_token_id)

        self.model = model
        self.criterion = {'ce': nn.CrossEntropyLoss(reduction='none'), 'focal': FocalLoss(focal_loss_gamma)}
        self.tokenizer = tokenizer
        self.cache_dir = cache_dir
        self.transformer_name = transformer_name
        self.edges_as_classes = edges_as_classes
        self.edge_classes = edge_classes
        self.focal_loss_gamma = focal_loss_gamma
        self.bos_token_id = bos_token_id
        self.eos_token_id = eos_token_id
        self.node_sep_id = node_sep_id
        self.max_nodes = max_nodes
        self.max_edges = max_edges
        self.noedge_cl = noedge_cl
        self.nonode_id = nonode_id
        self.noedge_id = noedge_id
        self.eval_dir=eval_dir
        self.lr = lr

    def training_step(self, batch, batch_idx):

        # target_nodes: batch_size X seq_len_node
        # target_edges: num_nodes X num_nodes X batch_size X seq_len_edge [FULL]
        # target_edges: batch_size X num_nodes X num_nodes [CLASSES]
        text_input_ids, text_input_attn_mask, target_nodes, target_nodes_mask, target_edges = batch

        # logits_nodes: batch_size X seq_len_node X vocab_size
        # logits_edges: num_nodes X num_nodes X batch_size X seq_len_edge X vocab_size [FULL]
        # logits_edges: num_nodes X num_nodes X batch_size X num_classes [CLASSES]
        logits_nodes, logits_edges= self.model(text_input_ids,
                                               text_input_attn_mask,
                                               target_nodes,
                                               target_nodes_mask,
                                               target_edges)

        loss = compute_loss(self.criterion, logits_nodes, logits_edges, target_nodes,
                            target_edges, self.edges_as_classes, self.focal_loss_gamma)

        self.log('train_loss', loss, on_step=True, on_epoch=True, logger=True, sync_dist=True, batch_size=text_input_ids.size(0))

        return loss

    def eval_step(self, batch, batch_idx, split):

        iteration = self.global_step
        rank = self.global_rank

        text_input_ids, text_input_attn_mask, target_nodes, target_nodes_mask, target_edges = batch

        logits_nodes, seq_nodes, logits_edges, seq_edges = self.model.sample(text_input_ids, text_input_attn_mask)

        text_dec = decode_text(self.tokenizer, text_input_ids, self.bos_token_id, self.eos_token_id)

        TB_str = []

        dec_target = decode_graph(self.tokenizer, self.edge_classes, target_nodes, target_edges, self.edges_as_classes,
                                  self.node_sep_id, self.max_nodes, self.noedge_cl, self.noedge_id,
                                  self.bos_token_id, self.eos_token_id)

        dec_pred = decode_graph(self.tokenizer, self.edge_classes, seq_nodes, seq_edges, self.edges_as_classes,
                                self.node_sep_id, self.max_nodes, self.noedge_cl, self.noedge_id,
                                self.bos_token_id, self.eos_token_id)

        if batch_idx == 0:
            for b_i in range(len(text_dec)):
                # ---- ground truth ----
                gt = '
'.join('-->'.join(tri) for tri in dec_target[b_i])

                # ---- predicted  -------
                pr = '
'.join('-->'.join(tri) for tri in dec_pred[b_i])

                strng = f'{b_i}
' + text_dec[b_i] + '
' \
                        + '-' * 40 + 'target' + '-' * 40 + '
' + gt + '
' \
                        + '-' * 40 + 'predicted' + '-' * 20 + '
' + pr + '
'
                TB_str.append(strng)

        decodes = {'text_dec': text_dec, 'dec_target': dec_target, 'dec_pred': dec_pred}

        for i, tb_str in enumerate(TB_str):
            self.logger.experiment.add_text(f'{split}_{rank}/{i}', tb_str, iteration)

        return decodes

    def eval_epoch_end(self, outputs, split):

        iteration = self.global_step
        rank = self.global_rank

        dec_target_all = []
        dec_pred_all = []

        for out in outputs:
            dec_target_all += out['dec_target']
            dec_pred_all += out['dec_pred']

        # make sure number of paths is smaller than 10
        dec_pred_all = [tr[:10] for tr in dec_pred_all]
        dec_target_all = [tr[:10] for tr in dec_target_all]

        # hack to avoid crashing the program if evaluation fails
        try:
            scores = compute_scores(dec_pred_all, dec_target_all, iteration, self.eval_dir, split, rank)
        except Exception:
            return

        for k, v in scores.items():
            self.logger.experiment.add_scalar(f'{split}_score/{k}', v, global_step=iteration)

        self.log_dict(scores)

    def validation_step(self, batch, batch_idx):
        return self.eval_step(batch, batch_idx, 'valid')

    def test_step(self, batch, batch_idx):
        return self.eval_step(batch, batch_idx, 'test')

    def validation_epoch_end(self, outputs):
        self.eval_epoch_end(outputs, 'valid')

    def test_epoch_end(self, outputs):
        self.eval_epoch_end(outputs, 'test')

    def configure_optimizers(self):
        optimizer = torch.optim.AdamW(self.parameters(), self.lr)
        return optimizer


from torch.utils.tensorboard import SummaryWriter
from utils.plotting import get_alignment_figure, get_specs_figure

class TBLogger(SummaryWriter):
    def __init__(self, log_dir):
        super(TBLogger, self).__init__(log_dir)

    def add_training_data(self, meta, grad_norm,
                          learning_rate, tb_step: int):

        for k, v in meta.items():
            self.add_scalar(f'train/{k}', v.item(), tb_step)
        self.add_scalar("train/grad_norm", grad_norm, tb_step)
        self.add_scalar("train/learning_rate", learning_rate, tb_step)

    def add_parameters(self, model, tb_step: int):

        for tag, value in model.named_parameters():
            tag = tag.replace('.', '/')
            self.add_histogram(tag, value.data.cpu().numpy(), tb_step)

    def add_sample(self, alignment, mel_pred,
                   mel_targ, mel_infer, len_targ,
                   tb_step: int):

        self.add_figure(
            "alignment",
            get_alignment_figure(alignment.detach().cpu().numpy().T),
            tb_step)

        self.add_figure(
            "spectrograms",
            get_specs_figure([
                mel_infer.detach().cpu().numpy(),
                mel_pred[:, :len_targ].detach().cpu().numpy(),
                mel_targ[:, :len_targ].detach().cpu().numpy(),
            ],
                ['Frames (inferred)', 'Frames (predicted)', 'Frames (target)']
            ), tb_step)


import matplotlib.pyplot as plt  # type: ignore
import matplotlib.ticker as ticker
from matplotlib.patches import Patch
import numpy as np
import os
from typing import Dict
import json
import pandas as pd
from typing import NewType, List
import numpy.typing as npt
import matplotlib.ticker as mticker
import string

# Global vars for tracking and labeling data at load time.
exp_idx = 0
units: Dict = dict()
Color = NewType("Color", npt.NDArray[np.float64])
Colorcode = NewType("Colorcode", List[int])

COLOR_FACTOR = 0.75  # How much to lighten previous color by
COLORS = [
    Colorcode([75, 0, 130]),  # Indigo
    Colorcode([0, 0, 255]),  # Blue
    # Colorcode([255, 105, 180]),  # Pink (Removed due to being too similar to the red median line)
    Colorcode([0, 255, 0]),  # Green
    Colorcode([255, 127, 0]),  # Orange
    #Colorcode([148, 0, 211]),  # Violet (Removed due to being too similar to indigo)
]

# Helper Functions
def create_color(id: int) -> Color:
    """Pick initial Colorcode based on id number, then offset it"""
    specific_color: Colorcode = COLORS[id % (len(COLORS))]  #
    if id > (len(COLORS) - 1):
        offset: int = (id * 22) % 255  # Create large offset for that base color, bounded by 255
        specific_color[id % 3] = (255 + specific_color[id % 3] - offset) % 255  # Perform the offset
    return Color(np.array(specific_color) / 255)

def lighten_color(color: Color, factor: float) -> Color:
    """increase tint of a color"""
    scaled_color = color * 255  # return to original scale
    return Color(np.array(list(map(lambda c: (c + (255 - c) * factor) / 255, scaled_color))))

def parse_exp_name(name, components, groups, exclude):
    first_split = name.split(" ")
    parts = []

    for name in first_split:
        parts += name.split("_")
    result = None
    category = None
    exclude_flag = False
    excluded = []
    for part in parts:
        if part in exclude:
            exclude_flag = True
            excluded += part
        elif part in components:
            assert result is None, "Multiple results match"
            result = part
        elif part in groups:
            assert category is None, "Multiple categories match"
            category = part

    return (result, category, None) if not exclude_flag else (None, None, excluded)

def get_data(logdir, condition=None):
    """
    Recursively look through logdir for results files. Assumes that any file "results_raw.json" is a valid hit.

    :param logdir: Root directory where data is contained.
    :param condition: Directory name to filter on. Ex: 'test1'
    """
    global exp_idx
    global units
    datasets = []
    for root, _, files in os.walk(logdir):
        if "results.json" in files:
            last_dir = os.path.split(root)[-1]

            # If filtering on a condition, only read in data from said condition.
            if condition:
                if condition in last_dir:
                    exp_idx += 1

                    with open(os.path.join(root, "results.json"), "r") as f:
                        exp_data = json.load(f)
                    f.close()

                    exp_data["experiment"] = last_dir

                    datasets.append(exp_data)
            else:
                with open(os.path.join(root, "results.json"), "r") as f:
                    exp_data = json.load(f)
                f.close()

                exp_data["experiment"] = last_dir

                datasets.append(exp_data)
    return datasets

def parse_data(data, components, groups, exclude, performance_markers):
    """Parse data into plt compatible datasets"""

    # Goal: have 3 graphs (lists), each containing each component (list) that have a tests (dict of stats):

    # TODO use sorting method and statically generated arrays instead of two loops and dicts
    # Eventual graphs to make - because run may be out of order, making a dict first
    accuracy_datasets = [[None for _ in range(len(groups))] for _ in range(len(components))]
    speed_datasets = [[None for _ in range(len(groups))] for _ in range(len(components))]
    score_datasets = [[None for _ in range(len(groups))] for _ in range(len(components))]

    # accuracy_sorted = {component: {test: None for test in groups} for component in components}
    # speed_sorted = {component: {test: None for test in groups} for component in components}
    # score_sorted = {component: {test: None for test in groups} for component in components}

    accuracy_sorted = {component: {} for component in components}
    speed_sorted = {component: {} for component in components}
    score_sorted = {component: {} for component in components}

    # Pulled from all different files
    for run in data:
        # Parse which component this is
        (comp, test, excluded) = parse_exp_name(name=run["experiment"], components=components, groups=groups, exclude=exclude)

        # Add component results to graph dicts (if not an excluded result)
        if comp and test:
            # Only one test per save, thus [0]
            accuracy_sorted[comp][test] = run["accuracy"][0] if isinstance(run["accuracy"], list) else run["accuracy"]
            speed_sorted[comp][test] = run["speed"][0] if isinstance(run["speed"], list) else run["speed"]
            score_sorted[comp][test] = run["score"][0] if isinstance(run["score"], list) else run["score"]

    # Rearrange in specified order
    for ci, component in enumerate(components):
        for ti, test in enumerate(groups):
            if test in accuracy_sorted[component]:
                accuracy_datasets[ci][ti] = accuracy_sorted[component][test]
                speed_datasets[ci][ti] = speed_sorted[component][test]
                score_datasets[ci][ti] = score_sorted[component][test]

    return accuracy_datasets, speed_datasets, score_datasets

def mock_data():
    low_whisker = 6.8
    q1 = 6.9
    median = 7.0
    q3 = 8.0
    high_whisker = 8.59

    # Set up stats way
    athens = [
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
    ]
    beijing = [
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
    ]
    london = [
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
    ]
    rio = [
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
    ]

    return [athens, beijing, london, rio]

def mock_data(small):
    low_whisker = 6.8
    q1 = 6.9
    median = 7.0
    q3 = 8.0
    high_whisker = 8.59

    # Set up stats way
    athens = [
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
    ]
    beijing = [
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
    ]
    london = [
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
    ]
    rio = [
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
        {"med": median, "q1": q1, "q3": q3, "whislo": low_whisker, "whishi": high_whisker},
    ]

    return [athens]

def plot(graphname, datasets, groups, tests, y_label, path=None, inverty=False, modes=None):
    # Datasets correspond to components. Groups correspond to tests

    # Make figures A6 in size
    A = 6
    plt.rc("figure", figsize=[46.82 * 0.5 ** (0.5 * A), 33.11 * 0.5 ** (0.5 * A)])
    # Use Latex
    plt.rc("text", usetex=True)
    plt.rc("font", family="serif")

    # Define which colours you want to use
    colors = []
    for id in range(len(groups)):
        colors.append(create_color(id))

    # Set x-positions for boxes
    x_pos_range = np.arange(len(datasets)) / (len(datasets) - 1) if len(datasets) > 1 else np.array(1)
    x_pos = (x_pos_range * 0.5) + 0.75 if x_pos_range.size > 1 else x_pos_range

    _, ax = plt.subplots()

    # Set labels
    for i in range(len(tests)):
        tests[i] = string.capwords(tests[i][0] + ' ' + tests[i][1:])

    ax.set_xticklabels(tests)

    # Plot each component
    for i, data_to_plot in enumerate(datasets):
        if len(data_to_plot[0]) != 0:
            # positions = [x_pos[i] + j * 1 for j in range(len(data.T))]
            if x_pos_range.size > 1:
                positions = [x_pos[i] + j * 1 for j in range(len(data_to_plot))]
            else:
                positions = [x_pos + j * 1 for j in range(len(data_to_plot))]

            # TODO do this in data parser
            # stats = list()
            # for test in data:
            #     stats.append({'med': data[test]['med'], 'q1':  data[test]['q1'], 'q3':  data[test]['q3'], 'whislo':  data[test]['whislo'], 'whishi': data[test]['whishi']})

            bp = ax.bxp(
                # stats,
                data_to_plot,
                showfliers=False,
                positions=positions,
                patch_artist=True,
                widths=0.6 / len(datasets),
            )

            # Fill the boxes with colours (requires patch_artist=True)
            k = i % len(colors)
            for box in bp["boxes"]:
                box.set(facecolor=colors[k])

            # Make the median lines more visible
            plt.setp(bp["medians"], color="red")

    # If invert y, invert - helps with visualizing things where minimizing the value is the goal (speed etc)
    if inverty:
        plt.gca().invert_yaxis()

    # Titles
    plt.ylabel(y_label)

    # Axis ticks and labels
    plt.xticks(np.arange(len(list(datasets[0]))) + 1)
    plt.gca().xaxis.set_minor_locator(ticker.FixedLocator(np.array(range(len(list(datasets[0])) + 1)) + 0.5))
    plt.gca().tick_params(axis="x", which="minor", length=4)
    plt.gca().tick_params(axis="x", which="major", length=0)

    # Change the limits of the x-axis
    plt.xlim([0.5, len(list(datasets[0])) + 0.5])

    # Legend
    legend_elements = []
    for i in range(len(datasets)):
        j = i % len(groups)
        k = i % len(colors)
        legend_elements.append(Patch(facecolor=colors[k], label=modes[groups[j]]))
    plt.legend(handles=legend_elements, fontsize=8)

    if path:
        plt.savefig(str(path) + f"/{graphname}_result.png", format="png")
        print(f"Saved to {path}/{graphname}_result.png")
        plt.savefig(str(path) + f"/{graphname}_result.eps", format="eps")
    else:
        plt.show()

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--data_dir",
        type=str,
        help="Directory where results are saved. Ex: ../models/train/gru_8_acts/bpf/model_dir",
        default=".",  # noqa
    )
    parser.add_argument("--save", type=bool, default=True)
    args = parser.parse_args()

    if args.data_dir == ".":
        args.data_dir = os.getcwd() + "/"
    args.data_dir = args.data_dir + "/" if args.data_dir[-1] != "/" else args.data_dir

    # Set up name parsing
    groups = ["1agent", "2agent", "4agent"]

    # Groups to represent in each x tick group
    # components = ["env", "PPO", "Optimizer", "StatBuf", "CNN"]
    components = ["collab", "coop", "control"]

    # Results to exclude from plotting
    exclude = [""]

    condition = None

    performance_markers = {
        "accuracy": "Accuracy [percentage]",
        "score": "Cumulative Return [raw]",
        "speed": "Speed [steps]",
    }

    modes = {
        "collab": "BTBE",
        "coop": "CTDE",
        "control": "RAD-A2C"
    }

    data = get_data(logdir=args.data_dir, condition=condition)

    accuracy_datasets, speed_datasets, score_datasets = parse_data(
        data=data, components=components, groups=groups, exclude=exclude, performance_markers=performance_markers
    )

    for graphname, graph in zip(
        [performance_markers["accuracy"], performance_markers["speed"], performance_markers["score"]],
        [accuracy_datasets, speed_datasets, score_datasets],
    ):
        inverty = True if graphname in performance_markers["speed"] else False
        # try:
        plot(graphname=graphname, datasets=graph, groups=components, tests=groups, y_label=graphname, path=os.getcwd(), inverty=inverty, modes=modes)
        # except Exception as e:
        #     print(e)

    print("Done with plot")