# Напишите программу, которая будет преобразовывать десятичное число в двоичное.
# Без использования встроенной функции преобразования, без строк.

def binarize(number):
    number_binarized = 0
    list_of_digits = []
    while number > 0:
        list_of_digits.append(number % 2)
        number //= 2
    for i in range(len(list_of_digits)):
        number_binarized += list_of_digits[i] * 10 ** i
    return number_binarized

number = int(input('Enter your number: '))
print(binarize(number))


import time

import pandas as pd
from pandas import DataFrame, Series
import numpy as np
from itertools import permutations

pd.options.mode.chained_assignment = None  # default='warn'

def build_empty_assignment_detail(users):
    """build the empty assignment detail obj from a list of users"""
    result = {user: {'total_time_used': 0, 'own_tasks': []} for user in users}
    return result

def assign_sub_matrix(sub_performance_matrix, assignment_detail):
    """Calculate the VDC users with sub-tasks"""
    users = sub_performance_matrix.columns.tolist()
    task_count = len(sub_performance_matrix.index)  # How many tasks in this sub-matrix, maybe less then users

    # Add offset to each users' baseline
    for idx, user in enumerate(users):
        sub_performance_matrix[user] += assignment_detail[user]['total_time_used']

    user_permutations = permutations(users, task_count)  # The permutations of all users on tasks = A(users, task_count)

    minimum_permutation_time_used = -1
    minimum_permutation = None

    # Loop each permutation to find the smallest
    for idx_p, permutation in enumerate(user_permutations):
        # print(permutation)

        maximum_time_used_in_permutation = -1  # the time_used of smallest permutation
        for idx_u, user in enumerate(permutation):
            time_used_for_user_on_task = sub_performance_matrix.iloc[idx_u][user]

            # the max time used in one permutation
            maximum_time_used_in_permutation = time_used_for_user_on_task if (
                time_used_for_user_on_task > maximum_time_used_in_permutation) else maximum_time_used_in_permutation

        # Find the smallest max time used permutation
        if (maximum_time_used_in_permutation < minimum_permutation_time_used) or (minimum_permutation_time_used == -1):
            minimum_permutation_time_used = maximum_time_used_in_permutation
            minimum_permutation = permutation

    print('Smallest permutation: ', minimum_permutation)

    for idx, user in enumerate(minimum_permutation):
        time_used_for_user_on_task = sub_performance_matrix.iloc[idx][user]
        task_name = sub_performance_matrix.index[idx]
        assignment_detail[user]['total_time_used'] = time_used_for_user_on_task
        assignment_detail[user]['own_tasks'].append(task_name)

    print(sub_performance_matrix)
    return

def assign_tasks(performance_matrix, assignment_detail):
    users = performance_matrix.columns.tolist()  # the columns of the DataFrame is userIds
    user_count = len(users)
    task_count = len(performance_matrix.index)  # The number of tasks
    # offsets = [0] * user_count  # Offsets means the init value of each user when starting a round of calculation

    start_row = 0  # the start index of task in a round of calculation
    stop_row = 0
    remain_task_count = task_count  # How many tasks left to be calculated

    # Loop of each round of calculation
    while remain_task_count > 0:
        # Split the sub matrix from performance_matrix
        stop_row += remain_task_count if (remain_task_count < user_count) else user_count
        sub_performance_matrix = performance_matrix.iloc[start_row:stop_row]

        # Do calculation!
        assign_sub_matrix(sub_performance_matrix, assignment_detail)

        # Update the remain count & start index after one round
        remain_task_count -= (stop_row - start_row)
        start_row = stop_row

        print(assignment_detail)

    return

def main():
    # Sample data
    users = ['P1', 'P2', 'P3']
    performance_2darray = np.array([
        [2, 5, 3],
        [2, 1, 4],
        [4, 3, 1],
        [3, 2, 1],
        [4, 5, 8],
        [7, 4, 4],

        [6, 3, 3],
        [3, 5, 2]
    ])
    performance_matrix = DataFrame(performance_2darray,
                                   columns=users,
                                   index=['T1', 'T2', 'T3', 'T4', 'T5', 'T6', 'T7', 'T8'])

    assignment_detail = build_empty_assignment_detail(users)

    assign_tasks(performance_matrix, assignment_detail)
    return

if __name__ == '__main__':
    start_time = time.time()
    main()
    stop_time = time.time()
    print('用时: ', (stop_time - start_time), ' (s)')


# mysql/__init__.py
# Copyright (C) 2005-2023 the SQLAlchemy authors and contributors
# <see AUTHORS file>
#
# This module is part of SQLAlchemy and is released under
# the MIT License: https://www.opensource.org/licenses/mit-license.php

from ...util import compat
from . import base  # noqa
from . import cymysql  # noqa
from . import mariadbconnector  # noqa
from . import mysqlconnector  # noqa
from . import mysqldb  # noqa
from . import oursql  # noqa
from . import pymysql  # noqa
from . import pyodbc  # noqa
from .base import (BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, DATETIME,
                   DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, JSON, LONGBLOB,
                   LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, NUMERIC,
                   NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP,
                   TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR)
from .dml import Insert, insert
from .expression import match

if compat.py3k:
    from . import aiomysql  # noqa
    from . import asyncmy  # noqa

# default dialect
base.dialect = dialect = mysqldb.dialect

__all__ = (
    "BIGINT",
    "BINARY",
    "BIT",
    "BLOB",
    "BOOLEAN",
    "CHAR",
    "DATE",
    "DATETIME",
    "DECIMAL",
    "DOUBLE",
    "ENUM",
    "DECIMAL",
    "FLOAT",
    "INTEGER",
    "INTEGER",
    "JSON",
    "LONGBLOB",
    "LONGTEXT",
    "MEDIUMBLOB",
    "MEDIUMINT",
    "MEDIUMTEXT",
    "NCHAR",
    "NVARCHAR",
    "NUMERIC",
    "SET",
    "SMALLINT",
    "REAL",
    "TEXT",
    "TIME",
    "TIMESTAMP",
    "TINYBLOB",
    "TINYINT",
    "TINYTEXT",
    "VARBINARY",
    "VARCHAR",
    "YEAR",
    "dialect",
    "insert",
    "Insert",
    "match",
)


#!/usr/bin/env waf

import os.path as osp

VERSION='0.0.0'
APPNAME='zio'

top = '.'
out = 'build'

def options(opt):
    opt.load('compiler_cxx')
    opt.load('waf_unit_test')

def configure(cfg):
    cfg.env.CXXFLAGS += ['-std=c++17', '-g', '-O2']
    cfg.load('compiler_cxx')
    cfg.load('waf_unit_test')

    cfg.env.LIBPATH_HDF5 = [osp.expandvars('$HOME/opt/hdf5/lib')]
    cfg.env.INCLUDES_HDF5 = [osp.expandvars('$HOME/opt/hdf5/include')]
    cfg.env.INCLUDES_H5CPPORG = [osp.expandvars('$HOME/opt/h5cpp/include')]

    cfg.check_cxx(
        msg = "Checking for HDF5",
        header_name = ["hdf5.h", "hdf5_hl.h"],
        lib = ['hdf5_hl', 'hdf5', 'z'],
        uselib_store = 'HDF5',
        use = ['HDF5'],
    )

    cfg.check_cxx(
        msg = "Checking for h5cpp.org",
        header_name = 'h5cpp/all',
        uselib_store = 'H5CPPORG',
        use = ['HDF5', 'H5CPPORG'],
    )

def build(bld):
    uses='HDF5 H5CPPORG'.split()

    rpath = [bld.env["PREFIX"] + '/lib', bld.path.find_or_declare(bld.out_dir)]
    print (rpath)
    rpath += [bld.env["LIBPATH_%s"%u][0] for u in uses if bld.env["LIBPATH_%s"%u]]
    rpath = list(set(rpath))

    tsources = bld.path.ant_glob('test/test*.cpp')
    for tmain in tsources:
        bld.program(features = 'test cxx',
                    source = [tmain], target = tmain.name.replace('.cpp',''),
                    ut_cwd = bld.path,
                    install_path = None,
                    includes = ['inc','build','test'],
                    rpath = rpath,
                    use = uses)

    from waflib.Tools import waf_unit_test
    bld.add_post_fun(waf_unit_test.summary)


from divisible import __version__
from divisible.main import is_divisible, is_even, is_odd

import pytest

def test_version():
    assert __version__ == "0.1.0"

@pytest.mark.parametrize(
    "numerator,denominator,expected",
    [(10, 2, True), (10, 3, False), (9, 7, False), (9, 10, False), (10, 0, False)],
)
def test_is_divisible(numerator, denominator, expected):
    assert is_divisible(numerator, denominator) == expected

@pytest.mark.parametrize(
    "number,expected",
    [(10, True), (3, False), (-2, True), (0, True)]
)
def test_is_even(number: int, expected: bool):
    assert is_even(number) == expected

@pytest.mark.parametrize(
    "number,expected",
    [(10, False), (3, True), (-5, True), (0, False)]
)
def test_is_odd(number: int, expected: bool):
    assert is_odd(number) == expected


# -*- coding:utf-8 -*-
"""
Description: A python 2.7 implementation of gcForest proposed in [1]. A demo implementation of gcForest library as well as some demo client scripts to demostrate how to use the code. The implementation is flexible enough for modifying the model or
fit your own datasets.
Reference: [1] Z.-H. Zhou and J. Feng. Deep Forest: Towards an Alternative to Deep Neural Networks. In IJCAI-2017.  (https://arxiv.org/abs/1702.08835v2 )
Requirements: This package is developed with Python 2.7, please make sure all the demendencies are installed, which is specified in requirements.txt
ATTN: This package is free for academic usage. You can run it at your own risk. For other purposes, please contact Prof. Zhi-Hua Zhou(zhouzh@lamda.nju.edu.cn)
ATTN2: This package was developed by Mr.Ji Feng(fengj@lamda.nju.edu.cn). The readme file and demo roughly explains how to use the codes. For any problem concerning the codes, please feel free to contact Mr.Feng.
"""
import numpy as np
from sklearn.model_selection import StratifiedKFold

from .base_layer import BaseLayer
from ..estimators import get_estimator_kfold
from ..utils.metrics import accuracy_pb, accuracy_win_vote, accuracy_win_avg
from ..utils.win_utils import get_windows
from ..utils.debug_utils import repr_blobs_shape
from ..utils.log_utils import get_logger

LOGGER = get_logger("gcforest.layers.fg_win_layer")

#CV_POLICYS = ["data", "win"]
#CV_POLICYS = ["data"]

class FGWinLayer(BaseLayer):
    def __init__(self, layer_config, data_cache):
        """
        est_config (dict):
            estimator的config
        win_x, win_y, stride_x, stride_y, pad_x, pad_y (int):
            configs for windows
        n_folds(int): default=1
             1 means do not use k-fold
        n_classes (int):

        """
        super(FGWinLayer, self).__init__(layer_config, data_cache)
        # estimator
        self.est_configs = self.get_value("estimators", None, list, required=True)
        self.win_x = self.get_value("win_x", None, int, required=True)
        self.win_y = self.get_value("win_y", None, int, required=True)
        self.stride_x = self.get_value("stride_x", 1, int)
        self.stride_y = self.get_value("stride_y", 1, int)
        self.pad_x = self.get_value("pad_x", 0, int)
        self.pad_y = self.get_value("pad_y", 0, int)
        self.n_classes = self.get_value("n_classes", None, int, required=True)
        #self.cv_policy = layer_config.get("cv_policy", "data")
        #assert(self.cv_policy in CV_POLICYS)
        assert len(self.bottom_names) >= 2
        assert len(self.est_configs) == len(self.top_names), "Each estimator shoud produce one unique top"
        # self.eval_metrics = [("predict", accuracy_pb), ("vote", accuracy_win_vote), ("avg", accuracy_win_avg)]
        self.eval_metrics = [("predict", accuracy_pb), ("avg", accuracy_win_avg)]
        self.estimator1d = [None for ei in range(len(self.est_configs))]

    def _init_estimators(self, ei, random_state):
        """
        ei (int): estimator index
        """
        top_name = self.top_names[ei]
        est_args = self.est_configs[ei].copy()
        est_name ="{}/{}_folds".format(top_name, est_args["n_folds"])
        # n_folds
        n_folds = int(est_args["n_folds"])
        est_args.pop("n_folds")
        # est_type
        est_type = est_args["type"]
        est_args.pop("type")
        # random_state
        random_state = (random_state + hash("[estimator] {}".format(est_name))) % 1000000007
        return get_estimator_kfold(est_name, n_folds, est_type, est_args, random_state=random_state)

    def fit_transform(self, train_config):
        LOGGER.info("[data][{}], bottoms={}, tops={}".format(self.name, self.bottom_names, self.top_names))
        phases = train_config.phases
        X_train_win, y_train_win = None, None
        test_sets = None

        for ti, top_name in enumerate(self.top_names):
            LOGGER.info("[progress][{}] ti={}/{}, top_name={}".format(self.name, ti, len(self.top_names), top_name))
            # check top cache
            if np.all(self.check_top_cache(phases, ti)):
                LOGGER.info("[data][{}] all top cache exists. skip progress".format(self.name))
                continue

            # init X, y, n_classes
            if X_train_win is None:
                for pi, phase in enumerate(phases):
                    bottoms = self.data_cache.gets(phase, self.bottom_names)
                    LOGGER.info('[data][{},{}] bottoms.shape={}'.format(self.name, phase, repr_blobs_shape(bottoms)))
                    X, y = np.concatenate(bottoms[:-1], axis=1), bottoms[-1]
                    # n x n_windows x channel
                    X_win = get_windows(X, self.win_x, self.win_y, self.stride_x, self.stride_y, self.pad_x, self.pad_y)
                    _, nh, nw, _ = X_win.shape
                    X_win = X_win.reshape((X_win.shape[0], -1, X_win.shape[-1]))
                    y_win = y[:,np.newaxis].repeat(X_win.shape[1], axis=1)
                    if pi == 0:
                        assert self.n_classes == len(np.unique(y)), \
                                "n_classes={}, len(unique(y))={}".format(self.n_classes, len(np.unique(y)))
                        X_train_win, y_train_win = X_win, y_win
                    else:
                        test_sets = [("test", X_win, y_win), ]

            # fit
            est = self._init_estimators(ti, train_config.random_state)
            y_probas = est.fit_transform(X_train_win, y_train_win, y_train_win[:,0], cache_dir=train_config.model_cache_dir,
                    test_sets = test_sets, eval_metrics=self.eval_metrics,
                    keep_model_in_mem=train_config.keep_model_in_mem)

            for pi, phase in enumerate(phases):
                y_proba = y_probas[pi].reshape((-1, nh, nw, self.n_classes)).transpose((0, 3, 1, 2))
                LOGGER.info('[data][{},{}] tops[{}].shape={}'.format(self.name, phase, ti, y_proba.shape))
                self.data_cache.update(phase, self.top_names[ti], y_proba)
            if train_config.keep_model_in_mem:
                self.estimator1d[ti] = est

    def transform(self):
        phase = "test"
        for ti, top_name in enumerate(self.top_names):
            LOGGER.info("[progress][{}] ti={}/{}, top_name={}".format(self.name, ti, len(self.top_names), top_name))

            bottoms = self.data_cache.gets(phase, self.bottom_names[:-1])
            LOGGER.info('[data][{},{}] bottoms.shape={}'.format(self.name, phase, repr_blobs_shape(bottoms)))
            X = np.concatenate(bottoms, axis=1)
            # n x n_windows x channel
            X_win = get_windows(X, self.win_x, self.win_y, self.stride_x, self.stride_y, self.pad_x, self.pad_y)
            _, nh, nw, _ = X_win.shape
            X_win = X_win.reshape((X_win.shape[0], -1, X_win.shape[-1]))

            est = self.estimator1d[ti]
            y_proba = est.predict_proba(X_win)
            y_proba = y_proba.reshape((-1, nh, nw, self.n_classes)).transpose((0, 3, 1, 2))
            LOGGER.info('[data][{},{}] tops[{}].shape={}'.format(self.name, phase, ti, y_proba.shape))
            self.data_cache.update(phase, self.top_names[ti], y_proba)

    def score(self):
        eval_metrics = [("predict", accuracy_pb), ("avg", accuracy_win_avg)]
        for ti, top_name in enumerate(self.top_names):
            for phase in ["train", "test"]:
                y = self.data_cache.get(phase, self.bottom_names[-1])
                y_proba = self.data_cache.get(phase, top_name)
                y_proba = y_proba.transpose((0,2,3,1))
                y_proba = y_proba.reshape((y_proba.shape[0], -1, y_proba.shape[3]))
                y = y[:,np.newaxis].repeat(y_proba.shape[1], axis=1)
                for eval_name, eval_metric in eval_metrics:
                    acc = eval_metric(y, y_proba)
                    LOGGER.info("Accuracy({}.{}.{})={:.2f}%".format(top_name, phase, eval_name, acc*100))


"""
- 투 포인터

idea> 부분합 체크!
- 두개의 포인터는 맨 처음부터 시작한다! 어차피 배열 끝까지 탐색해야되므로 end포인터(p2)는 for문으로 해줌
- 차례대로 sub_total을 구해준 후, sub_total >= s 가 되는 경우 p1 포인터를 하나씩 땡겨준다. (조건을 만족하는 최소 길이를 구해야 하므로)
- p1을 땡기다가 sub_total < s 가 되는 경우 다시 p2 포인터를 증가시킨다.
- p1, p2 모두 n-1이 되면 탐색 종료
"""
import sys
input = sys.stdin.readline

n, s = map(int, input().split())
arr = list(map(int, input().split()))

answer = n + 1
p1 = 0
sub_total = 0
for p2 in range(n):
    sub_total += arr[p2]

    while sub_total >= s and p1 <= p2:
        answer = min(answer, p2 - p1 + 1)
        sub_total -= arr[p1]
        p1 += 1

print(answer if answer != n+1 else 0)


import os
import sys
sys.path.append("..")
from BetaGo.training.supervised_policy_trainer import run_training
import unittest

class TestSupervisedPolicyTrainer(unittest.TestCase):
    def testTrain(self):
        model = 'test_data/minimodel.json'
        data = 'test_data/hdf5/alphago-vs-lee-sedol-features.hdf5'
        output = 'test_data/.tmp.training/'
        args = [model, data, output, '--epochs', '1']
        run_training(args)

        os.remove(os.path.join(output, 'metadata.json'))
        os.remove(os.path.join(output, 'shuffle.npz'))
        os.remove(os.path.join(output, 'weights.00000.hdf5'))
        os.rmdir(output)

if __name__ == '__main__':
    unittest.main()


import random

class Conway_Base:
    "Base class for a Conway's game of life simulation"

    def __init__(self,row,col):
        """
        Initialises the class
        Takes in length of a square grid
        Creates an array that will hold the state of each grid element
        Precalculates the array indicies that are corners and edges

        data input: a single int for the edge length of the grid
        data output: A list that stores the grids states
                     A series of tuples that stores the indicies of the grid elements that are on the corners and the edges
        """
        #####Declerations#####
        ####Variables####
        #Initialises the grid dimensions
        self.row = row
        self.col = col

        #####Computations#####

    def create_grid(self):
        """
        creates an initial randomised grid layout

        data input: The int variables for the number of rows and columns, and the grid storage list filled with dead elements
        data output: The grid storage list with randomised alive and dead elements
        """
        #####Declerations#####

        #####Computations#####
        #Loops through each grid element. If the random number generator produces less than 0.5 it floors
        #Others the element is one
        self.grid = [0 if random.random() < 0.5 else 1 for x in range(self.row * self.col)]

        #Pre-calculates the edges and corner positions in the array
        self.edge_and_corner_calc()

    def edge_and_corner_calc(self):
        """
        Does the pre-calculations of all the corners and edges positions in the grid

        data input: Int values of the number of rows and columns
        data output: A Corner tuple of length 4, each element is a corner index
                     4 tuples that store the grid indicies for the edge elements
        """
        #####Declerations#####

        #####Computations#####
        #Creates an array with the corner positions of the grid in the array
        #Ordered top left, top right, bottom left, bottom right
        self.corner = (0, self.col-1, (self.row - 1) * self.col, self.row * self. col - 1)

        #Edge arrays
        #Top edge
        self.top_edge = tuple([i for i in range(1, self.col-1)])
        #Left edge
        self.left_edge = tuple([i for i in range(self.col, (self.row - 2)* self.col + 1, self.col)])
        #Right edge
        self.right_edge = tuple([i for i in range(2 * self.col - 1, (self.row - 1) * self.col, self.col)])
        #Bottom edge
        self.bottom_edge = tuple([i for i in range((self.row - 1) * self.col + 1, self.row * self. col - 1)])

    def print_grid(self):
        """
        Prints the conway grid in the terminal

        data input: The int variables for the number of rows and columns, and the randomised grid storage list
        data output: n/a
        """
        #####Declerations#####

        #####Computations#####
        #Loops through each row in the grid
        for i in range(self.row):
            #Prints out each row of the grid
            print(self.grid[0 + self.col*i : self.col * (1 + i)] )

        #New line to seperate the grids in the terminal
        print('\n')

    def grid_update(self):
        """
        Updates the grid for each iteration of the run

        data input: The int variables for the number of rows and columns, and the randomised grid storage array
        data output: The grid storage list with updated states for all the elements
        """
        #####Declerations#####
        ####Data Structures####
        #Makes an immutable copy of the current state of the grid
        reference = tuple(self.grid)
        ####Variables####
        #Variable for holding the number of neighbours of each grid
        sum = 0

        #####Computations#####
        #Loops through each element in the grid
        for i in range(len(self.grid)):
            # CORNERS
            #Checks if the current element is one of the corners
            #Each corner element has three neighbours
            #Top left corner
            if (i == self.corner[0]):
                #Sums the state three grid neighbours of the top left corner element
                sum = (reference[i + 1] +
                reference[i + self.col] + reference[i + self.col + 1])

            #Top Right corner
            elif(i == self.corner[1]):
                sum = (reference[i - 1] +
                reference[i + self.col - 1] + reference[i + self.col])

            #Bottom left corner
            elif(i == self.corner[2]):
                sum = (reference[i - self.col] + reference[i - self.col + 1] +
                reference[i + 1])

            #Bottom right corner
            elif(i == self.corner[3]):
                sum = (reference[i - (self.col + 1)] + reference[i - self.col] +
                reference[i - 1])

            #EDGE
            #Checks if the current element is an edge element
            #Each edge element has five neighbours
            #Top Edge
            elif(i in self.top_edge):
                sum = (reference[i - 1] + reference[i + 1] +
                reference[i + self.col - 1] + reference[i + self.col] + reference[i + self.col + 1])

            #Left Edge
            elif(i in self.left_edge):
                sum = (reference[i - self.col] + reference[i - self.col + 1] +
                reference[i + 1] +
                reference[i + self.col] + reference[i + self.col + 1])

            #Right Edge
            elif(i in self.right_edge):
                sum = (reference[i - (self.col + 1)] + reference[i - self.col] +
                reference[i - 1] +
                reference[i + self.col - 1] + reference[i + self.col])

            #Bottom Edge
            elif(i in self.bottom_edge):
                sum = (reference[i - (self.col + 1)] + reference[i - self.col] + reference[i + 1 - self.col] +
                reference[i - 1] + reference[i + 1])

            #Core positions
            #Each core position has eight neighbours
            else:
                sum = (reference[i - self.col - 1] + reference[i - self.col] + reference[i - self.col + 1] +
                reference[i - 1] + reference[i + 1] +
                reference[i + self.col - 1] + reference[i + self.col] + reference[i + self.col + 1])

            #Checks the sum of the elements neighbours and updates the state
            self.conway_life_death_comp(sum, i)

            ####Garbage####
            #Resets the neighbours sum
            sum = 0

    def conway_life_death_comp(self, sum, index):
        """
        Does the comparison to see if the state is alive or dead in the next iteration

        data input: The current element index (int), the sum value for the current element (int <= 8) and the grid state list
        data output: The grid state list with the index updated for the next iteration
        """
        #####Declerations#####

        #####Computations#####
        #If the state is alive
        if self.grid[index] == 1:
            #If the element has less than 2 or greater 3 alive neighbours it dies in the next iteration
            if 2 > sum or sum > 3:
                self.grid[index] = 0

        #If the state is dead
        else:
            #If the dead state has three alive neighbours that it becomes alive
            if sum == 3:
                self.grid[index] = 1


import json

file = open("fr.json")
france_cities = json.load(file)
idf_major_cities = []

PARIS_AREA_NAME = "Île-de-France"
LARGE_POPULATION_NUMBER = 50000

for index, city in enumerate(france_cities):
    if (
        city["admin_name"] == PARIS_AREA_NAME
        and int(city["population"]) >= LARGE_POPULATION_NUMBER
    ):
        major_idf_city = {
            "City_index": index,
            "Latitude": city["lat"],
            "Longitude": city["lng"],
            "City": city["city"],
        }
        idf_major_cities.append(major_idf_city)

print(f"len of major cities in idf: {len(idf_major_cities)}")

with open("idf_major_cities.json", "w") as f:
    json.dump(idf_major_cities, f, ensure_ascii=False)


# ========================================================================
# This class performs the global analysis with regard the Overap between
# Motifs (SLims) and the region of Iupred or ANCHOR or DisoRDPbind
# =======================================================================

from core.util.file.FileWriter import FileWriter
from core.util import Constants
from core.util.log.Logger import Logger
from core.util.tools.OverlapMotifsRegions import OverlapMotifsRegions
import math

class GlobalOverlapRegionAnalysis():

    #
    # overlap_file_write
    # -------------------
    #
    # This method allows to perform the OverlapMotifsRegions for a list of proteins
    # Argument:
    #    - table_tool: containing the slim tool region overlaps and the slim domain overlaps [the table_tool[0] have not to be empty!!!]
    #    - protein: protein analyzed
    #    - toolname: type of toolname
    #    - output folder: folder to save the output data
    # Returns:
    #    - file_slim_region: containing the table of overlap between the slim region and iupred region
    #    - file_domain: containing the table of overlap between the slim region and iupred region
    #                    only if the slim fall into a domain
    @staticmethod
    def overlap_file_write(table_tool,output_folder,protein,toolname):

        dict_filename = Constants.DICT_FILE_OUTPUT

        filename_slim_region =  dict_filename[toolname][0] + protein + Constants.EXTENSION_TXT_FILE
        filename_doamin_region = dict_filename[toolname][1] + protein + Constants.EXTENSION_TXT_FILE
        filepath_slim = output_folder + filename_slim_region
        filepath_domain = output_folder + filename_doamin_region

        title_slim_region = [[Constants.SLIM_NAME,Constants.SLIM_REGION_COUNTER,Constants.START_SLIM_REGION,Constants.END_SLIM_REGION,Constants.TOOLNAME,
                              Constants.TOOLNAME_REGION_COUNTER,Constants.START_TOOL_REGION, Constants.END_TOOL_REGION,
                              Constants.OVERLAP_OUTCOME, Constants.OVERLAP_LENGTH]]

        table_slim_region = title_slim_region + table_tool[0]
        table_domain_region = table_tool[1]

        if table_tool[0] != []:
            FileWriter.write_table(filepath_slim, table_slim_region)
        else:
            Logger.get_instance().debug("  The slim overlap file " + filename_slim_region+ " has not been written")
        if table_tool[1] != []:
            FileWriter.write_table(filepath_domain, table_domain_region)
        else:
            Logger.get_instance().debug("  The domain overlap file " + filename_doamin_region+ " has not been written\n\n")

    #
    # iupred_overlap_analysis
    # -------------------
    #
    # This method allows to perform the OverlapMotifsRegions for a list of proteins
    # Argument:
    #    - protein_list: list of protein
    #    - iupred folder: directory containing the tool prediction
    #    - motif_folder: directory containing the SLiM Found with ANCHOR tool
    #    - output folder: folder to save the output data
    #    - threshold: threshold to select the iupred region
    # Returns:
    #    the following file for all proteins in protin_list
    #    - file_slim_region: containing the table of overlap between the slim region and iupred region
    #    - file_domain: containing the table of overlap between the slim region and iupred region
    #                    only if the slim fall into a domain
    @staticmethod
    def iupred_overlap_analysis(protein_list,iupred_folder, output_folder, threshold, motif_folder,domain_region_file):

        toolname = Constants.IUPRED_TOOL

        for num, protein in enumerate(protein_list):
            prot_counter = num +1
            result = math.fmod(prot_counter,100)
            if result == 0.0:
                Logger.get_instance().info('        ' +str(prot_counter)+ "  Protein analyzed: " + protein)
            # This line takes the iupred region of a protein
            iupred_region =  OverlapMotifsRegions.iupred_region(iupred_folder,threshold, protein )
            # This line calls the method in order to check the overlap
            table_iupred = OverlapMotifsRegions.overlap_analysis(protein,iupred_region,toolname, motif_folder,domain_region_file)
            # table Writing
            GlobalOverlapRegionAnalysis.overlap_file_write(table_iupred, output_folder, protein, toolname)

    #
    # anchor_overlap_analysis
    # -------------------
    #
    # This method allows to perform the OverlapMotifsRegions for a list of proteins
    # Argument:
    #    - protein_list: list of protein
    #    - anchor folder: directory containing the tool prediction
    #    - motif_folder: directory containing the SLiM Found with ANCHOR tool
    #    - output folder: folder to save the output data
    # Returns:
    #    the following file for all proteins in protin_list
    #    - file_slim_region: containing the table of overlap between the slim region and iupred region
    #    - file_domain: containing the table of overlap between the slim region and iupred region
    #                    only if the slim fall into a domain
    @staticmethod
    def anchor_overlap_analysis(protein_list,anchor_folder,output_folder,motif_folder,domain_region_file):

        toolname = Constants.ANCHOR_TOOL

        for num, protein in enumerate(protein_list):
            prot_counter = num +1
            result = math.fmod(prot_counter,100)
            if result == 0.0:
                Logger.get_instance().info('        ' + str(prot_counter)+ "th  Protein analyzed: " + protein)
            # This line takes the iupred region of a protein
            anchor_region = OverlapMotifsRegions.anchor_region(anchor_folder, protein )
            table_anchor = OverlapMotifsRegions.overlap_analysis(protein,anchor_region,toolname, motif_folder,domain_region_file )
            # table Writing
            GlobalOverlapRegionAnalysis.overlap_file_write(table_anchor, output_folder, protein, toolname)

    #
    # disordp_overlap_analysis
    # -------------------
    #
    # This method allows to perform the Overlapanalysis for a list of proteins
    # Argument:
    #    - protein_list: list of protein
    #    - disordp folder: directory containing the tool prediction file
    #    - filename: file containing the disordpb prediction
    #    - motif_folder: directory containing the SLiM Found with ANCHOR tool
    #    - output folder: folder to save the output data
    # Returns:
    #    the following file for all proteins in protin_list
    #    - file_slim_region: containing the table of overlap between the slim region and iupred region
    #    - file_domain: containing the table of overlap between the slim region and iupred region
    #                    only if the slim fall into a domain
    @staticmethod
    def disordp_overlap_analysis(protein_list, disordp_folder, filename, motif_folder,domain_region_file,output_folder):

        toolname = Constants.DISORDP_TOOL

        for num, protein in enumerate(protein_list):
            prot_counter = num +1
            result = math.fmod(prot_counter,100)
            if result == 0.0:
                Logger.get_instance().info('        ' +str(prot_counter)+ " Protein analyzed: " + protein)
            # This line takes the iupred region of a protein
            disordp_region = OverlapMotifsRegions.disordp_region(disordp_folder, filename, protein )
            table_disordp = OverlapMotifsRegions.overlap_analysis(protein,disordp_region, toolname, motif_folder,domain_region_file )
            # table Writing
            GlobalOverlapRegionAnalysis.overlap_file_write(table_disordp, output_folder, protein, toolname)


from PIL import Image
import numpy as np
import os
j=np.arange(1,325)
FindPath=r"F:/Desktop/train7"
filenames=os.listdir(FindPath)
imlist=[]
for name in filenames:
    filePath=os.path.join(FindPath,name)
    imlist.append(filePath)
for i in j:
    outfile7 = r"F:/Desktop/train1/" +str(i+324)+'.jpg'
    im1 = Image.open(imlist[i - 1])
    im2=im1.convert('L')
    im3=im2.resize((512,512))
    im3.save(outfile7)


from __future__ import division

import sys
sys.path.insert(0, '../../')

import numpy as np
import pandas as pd
import math
import torch
from grammar_models import EquationGrammarModel
from Network import GrammarVariationalAutoEncoder
import argparse
import os

def get_arguments():
    parser = argparse.ArgumentParser(
        description='Molecular autoencoder network')
    parser.add_argument('--model_name', type=str, metavar='N', default="eq2_grammar_dataset_L56_E50_val.hdf5")
    return parser.parse_args()

def func(x):
    return 1/3 + x + math.sin(x * x)

if __name__ == '__main__':

    # We load the data
    fname = '../../data/equation2_15_dataset.txt'

    args = get_arguments()
    model_path = os.path.join('..','..','final_models', args.model_name)

    x = np.linspace(-10,10,1000)
    y = np.array([func(x_) for x_ in x])

    with open(fname) as f:
        eqs = f.readlines()

    for i in range(len(eqs)):
        eqs[i] = eqs[i].strip()
        eqs[i] = eqs[i].replace(' ', '')

    final_model = GrammarVariationalAutoEncoder()
    final_model.load_state_dict(torch.load(model_path))

    grammar_model = EquationGrammarModel(final_model)
    WORST = 1000
    MAX = 100000

    targets = []
    scores = []
    equations = []
    for i in range(len(eqs[:MAX])):
        try:
            score = np.log(1+np.mean(np.minimum((np.array(eval(eqs[i])) - y)**2, WORST)))
        except:
            score = np.log(1+WORST)
        if not np.isfinite(score):
            score = np.log(1+WORST)
        print(i, eqs[i], score)
        scores.append(score)
        equations.append(eqs[i])
        targets.append(score)

    data_tuples = list(zip(equations, scores))
    df = pd.DataFrame(data_tuples, columns=['equations', 'scores'])
    df = df.sort_values(by=['scores'])

    targets = np.array(targets)

    np.savetxt('targets_eq.txt', targets)
    np.savetxt('x_eq.txt', x)
    np.savetxt('true_y_eq.txt', y)

    batch_size = 5000
    num_batches = int(np.ceil(len(targets) / batch_size))
    for i in range(num_batches):
        if i == 0:
            latent_points = grammar_model.encode(eqs[:batch_size])
            latent_points = latent_points.cpu().detach().numpy()
        else:
            new_latents = grammar_model.encode(eqs[i*batch_size:(i+1)*batch_size])
            latent_points = np.concatenate((latent_points, new_latents.cpu().detach().numpy()), axis=0)
        print(i)

    # We store the results
    np.savetxt('latent_features_eq.txt', latent_points)
    print('Done!')


import torch
import torch.nn as nn
import torch.nn.functional as F

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(128, 256, 1)
        self.conv2 = nn.Conv2d(128, 256, 3, padding=1)
        self.conv3 = nn.Conv2d(128, 256, 5, padding=2)
        self.max_pooling = nn.MaxPool2d(3, stride=1, padding=1)

    def forward(self, x):
        out1 = F.relu(self.conv1(x))  # 1,256, 14, 14
        out2 = F.relu(self.conv2(x)) # 1, 256, 14, 14
        out3 = F.relu(self.conv3(x))  # 1, 256, 14, 14
        out_max = self.max_pooling(x)  # 1,128, 14, 14

        output = torch.cat((out1, out2, out3, out_max), dim=1)
        return output

def function(x, y):
    x, y = torch.tensor(x, requires_grad=True), torch.tensor(y, requires_grad=True)
    loss = 2 * x * y + x ** 2
    loss.backward()
    return x.grad

print (function(3, 1))


from flask import Flask, render_template, request, jsonify, send_from_directory
import logging
import time
import sys
import os
from llama_index import (GPTVectorStoreIndex, LLMPredictor,
                         SimpleDirectoryReader, ServiceContext,
                         StorageContext, load_index_from_storage)
from llama_index import QuestionAnswerPrompt, RefinePrompt
from llama_index.query_engine.transform_query_engine import TransformQueryEngine
from langchain.chat_models import ChatOpenAI
import backoff
import openai
import socket
from llama_index.indices.postprocessor import MetadataReplacementPostProcessor

# Constants
MODEL = "gpt-4"

# Get API key from environment variable or prompt user
api_key = os.getenv("OPENAI_API_KEY")
if not api_key:
    api_key = input("Enter OpenAI API key: ")

# Get folder path from command line argument or prompt user
folder_path = sys.argv[1] if len(sys.argv) > 1 else input("Enter folder path: ")

# Set OpenAI API key
os.environ["OPENAI_API_KEY"] = api_key
openai.api_key = api_key

# Setup logging
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))

# Initialize LLM Predictor and ServiceContext
llm_predictor = LLMPredictor(llm=ChatOpenAI(temperature=0.4, model_name=MODEL))
service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor)

# Load the index from storage
storage_context = StorageContext.from_defaults(persist_dir=folder_path)
index = load_index_from_storage(storage_context)

# Define your ENGLISCH_QA_PROMPT and ENGLISCH_REFINE_PROMPT templates here
# ...

# Time function
def timeit(func):
    def wrapper(*args, **kwargs):
        start_time = time.time()
        result = func(*args, **kwargs)
        end_time = time.time()
        elapsed_time = end_time - start_time
        return result, elapsed_time
    return wrapper

# Function to handle retries with exponential backoff
@backoff.on_exception(backoff.expo,
                      Exception,
                      max_tries=10,
                      giveup=lambda e: False,  # Don't give up on any exception
                      base=10)
@timeit
def query_index(query_engine, input_text):
    return query_engine.query(input_text)

# Create a QueryEngine
query_engine = index.as_query_engine(
    service_context=service_context,
    similarity_top_k=15,
    response_mode="compact",
    text_qa_template=ENGLISCH_QA_PROMPT,
    refine_template=ENGLISCH_REFINE_PROMPT,
    node_postprocessors=[
        MetadataReplacementPostProcessor(target_metadata_key="window")
    ],
)

def handle_user_query(casedesc):
    input_text = casedesc
    response, response_time_query = query_index(query_engine, input_text)
    output_accGPT = response.response

    filenames_pages = dict()
    for doc_id, metadata in response.metadata.items():
        filename_label = metadata.get("file_name")
        page_label = metadata.get("page_label")
        if filename_label is not None:
            filename_label = os.path.basename(filename_label)
            if filename_label not in filenames_pages:
                filenames_pages[filename_label] = set()
            if page_label is not None:
                filenames_pages[filename_label].add(f"{page_label}")
    summary = []
    for filename, pages in filenames_pages.items():
        sorted_pages = sorted(pages, key=lambda x: (not x.isdigit(), int(x) if x.isdigit() else x))

        summary.append(f"{filename} Page: ({', '.join(sorted_pages)})" if pages else filename)
    return output_accGPT, ", ".join(summary), response_time_query

app = Flask(__name__)

@app.route('/')
def home():
    return render_template('index.html')

@app.route('/query', methods=['POST'])
def query():
    try:
        casedesc = request.form['casedesc']
        result, filenames, response_time_query = handle_user_query(casedesc)
        return jsonify({
            'result': result,
            'filenames': filenames,
            'time': str(response_time_query)
        })
    except Exception as e:
        logging.error("Error while processing the query: ", exc_info=True)
        return jsonify({'error': str(e)}), 500

@app.route('/pdfs/<filename>')
def serve_pdf(filename):
    response = send_from_directory(folder_path, filename, as_attachment=False, mimetype='application/pdf')
    response.headers['Content-Disposition'] = f'inline; filename={filename}'
    return response

def find_unused_port(start_port=8000):
    port = start_port
    while True:
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
            try:
                s.bind(('', port))
                s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                return port
            except OSError:
                port += 1

if __name__ == "__main__":
    port = find_unused_port()
    app.run(host='0.0.0.0', port=port, use_reloader=False)


import matplotlib
matplotlib.use('Agg')

import argparse
import glob
import os
import shutil
import sys
import json
from tqdm import tqdm
import tensorflow as tf

import numpy as np
from gym import spaces

from deepagent.agents.common import AbstractAgentTrainer
from deepagent.agents.random_agent import DebugAgentTester, RacerDijsktraTester
from deepagent.common.auto_dir import get_numbered_directories, get_next_numbered_directory
from deepagent.envs.environment_wrappers import TrainingEnvironment
from deepagent.envs.factory import EnvironmentFactory
from deepagent.envs.spaces import DeepAgentSpace
from deepagent.envs.env_constants import EnvType
from deepagent.experiments.params import params
from deepagent.experiments.params.params import ExperimentType, ImitationLearning
from deepagent.experiments.params.utils import load_params
from deepagent.recording.recording import record_atari, MultiPlayerDemoReader
from deepagent.agents.memory import BCMemory

import logging
logging.basicConfig(level=logging.INFO)
from deepagent.experiments.params import utils as param_utils

def setup_environment(env_file_str, env_type, params_name, random_seed=0, arena_nbr=0):
    seed = 123 + random_seed
    np.random.seed(seed)

    env1 = EnvironmentFactory.get_env(env_file_str, env_type, seed, params_name, random_seed, arena_nbr)
    env_wrapper = TrainingEnvironment(env1)

    return env_wrapper

def setup_multi_agent(env_wrapper):
    agent_factory = params.AgentParams.agent_factory
    multi_agent_training_class = agent_factory.multi_agent_training_class()
    return multi_agent_training_class(environment_wrapper=env_wrapper, agent_factory=agent_factory)

import os
from typing import List, Tuple
import numpy as np

INITIAL_POS = 33
SUPPORTED_DEMONSTRATION_VERSIONS = frozenset([0, 1])

def get_demo_files(path: str) -> List[str]:
    """
    Retrieves the demonstration file(s) from a path.
    :param path: Path of demonstration file or directory.
    :return: List of demonstration files

    Raises errors if |path| is invalid.
    """
    if os.path.isfile(path):
        if not path.endswith(".demo"):
            raise ValueError("The path provided is not a '.demo' file.")
        return [path]
    elif os.path.isdir(path):
        paths = [
            os.path.join(path, name)
            for name in os.listdir(path)
            if name.endswith(".demo")
        ]
        if not paths:
            raise ValueError("There are no '.demo' files in the provided directory.")
        return paths
    else:
        raise FileNotFoundError(
            f"The demonstration file or directory {path} does not exist."
        )

def get_state_shape(file_path: str) -> Tuple[int]:
    state = None
    with open(file_path) as f:
        for line_idx, line in enumerate(tqdm(f)):
            if 'Prev Obs: [ ' not in line:
                continue
            items = line.strip().split(' | ')
            state = items[2][12:-1]
            state = np.fromstring(state, dtype=np.float32, sep=' ')
            break
    if state is None:
        raise Exception('Could not find state shape from recording file')
    return state.shape

def create_pretraining_generator(recording_files: List[str], action_shape: Tuple[int]):
    def load_data_generator():
        num_episodes = 0
        for recording_file in recording_files:
            with open(recording_file) as f:
                episode_idx = 0
                for line_idx, line in enumerate(f):
                    if 'Prev Obs: [ ' not in line:
                        continue
                    items = line.strip().split(' | ')
                    state = items[2][12:-1]
                    state = np.fromstring(state, dtype=np.float32, sep=' ')
                    action = items[1].split()[-1]
                    action = np.eye(action_shape[0])[int(action)]
                    episode_idx = int(line.strip().split('UnitId: (0, ')[1].split(')')[0])
                    yield state, action
                num_episodes += episode_idx
        print(f'Loaded {num_episodes} episodes from recording files.')
    return load_data_generator

def get_pretraining_length(recording_files):
    steps = 0
    for recording_file in recording_files:
        with open(recording_file) as f:
            for line_idx, line in enumerate(tqdm(f)):
                if 'Prev Obs: [ ' in line:
                    steps += 1
    return steps

def pre_train():
    for brain_name, pre_train_args in params.ImitationLearning.pre_training_args.items():
        # init policies
        if pre_train_args.recording_files is not None:
            # Loads pretraining data from recording files
            if pre_train_args.action_shape_override is None:
                raise Exception(
                    'Action shape override was found to be None. Action shape override must be supplied in '
                    'ImitationLearning.PreTrainArgs if recording files are used.'
                )
            action_shape = pre_train_args.action_shape_override
            state_shape = get_state_shape(pre_train_args.recording_files[0])

            # TF dataset is used so training can be done even if the full data doesn't fit in memory
            print(f'Loading pretraining data from recording files: {", ".join(pre_train_args.recording_files)}')
            dataset = tf.data.Dataset.from_generator(create_pretraining_generator(pre_train_args.recording_files, action_shape),
                                                     output_signature=(
                                                      tf.TensorSpec(shape=state_shape, dtype=tf.float32),
                                                      tf.TensorSpec(shape=action_shape, dtype=tf.float32,))
                                                     )
            dataset_len = get_pretraining_length(pre_train_args.recording_files)
            print(f'{dataset_len} examples found in pretraining data')
            dataset = dataset.apply(tf.data.experimental.assert_cardinality(dataset_len))
            vec_space = spaces.Box(-1.0, 1.0, state_shape)
            state_space = DeepAgentSpace(vector_space=vec_space, agent_name=brain_name)
            vector_space = spaces.Box(0.0, 1.0, shape=action_shape, dtype=np.float32)
            action_space = DeepAgentSpace(vector_space=vector_space, agent_name=brain_name)
        else:
            reader = MultiPlayerDemoReader(pre_train_args.player_runs)
            meta = reader.load_meta()
            state_shape = meta.state_shape
            action_shape = meta.action_shape
            brain_name = meta.student_brain

            print(state_shape)
            print(action_shape)

            if state_shape is None:
                state_shape = pre_train_args.state_shape_override

            if action_shape is None:
                action_shape = pre_train_args.action_shape_override

            if params.EnvironmentParams.env_type.is_atari():
                frame_stack = 4
            elif params.EnvironmentParams.env_type.is_unity_ml():
                frame_stack = params.UnityParams.unity_frame_stack
            else:
                raise ValueError(f"Don't know how to pre_train for env_type: {params.EnvironmentParams.env_type}")

            num_actions = action_shape[0]

            state_shape = (state_shape[0], state_shape[1], state_shape[2] * 4)

            # TODO: I don't think the low and high actually matter here, but space management needs to be cleaned up and made consistent in general
            state_space = DeepAgentSpace(image_spaces=spaces.Box(low=-1.0, high=1.0, shape=state_shape))

            if params.EnvironmentParams.env_type.is_atari():
                action_space = DeepAgentSpace(vector_space=spaces.Discrete(n=action_shape[0]))
            elif params.EnvironmentParams.env_type.is_unity_ml():
                action_space = DeepAgentSpace(vector_space=spaces.Box(low=-1.0, high=1.0, shape=action_shape))
            else:
                raise ValueError(f"Don't know how to pre_train for env_type: {params.EnvironmentParams.env_type}")

            state_shape = [x.shape for x in state_space.image_spaces]

        agent_factory = params.AgentParams.agent_factory

        policy = agent_factory.create_agent(agent_type=brain_name, state_space=state_space, action_space=action_space)

        policy.print_summary()
        # import tensorflow as tf
        #policy.set_tensorboard_writer(tf.summary.create_file_writer(params.ModuleParams.tensorboard_dir))
        from deepagent.loggers.logging import get_tensorboard_writer
        policy.set_tensorboard_writer(get_tensorboard_writer())
        print('Loading pre-training data ...')
        if pre_train_args.recording_files is not None:
            memory = BCMemory()
            memory.dataset = dataset
        else:
            memory = reader.bc_memory(frame_stack=frame_stack, num_actions=num_actions,
                                      use_terminals_as_start_state=params.EnvironmentParams.env_type.is_atari(),
                                      tile_first=params.EnvironmentParams.env_type.is_unity_ml())

            action_counts, action_distribution = memory.action_distribution()
            print(f'\nAction Counts: {action_counts}, Action Distributions: {action_distribution}')

        # np.set_printoptions(threshold=500, precision=4, linewidth=150)
        # mean, std = memory.state_mean_and_std()
        # print(f'State Index Mean:\n{mean}')
        # print(f'\nState Index Standard Deviations:\n{std}')
        # np.set_printoptions(threshold=1000, precision=8, linewidth=75) #defaults
        # print(mean.shape)
        # print(std.shape)

        policy.pre_train(memory)

def train(multi_agent: AbstractAgentTrainer):
    multi_agent.train(steps=params.TrainingParams.steps, gamma=params.TrainingParams.gamma,
                      lam=params.TrainingParams.lam)
    #multi_agent.write_report_summary()

def test(multi_agent: AbstractAgentTrainer, test_suffix: str = ''):
    multi_agent.test(num_episodes=params.TestingParams.num_episodes, gamma=params.TestingParams.gamma,
                            testing_suffix=test_suffix)
    #multi_agent.write_report_summary()

def population_test(multi_agent: AbstractAgentTrainer):
    return multi_agent.population_test()

def copy_weights(from_dir, to_dir):
    print('Copying previous weights from {} to {}'.format(from_dir, to_dir))
    files = glob.iglob(os.path.join(from_dir, '*.h5'))
    for file in files:
        shutil.copy2(file, to_dir)

def copy_reward_masks(from_dir, to_dir):
    print('Copying previous reward masks from {} to {}'.format(from_dir, to_dir))
    files = glob.iglob(os.path.join(from_dir, '*.pkl'))
    for file in files:
        shutil.copy2(file, to_dir)

def set_weights_dir(experiment_type, prev_weights, weights, weights_num):
    if experiment_type == ExperimentType.train or experiment_type == ExperimentType.pre_train:
        params.ModuleParams.weights_dir = get_next_numbered_directory(params.ModuleParams.params_dir)
        if prev_weights is not None:
            prev_weights = os.path.join(params.ModuleParams.params_dir, prev_weights)
            copy_weights(from_dir=prev_weights, to_dir=params.ModuleParams.weights_dir)
            copy_reward_masks(from_dir=prev_weights, to_dir=params.ModuleParams.weights_dir)
    elif experiment_type == ExperimentType.test or experiment_type == ExperimentType.record or experiment_type == ExperimentType.population_test:
        if params.AgentParams.agent_factory.multi_agent_training_class() == DebugAgentTester or params.AgentParams.agent_factory.multi_agent_training_class() == RacerDijsktraTester:
            weights = params.ModuleParams.params_dir
        elif weights == "latest":
            weights = str(sorted(get_numbered_directories(params.ModuleParams.params_dir))[-1])
        elif weights is None or weights == '' or weights not in os.listdir(params.ModuleParams.params_dir):
            raise ValueError(
                'Weights number must be specified for testing or recording if not using RandomAgentTester for the AgentTrainer.'
                '\n\tTraining directories are created in ascending order and found in subdirectories of the parameters folders.'
                '\n\tAvailable training folders for {} are {}'.format(params.ModuleParams.name, sorted(
                    get_numbered_directories(params.ModuleParams.params_dir))))

        params.ModuleParams.weights_dir = os.path.join(params.ModuleParams.params_dir, weights)
        params.ModuleParams.weights_num = weights_num

def set_tensorboard_dir():
    tensorboard_top_dir = os.path.join(params.ModuleParams.params_dir, 'tensorboard')
    if not os.path.exists(tensorboard_top_dir):
        os.makedirs(tensorboard_top_dir)
    params.ModuleParams.tensorboard_dir = get_next_numbered_directory(tensorboard_top_dir)

parser = argparse.ArgumentParser()
parser.add_argument('--params', '-p', help="The params module to use: ex: 'params1', 'params2', etc.")
parser.add_argument('--experiment_type', '-et', type=ExperimentType, choices=list(ExperimentType),
                    help="The type of experiment to run:\n\tperform supervised imitation learning\n\tperform "
                         "reinforcement learning training\n\ttest the trained network", required=True)
parser.add_argument('--weights', '-w', type=str,
                    help='The training weights folder to test with. Training directories are numbered in ascending '
                         'order and found in subdirectories of the parameters folders.')
parser.add_argument('--weights_num', '-wn', type=str,
                    help='# The step number at which the weights were saved (or None to use the final weights)'
                         'For example: 400000')
parser.add_argument('--prev_weights', '-pw', type=str,
                    help='A weights folder to use at the start of training. Training will still occur in the next '
                         'highest numbered subdirectory of the parameters folders, but the weights can be loaded '
                         'from any of the previous numbered directories.')
parser.add_argument('--test_suffix', '-ts', type=str, default='',
                    help='# This string is added to the test log filename (default="")')
parser.add_argument('-u', action='store_true', help='This argument is just here because pycharm is dumb at '
                                                    'executing modules on remote interpreters. The workaround means '
                                                    'that it adds a "-u" field in the wrong place.')
parser.add_argument('--env_args', '-ea', nargs='*')
parser.add_argument('--param_overrides', '--po', '-po', type=str,
                    help='Parameters to override in the params module in JSON format. '
                         'Ex: --param_overrides="{\'UnityParams.unity_port\': 5006, \'TrainingParams.batch_size\': 8}"')

def jsonstr_to_kwarg(jsonStr):
    param_overrides = jsonStr.replace("\'", "\"")
    param_overrides = json.loads(param_overrides)
    param_kwargs = {}

    # parse params into a dict (class name) of kwargs (param_name:param_val)
    for param_key, param_value in param_overrides.items():
        param_class, param_var = param_key.split('.')
        if param_class not in param_kwargs.keys():
            param_kwargs[param_class] = {}  # first time seeing override for this class, create default/empty kwargs
        param_kwargs[param_class][param_var] = param_value

    return param_kwargs

def main(argv=sys.argv[1:]):
    print('CLEARED FOR RELEASE')
    print('''

                                ▓▀^^"""  """""▀▒
                                ╬              ╬
                               ╒▓              ▒∩
                              ,▓▄              ╓╬⌐
                                ╬              ╬⌐
                                ╬              ╬
                                ▀▒ÆÆÆÆÆ  ╦╦╦╦Æ▒▀

   #▒▀▀▀▒       ,,                       ╚▀▀▀▓▓▀▀▀              ,
  ▐╬        ,▒▀^ `╙▒╦    @▀▒⌐    ▒▌^^^╙▒▄    ╬Γ    ║╬^^^^^` ╓▒▀^ ^▀▒ j╬     ▒Γ
   "▀▒▒Æ╗   ╬⌐      ▒∩  @▀  ▒µ   ▒▌    ╫▀    ╬Γ    ║╬,,,,, ]╬        j╬,,,,,▒Γ
         ╬∩ ╬∩      ▓∩ @╬╗╗╗╣╬⌐  ▒▌╙╙▀▓      ╬Γ    ║╬      └╬        j╬     ▒Γ
   ▒Æ╗╖╖▒▀   ▀▒╖╓╓#▓` @▀      ▒µ ▒▌   ╙▓╕    ╬Γ    ║▓╖╖╖╖╖⌐ ^▀▄╖╓╓#▀ j╬     ▒Γ

 _____     ______     ______     ______   ______     ______     ______     __   __     ______
/\  __-.  /\  ___\   /\  ___\   /\  == \ /\  __ \   /\  ___\   /\  ___\   /\ "-.\ \   /\__  _\
\ \ \/\ \ \ \  __\   \ \  __\   \ \  _-/ \ \  __ \  \ \ \__ \  \ \  __\   \ \ \-.  \  \/_/\ \/
 \ \____-  \ \_____\  \ \_____\  \ \_\    \ \_\ \_\  \ \_____\  \ \_____\  \ \_\\\\"\_\    \ \_\
  \/____/   \/_____/   \/_____/   \/_/     \/_/\/_/   \/_____/   \/_____/   \/_/ \/_/     \/_/

''')
    args = parser.parse_args(argv)
    print('argv=',argv)
    params_name = args.params
    experiment_type = args.experiment_type
    params.RuntimeParams.experiment_type = experiment_type
    prev_weights = args.prev_weights
    weights = args.weights
    weights_num = args.weights_num
    test_suffix = args.test_suffix
    load_params(name=params_name)
    if args.env_args:
        params.AgentParams.agent_kwargs = dict([pair.split('=') for pair in args.env_args])
    else:
        params.AgentParams.agent_kwargs = {}

    # Override params if they are in the command line args
    if args.param_overrides:
        param_kwargs = jsonstr_to_kwarg(args.param_overrides)
        params.param_override(param_kwargs)

    invalid_keys = ['']
    for k in invalid_keys:
        if k in params.AgentParams.agent_kwargs:
            del params.AgentParams.agent_kwargs[k]
    if len(params.AgentParams.agent_kwargs) == 0:
        params.AgentParams.agent_kwargs = None

    run_experiment(experiment_type, prev_weights, weights, weights_num, params_name, test_suffix, args=args)

def run_experiment(experiment_type, prev_weights, weights, weights_num, params_name, test_suffix='', tensorboard_logging=True, args=None):
    print(f'setting weights->{weights}')
    set_weights_dir(experiment_type=experiment_type, prev_weights=prev_weights, weights=weights, weights_num=weights_num)

    param_utils.write_commandline_params(args.__str__(), "commandline_full")
    if args.param_overrides:
        param_kwargs: dict = jsonstr_to_kwarg(args.param_overrides)
        param_utils.write_commandline_params(param_kwargs.__str__(), "commandline_overrides")
    param_utils.write_params_state()

    if tensorboard_logging:
        set_tensorboard_dir()
    else:
        params.ModuleParams.tensorboard_dir = None

    print('Experiment type:{}'.format(experiment_type))
    print("Loaded params:{}".format(params.ModuleParams.params_dir))
    print('Weights dir set to {}'.format(params.ModuleParams.weights_dir))
    if prev_weights is not None:
        params.ModuleParams.prev_weights = prev_weights
        print('Using prev weights: {}'.format(prev_weights))

    env_type = params.EnvironmentParams.env_type
    runtime = params.RuntimeParams

    # Pre-Train the network from previously recorded data
    if runtime.is_pre_training():
        pre_train()
        return

    # Record an atari environment
    if runtime.is_recording() and env_type.is_atari():
        # This is always the recording setup to use for atari, setting the teacher_brain and student_brain
        # just affects the data in the meta data files for the atari recordings.
        record_atari(player_id=params.ImitationLearning.atari_player_id)
        return

    # Setup environments for training, testing, or unity recording.
    # Population trainers setup and tear-down environments by themselves
    agent_factory = params.AgentParams.agent_factory
    multi_agent_training_class = agent_factory.multi_agent_training_class()
    if multi_agent_training_class.instantiates_envs():
        multi_agent = multi_agent_training_class()
    else:
        env_wrapper = setup_environment(params.EnvironmentParams.environment, params.EnvironmentParams.env_type, params_name)
        multi_agent = setup_multi_agent(env_wrapper)

    # Train atari or unity
    if runtime.is_training():
        train(multi_agent)
    # Test atari or unity
    elif runtime.is_testing():
        test(multi_agent, test_suffix=test_suffix)

if __name__ == '__main__':
    main()


import asyncio
import datetime

async def gerar_dados(quantidade: int, dados: asyncio.Queue) -> None:
    print(f'Aguarde a geração de {quantidade} dados...')
    for idx in range(1, quantidade + 1):
        item: int = idx * idx
        await dados.put((item, datetime.datetime.now()))
        await asyncio.sleep(0.001)
    print(f'{quantidade} dados gerados com sucesso!')

async def processar_dados(quantidade: int, dados: asyncio.Queue) -> None:
    print(f'Aguarde o processamento de {quantidade} dados...')
    processados = 0
    while processados < quantidade:
        await dados.get()
        processados += 1
        await asyncio.sleep(0.001)
    print(f'Foram processados {processados} itens')

if __name__ == '__main__':
    total: int = 5_000
    dados = asyncio.Queue()
    el = asyncio.new_event_loop()
    asyncio.set_event_loop(el)

    print(f'Computando {total * 2:.2f} dados.')

    el.run_until_complete(gerar_dados(total, dados))
    el.run_until_complete(gerar_dados(total, dados))
    el.run_until_complete(processar_dados(total * 2, dados))

    el.close()


import numpy as np
import pandas as pd
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import Dropout
from pandas import DataFrame
from pandas import concat
from itertools import chain
from sklearn.metrics import mean_squared_error
from sklearn.metrics import accuracy_score
from sklearn import metrics
import matplotlib.pyplot as plt
from keras.callbacks import ModelCheckpoint
from keras.models import load_model
from keras.utils import to_categorical
from data_operate import getDataSet
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import cohen_kappa_score
from sklearn.model_selection  import train_test_split

def getdata(df, chunk_size_x,chunk_size_y):
    list = df['Unnamed: 0'].tolist()
    data_set = df.iloc[:, 15:].values
    data_set = data_set.astype('float64')
    #归一化
    scaler = StandardScaler()
    # scaler = MinMaxScaler()

    train_data_set = np.array(data_set)
    #得到数据集
    reframed_train_data_set = np.array(getDataSet(train_data_set, chunk_size_x, chunk_size_y,True,list).values)
    reframed_train_data_set = reframed_train_data_set[:,:16*chunk_size_x+1]
    # 数据集划分,选取前60%天的数据作为训练集,中间20%天作为验证集,其余的作为测试集
    train_days = int(len(reframed_train_data_set) * 0.6)
    valid_days = int(len(reframed_train_data_set) * 0.2)

    data = reframed_train_data_set[:,:-chunk_size_y]
    scaler.fit(data)
    data = scaler.transform(data)

    train_x = data[:train_days, :]
    valid_x = data[train_days:train_days + valid_days, :]
    test_x = data[train_days + valid_days:, :]

    data_y = reframed_train_data_set[:, -chunk_size_y:]
    label = np.zeros_like(data_y)
    label[data_y > 0] = 1
    label[data_y > 0.5] = 2
    label[data_y > 1.5] = 3
    label = to_categorical(label, num_classes=4)
    train_y = label[:train_days]
    valid_y = label[train_days:train_days + valid_days]
    test_y = label[train_days + valid_days:]

    # 将数据集重构为符合LSTM要求的数据格式,即 [样本，时间步，特征维度：16]
    train_x = train_x.reshape((train_x.shape[0], chunk_size_x, 16))
    valid_x = valid_x.reshape((valid_x.shape[0], chunk_size_x, 16))
    test_x = test_x.reshape((test_x.shape[0], chunk_size_x, 16))

    return train_x, train_y, valid_x, valid_y, test_x, test_y , scaler

def gettrain_rand(df, chunk_size_x,chunk_size_y):
    list = df['Unnamed: 0'].tolist()
    trainsize = int(df.shape[0]*0.9)
    data_set = df.iloc[:trainsize, 15:].values
    data_set = data_set.astype('float64')
    # 归一化
    scaler = StandardScaler()
    train_data_set = np.array(data_set)
    # 得到数据集
    reframed_train_data_set = np.array(getDataSet(train_data_set, chunk_size_x, chunk_size_y, True, list).values)
    reframed_train_data_set = reframed_train_data_set[:, :16*chunk_size_x + 1]
    data = reframed_train_data_set[:, :-chunk_size_y]
    scaler.fit(data)
    data = scaler.transform(data)
    data_y = reframed_train_data_set[:, -chunk_size_y:]
    label = np.zeros_like(data_y)
    label[data_y > 0] = 1
    label[data_y > 0.5] = 2
    label[data_y > 1.5] = 3
    label = to_categorical(label, num_classes=4)
    train_x,rest_x,train_y,rest_y = train_test_split(data,label,test_size = 0.2,random_state= 0)
    valid_x,test_x,valid_y,test_y = train_test_split(rest_x,rest_y,test_size = 0.5,random_state= 0)

    train_x = train_x.reshape((train_x.shape[0], chunk_size_x, 16))
    valid_x = valid_x.reshape((valid_x.shape[0], chunk_size_x, 16))
    test_x = test_x.reshape((test_x.shape[0], chunk_size_x, 16))

    return train_x,train_y,valid_x,valid_y,test_x,test_y

def lstm_model(train_X,chunk_size_y):
    model = Sequential()
    #第一层
    model.add(LSTM(64, return_sequences=True, activation='relu', input_shape=(train_X.shape[1], train_X.shape[2])))
    # 第二层
    model.add(LSTM(64, return_sequences=False))
    model.add(Dropout(0.5))
    # 第三层 因为是回归问题所以使用linear
    model.add(Dense(4 * chunk_size_y, activation='softmax'))
    model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer='adam')
    return model

def getdata_test(df, chunk_size_x,chunk_size_y):
    list = df['Unnamed: 0'].tolist()
    data_set = df.iloc[:, 15:].values
    data_set = data_set.astype('float64')
    #归一化
    scaler = StandardScaler()

    train_data_set = np.array(data_set)
    #得到数据集
    reframed_train_data_set = np.array(getDataSet(train_data_set, chunk_size_x, chunk_size_y,True,list).values)
    reframed_train_data_set = reframed_train_data_set[:,:16*chunk_size_x+1]

    data = reframed_train_data_set[:,:-chunk_size_y]
    scaler.fit(data)
    data = scaler.transform(data)
    test_x = data

    data_y = reframed_train_data_set[:, -chunk_size_y:]
    label = np.zeros_like(data_y)
    label[data_y > 0] = 1
    label[data_y > 0.5] = 2
    label[data_y > 1.5] = 3
    # label = to_categorical(label, num_classes=4)
    test_y = label
    # 将数据集重构为符合LSTM要求的数据格式,即 [样本，时间步，特征维度：16]
    test_x = test_x.reshape((test_x.shape[0], chunk_size_x, 16))
    return test_x, test_y , scaler

def train_vote(path):
    df = pd.read_csv(path + '.csv')
    chunk_size_x = 6
    chunk_size_y = 1
    input_epochs = 20
    input_batch_size = 100

    train_x_, label_y_, valid_x, valid_y, test_x, test_y = \
        gettrain_rand(df, chunk_size_x, chunk_size_y)
    for i in range(10):
        train_x, _,label_y,_ =  train_test_split(train_x_, label_y_,test_size = 0.4,random_state= None)
        model = lstm_model(train_x, chunk_size_y)
        model.fit(train_x, label_y, epochs=input_epochs, batch_size=input_batch_size,
                  validation_data=(valid_x, valid_y), verbose=2, shuffle=False)
        model.save(path + 'lstm' + str(i) + '.h5')
        # test_predict = model.predict(test_x)
        # pred_cls = np.argmax(test_predict, axis=1)
        # test_y_cl = np.argmax(test_y, axis=1)
        # kappa = cohen_kappa_score(test_y_cl, pred_cls)
        # print(kappa)
        # acc = accuracy_score(test_y_cl, pred_cls)
        # acc0 = accuracy_score(test_y_cl, np.zeros_like(test_y_cl))
        # print(acc, acc0)
        # recall = metrics.recall_score(test_y_cl, pred_cls, average='macro')
        # print('recall:%f' % recall)

def vote(path,pathtest):
    models = []
    for i in range(10):
        model = load_model(path + 'lstm' + str(i) + '.h5')
        models.append(model)

    df = pd.read_csv('./testset/' + pathtest + '.csv')
    chunk_size_x = 6
    chunk_size_y = 1
    test_x, test_y,scaler =  getdata_test(df, chunk_size_x, chunk_size_y)

    pred_cls = np.zeros((test_x.shape[0],10))
    for i in range(10):
        model = models[i]
        test_predict = model.predict(test_x)
        pred_cls[:,i] = np.argmax(test_predict, axis=1)
    pred_cl = np.zeros((test_x.shape[0],1))
    for i in range(test_x.shape[0]):
        counts = np.bincount(pred_cls[i,:].astype(np.int32))
        pred_cl[i] = np.argmax(counts)

    kappa = cohen_kappa_score(test_y, pred_cl)
    print('kappa:%f'%kappa)
    acc = accuracy_score(test_y, pred_cl)
    print('accuracy:%f'%acc)
    recall = metrics.recall_score(test_y, pred_cl, average='macro')
    print('recall:%f' % recall)

if __name__ == '__main__':
    trainSets = ['station178_filled','station303_filled','station304_filled','station305_filled','station306_filled']
    testSets = ['station1','station2','station3','station4','station5']
    for i in range(5):
        print(trainSets[i]+' start training')
        train_vote(trainSets[i])
        print('Bagging trained')

    # for i in range(5):
    #     print('\n' + trainSets[i]+' start testing')
    #     vote(trainSets[i],testSets[i])
    #     print('Bagging result:')


class Solution:
    @staticmethod
    def solve(s):
        n = len(s)
        dp = [0] * n
        dp[0] = 1
        start = 0
        dict = {s[0]: 0}
        for i in range(1,n):
            index = dict.get(s[i])
            if index is None:
                dp[i] = max(dp[i-1], i - start + 1)
            else:
                if index < start:
                    dp[i] = dp[i - 1]
                else:
                    start = index + 1
                    dp[i] = max(dp[i-1], i - start + 1)
            dict[s[i]] = i
        return dp[n-1]

print(Solution.solve('arabcacfr'))
print(Solution.solve('abcadeafgh'))


# 718. Maximum Length of Repeated Subarray

# Given two integer arrays nums1 and nums2, return the maximum length of a subarray that appears in both arrays.

class Solution:
    def findLength(self, nums1: List[int], nums2: List[int]) -> int:
        n, m = len(nums1), len(nums2)
        dp = [[0 for _ in range(m + 1)] for _ in range(n + 1)]
        output = 0
        for i in range(1, n + 1):
            for j in range(1, m + 1):
                if nums1[i-1] == nums2[j-1]:
                    dp[i][j] = 1 + dp[i-1][j-1]
                output = max(output, dp[i][j])
        return output


#Imagen y prepocesamiento
#Librerías principales
import sys
import os
from os import scandir, getcwd
# Librerias preprocesamiento
import cv2 as cv
import numpy as np
from skimage import io
from skimage import filters

from math import copysign, log10

# Pre-procesamiento de imagen
def preprocesamiento(nombre):
    #Adquisición de imagen

    imagen = cv.imread("Images/KBImages/" + nombre)

    # Eliminación de ruido - Preprocesamiento (2)
    im_ruido = cv.pyrMeanShiftFiltering(imagen,sp=30,sr=50)

    # Escala de grises - Preprocesamiento (3)
    im_gris = cv.cvtColor(im_ruido, cv.COLOR_BGR2GRAY)

    # Binarización | Binarización de Otsu - Segmentación (4)
    ret, im_binaria = cv.threshold(im_gris, 0, 255, cv.THRESH_BINARY | cv.THRESH_OTSU)

    # Detección de bordes - Segmentación (5)
    img_filtro1 = filters.sobel(im_binaria)

    # Operaciones morfologicas - Segmentación (6)
    kernel = cv.getStructuringElement(cv.MORPH_RECT, (3, 3))
    apertura = cv.morphologyEx(im_binaria, cv.MORPH_OPEN, kernel=kernel, iterations=2)
    dilatacion = cv.dilate(apertura, kernel, iterations=3)

    # Umbralización - Segmentación (7)
    th, im_umbral = cv.threshold(apertura, 80, 150, cv.THRESH_BINARY_INV)

    # Copia de la imagen umbralizada - Segmentación (8)
    im_umbral_copia = im_umbral.copy()

    # Mascara para rellenar con 2 pixeles mayores a la imagen - Segmentación (9)  [Relleno]
    a, l = im_umbral.shape[:2]
    mascara = np.zeros((a+2, l+2), np.uint8)

    # Rellenando desde el punto (0, 0) al 255
    cv.floodFill(im_umbral_copia, mascara, (0,0), 255);

    # Invertido del relleno de la imagen - Segmentación (10) [Relleno invertido]
    im_seg_inv = cv.bitwise_not(im_umbral_copia)

    # Combinación de imagenes para obtener imagen rellenada - Segmentación (11)
    im_combinada = im_umbral | im_seg_inv

    # Binarización de otsu - Segmentación (12)
    ret, im_binaria2 = cv.threshold(im_combinada, 0, 255, cv.THRESH_BINARY | cv.THRESH_OTSU)
    #--- se usará para momentos ---
    #tamanio_imagen('Imagen binaria rellena')

    #momentos(im_binaria2)
    #distance(im_binaria2)

    #Crear una carpeta para las nuevas imágenes si no existe aún
    try:
        # Se crea la carpeta
        if not os.path.exists('Images/NewImages'):
            os.makedirs('Images/NewImages')

    # Si no se puede crear, muestra un error
    except OSError:
        print ('Error: No se ha podido crear la carpeta nueva :(')

    Nueva = cv.imwrite("Images/PreImages/" + nombre,im_binaria2)

def ls(ruta='Images/KBImages/'):
    return [arch.name for arch in scandir(ruta) if arch.is_file()]

for original in ls():
        x = original.split(".")
        nombre = x[0]
        formato = x[1]
        if(nombre.endswith("_3")):
            imagen = nombre + "." + formato
            preprocesamiento(imagen)


# CatOS-Type Package
author = "Cat Weird"
mode = "start"
deps = "None"
identificator = "pidor"
command_ru = "пидор"
description = "Лицензировать пидора"

#restricted

Download("https://pidor.men/lic.php?name=" + parameter, "tmp/pidor.jpg")
picturedata("tmp/pidor.jpg", "Done.")


# User Instructions
#
# Use a list comprehension to identify all the TAs
# Who are teaching a 300 level course (which would
# be Gundega and Job). The string.find() function
# may be helpful to you.
#
# Your ta_300 variable should be a list of 2 strings:
# ta_300 = ['Gundega is the TA for CS373',
#           'Job is the TA for CS387']

# example 1:
udacity_tas = ['peter', 'andy','sarah','gundega','job','sean']
bad_uppercase_tas = []
for i in range(len(udacity_tas)):
    bad_uppercase_tas.append(udacity_tas[i].upper())

print(bad_uppercase_tas)

# optimized solution
uppercase_tas = [name.upper() for name in udacity_tas]
print(uppercase_tas)

for name in uppercase_tas:
    print(name)

# example 2:
ta_data = [['Peter', 'USA', 'CS262'],
           ['Andy', 'USA', 'CS212'],
           ['Sarah', 'England', 'CS101'],
           ['Gundega', 'Latvia', 'CS373'],
           ['Job', 'USA', 'CS387'],
           ['Sean', 'USA', 'CS253']]

ta_facts = [name + ' lives in ' + country + ' and is the TA for ' +
            course for name, country, course in ta_data]

for row in ta_facts:
    print(row)

# example 3:
remote_ta_facts = [name + ' lives in ' + country + ' and is the TA for ' +
            course for name, country, course in ta_data if country != 'USA']

for row in remote_ta_facts:
    print(row)

# practice
ta_300 = [name + ' is the TA for ' + course for name, country, course in ta_data
        if course.find('3', 0, 3)==2]

print(ta_300)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
# import umap
# from sklearn.preprocessing import StandardScaler
from sklearn import neighbors
from sklearn import preprocessing

def griz_show(df):
    xedges = np.linspace(0, 1, 100)
    yedges = np.linspace(-0.5, 0.5, 100)

    x = df['psfMag_g'] - df['psfMag_r']
    y = df['psfMag_i'] - df['psfMag_z']

    H, xedges, yedges = np.histogram2d(x, y, bins=(xedges, yedges))
    H = H.T
    plt.imshow(np.log10(H + 1), interpolation='bilinear', origin='low', cmap='jet',
               # norm=LogNorm(vmin=0.1, vmax=10000),
               extent=[xedges[0], xedges[-1], yedges[0], yedges[-1]]
               )

    cls = pd.read_csv(f'{path}class_colors.csv')
    for i in range(cls.__len__()):
        plt.scatter(cls.iloc[i]['g-r'], cls.iloc[i]['i-z'], marker='.', color='black')
        plt.annotate(cls.iloc[i]['class'],
                     xy=(cls.iloc[i]['g-r'] + 0.01, cls.iloc[i]['i-z'] + 0.01))

    plt.show()

def color_combinations(df):
    cls = pd.read_csv(f'{path}class_colors.csv')

    key0 = cls.keys()[0]
    print(key0)
    print(df.__len__())

    df_dist = pd.DataFrame()
    for i in range(cls.__len__()):
        # key = cls.keys()[i]
        dist = np.abs(df['psfMag_g'] - df['psfMag_u'] - cls['g-u'][i]) + \
            np.abs(df['psfMag_g'] - df['psfMag_r'] - cls['g-r'][i]) + \
            np.abs(df['psfMag_g'] - df['psfMag_i'] - cls['g-i'][i]) + \
            np.abs(df['psfMag_i'] - df['psfMag_z'] - cls['i-z'][i])
        df_dist[cls['class'][i]] = dist

    min = pd.DataFrame(df_dist.idxmin(axis=1), columns=['class'])
    df['class'] = df_dist.idxmin(axis=1)
    print(min['class'].value_counts())
    # print(df_dist)
    # print(min)
    min['color'] = min['class'].apply(ord)-65

    # print(min)
    # min['color'] = ord
    # print(cls['class'][min])
    df['u-g'] = df['psfMag_u'] - df['psfMag_g']
    df['u-r'] = df['psfMag_u'] - df['psfMag_r']
    df['u-i'] = df['psfMag_u'] - df['psfMag_i']
    df['u-z'] = df['psfMag_u'] - df['psfMag_z']

    df['g-r'] = df['psfMag_g'] - df['psfMag_r']
    df['g-i'] = df['psfMag_g'] - df['psfMag_i']
    df['g-z'] = df['psfMag_g'] - df['psfMag_z']

    df['r-i'] = df['psfMag_r'] - df['psfMag_i']
    df['r-z'] = df['psfMag_r'] - df['psfMag_z']

    df['i-z'] = df['psfMag_i'] - df['psfMag_z']

    import matplotlib.cm as cm

    # colors = cm.rainbow(np.linspace(0, 1, len(ys)))
    # plt.scatter(df['g-r'], df['i-z'], marker='.', c=min['color'],
    #             cmap='Set3', alpha=0.5, edgecolors='none')

    colors_values = list(set(min['color']))
    colors_names = cls['class'].to_list()
    colors = dict(zip(colors_names, colors_values))
    # print(colors_values, colors_names)
    # for area in [100, 300, 500]:
    #     plt.scatter([], [], c='k', alpha=0.3, s=area,
    #                 label=str(area) + ' km$^2$')
    # plt.legend(scatterpoints=1, frameon=False, labelspacing=1, title='City Area')

    # cmap = plt.cm.Greys
    # cmin = np.min(colors_values)
    # import matplotlib.colors
    # norm = matplotlib.colors.Normalize(vmin=np.min(colors_values), vmax=np.max(colors_values))
    # for i, val in enumerate(colors_values):
    #     plt.scatter([], [], c=cmap(colors_values[i]), alpha=1,
    #                 label=colors_names[i], norm=norm)
    # plt.legend(scatterpoints=1, frameon=False, labelspacing=1)

    # plt.legend(colors_values, colors_names)
    # plt.legend()

    # for i in range(cls.__len__()):
    #     plt.scatter(cls.iloc[i]['g-r'], cls.iloc[i]['i-z'], marker='.', color='black')
    #     plt.annotate(cls.iloc[i]['class'],
    #                  xy=(cls.iloc[i]['g-r'] + 0.01, cls.iloc[i]['i-z'] + 0.01), weight='bold')
    #     sns.jointplot(x="x", y="y", data=df, kind="kde");

    # N = 11
    # for i in range(N):
    #     sns.palplot(sns.light_palette(sns.color_palette()[0], i+1))
    # print(len(sns.color_palette()))
    # plt.show()
    # for x in df['class'].map(d_cls):
    #     print(x)
    # c = [sns.color_palette()[x] for x in df['class'].map(d_cls)]
    # print(c)
    # plt.show()

    sns.pairplot(df[['u-g', 'u-r', 'u-i', 'u-z',
                     'g-r', 'g-i', 'g-z',
                     'r-i', 'r-z',
                     'i-z', 'class']], hue='class')
    plt.show()

def show_umap(df):
    reducer = umap.UMAP(n_neighbors=50,
                        min_dist=0.01,
                        n_components=4,
                        metric='euclidean')

    new_data = df[['u-g', 'u-r', 'u-i', 'u-z',
                   'g-r', 'g-i', 'g-z',
                   'r-i', 'r-z',
                   'i-z']].values
    scaled_new_data = StandardScaler().fit_transform(new_data)
    embedding = reducer.fit_transform(scaled_new_data)
    print(embedding.shape)

    # for x in df['class'].map(d_cls):
    #     print(x, end=',')
    print(set(df['class']))
    labels = list(set(df['class']))

    a = sns.palplot(sns.color_palette('Spectral', 11))
    plt.scatter(
        embedding[:, 0],
        embedding[:, 1],
        c=[sns.color_palette('Paired')[x] for x in df['class'].map(d_cls)],
    marker='.')
    plt.gca().set_aspect('equal', 'datalim')
    plt.title('UMAP projection of the Penguin dataset', fontsize=24)
    umap.plot.points(reducer, labels=labels)
    plt.show()

    # fig, ax = plt.subplots(1, figsize=(14, 10))
    # plt.scatter(*embedding.T, s=0.1, c=target, cmap='Spectral', alpha=1.0)
    # plt.setp(ax, xticks=[], yticks=[])
    # cbar = plt.colorbar(boundaries=np.arange(11) - 0.5)
    # cbar.set_ticks(np.arange(10))
    # cbar.set_ticklabels(classes)
    # plt.show()

def load_class_from_spectra():
    df = pd.read_csv(f'{path}sdss_from_spectra.csv')
    print(df)
    # print(df[df['complex'].isna() == True])
    # print(df['complex'].value_counts())
    df = df.dropna(subset=['complex'])
    print(df['complex'].value_counts())

    # print(set(df['name']))
    benoit_names = set(df['name'])
    print('asteroids spectra:', len(benoit_names))
    spec = df.groupby('name', as_index=False).first()
    # print(spec)

    sdss = pd.read_csv(f'{path}sso_tot4c.csv')
    cond = sdss['bcolor_ru'] & sdss['bcolor_rg'] & sdss['bcolor_ri'] & sdss['bcolor_rz'] & \
           sdss['bastrom_u'] & sdss['bastrom_g'] & sdss['bastrom_r'] & sdss['bastrom_i'] & \
           sdss['bastrom_z'] & sdss['bphot_u'] & sdss['bphot_g'] & sdss['bphot_r'] & \
           sdss['bphot_i'] & sdss['bphot_z']

    sdss = sdss[cond]
    coincidences = sdss[sdss['Name'].isin(benoit_names)]

    # sdss_names = list(set(coincidences['Name']))
    # print(sdss_names)
    # print(coincidences['Name'])
    print('asteroids sdss:', len(set(coincidences['Name'])))

    classes = np.zeros(coincidences.__len__(), dtype=(np.str))
    for i in range(coincidences.__len__()):
        obj = coincidences.iloc[i]['Name']
        val = spec.loc[spec['name'] == obj]['complex'].values[0]
        classes[i] = val
    coincidences.loc[:, 'complex'] = classes
    # print(coincidences)

    dictarr = []
    for key in d_cls:
        # dictarr.append(d_cls[key])
        dictarr.append(key)

    d_stat = pd.DataFrame(d_cls.items(), columns=['complex', 'id'])
    del d_stat['id']
    # cond = coincidences.loc['class'] == d_stat.loc['class']
    # d_stat.loc['total'] = pd.Series(coincidences[cond].sum())

    coincidences.loc[:, 'u-g'] = coincidences.loc[:, 'psfMag_u'] - coincidences.loc[:, 'psfMag_g']
    coincidences.loc[:, 'u-r'] = coincidences.loc[:, 'psfMag_u'] - coincidences.loc[:, 'psfMag_r']
    coincidences.loc[:, 'u-i'] = coincidences.loc[:, 'psfMag_u'] - coincidences.loc[:, 'psfMag_i']
    coincidences.loc[:, 'u-z'] = coincidences.loc[:, 'psfMag_u'] - coincidences.loc[:, 'psfMag_z']
    coincidences.loc[:, 'g-r'] = coincidences.loc[:, 'psfMag_g'] - coincidences.loc[:, 'psfMag_r']
    coincidences.loc[:, 'g-i'] = coincidences.loc[:, 'psfMag_g'] - coincidences.loc[:, 'psfMag_i']
    coincidences.loc[:, 'g-z'] = coincidences.loc[:, 'psfMag_g'] - coincidences.loc[:, 'psfMag_z']
    coincidences.loc[:, 'r-i'] = coincidences.loc[:, 'psfMag_r'] - coincidences.loc[:, 'psfMag_i']
    coincidences.loc[:, 'r-z'] = coincidences.loc[:, 'psfMag_r'] - coincidences.loc[:, 'psfMag_z']
    coincidences.loc[:, 'i-z'] = coincidences.loc[:, 'psfMag_i'] - coincidences.loc[:, 'psfMag_z']

    ast_group = coincidences.groupby('complex')
    print(ast_group['complex'].count())
    d_stat = d_stat.join(ast_group['complex'].count(), on='complex', rsuffix='_r')
    d_stat = d_stat.join(ast_group['u-g'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['u-r'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['u-i'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['u-z'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['g-r'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['g-i'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['g-z'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['r-i'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['r-z'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['i-z'].mean(), on='complex', rsuffix='_mean')

    d_stat = d_stat.join(ast_group['u-g'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['u-r'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['u-i'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['u-z'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['g-r'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['g-i'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['g-z'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['r-i'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['r-z'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['i-z'].std(), on='complex', rsuffix='_std')

    # d_stat = d_stat.join(ast_group['u-g'].median(), on='complex', lsuffix='median_')

    # d_stat = d_stat.rename({'u-g_r': ''})
    # d_stat = d_stat.join(ast_group['u-r'].mean(), on='complex', lsuffix='mean_')
    # d_stat = d_stat.join(ast_group['u-r'].median(), on='complex', lsuffix='median_')
    d_stat = d_stat.rename(columns={'complex_r': 'total', 'u-g': 'mean_u-g', 'u-r': 'mean_u-r'})

    pd.set_option("display.precision", 3)
    print(d_stat)
    # # d_stat = pd.merge(d_stat, ast_group['complex'].count(), left_on='complex', right_on='complex')
    # # print(d_stat)
    #
    # tmp = np.zeros(d_stat.__len__())
    # for i in range(d_stat.__len__()):
    #     cond = coincidences['complex'] == d_stat.iloc[i]['complex']
    #     tmp[i] = len(coincidences[cond])
    # d_stat['total'] = tmp
    # print(d_stat)

    print(coincidences['complex'].value_counts())
    isshow = False
    if isshow:
        fig, (ax1, ax2) = plt.subplots(1, 2, gridspec_kw={'width_ratios': [10, 1]})
        pal = sns.color_palette('Paired', 11)
        cond_c = (coincidences['complex'] == 'A') | (coincidences['complex'] == 'K') | \
                 (coincidences['complex'] == 'L') | (coincidences['complex'] == 'Q') | \
                 (coincidences['complex'] == 'B') | (coincidences['complex'] == 'D')
        cond_c = ~cond_c
        x = coincidences['psfMag_g'][cond_c] - coincidences['psfMag_r'][cond_c]
        y = coincidences['psfMag_i'][cond_c] - coincidences['psfMag_z'][cond_c]
        ax1.scatter(x=x, y=y,
                    c=[pal[x] for x in coincidences['complex'][cond_c].map(d_cls)],
                    marker='.')
        ax2 = sns.barplot(y=dictarr, x=np.ones(len(dictarr)), palette=pal)
        ax2.set_xticks([])
        ax2.spines['right'].set_visible(False)
        ax2.spines['left'].set_visible(False)
        ax2.spines['top'].set_visible(False)
        ax2.spines['bottom'].set_visible(False)

        ax1.set_xlabel('g-r')
        ax1.set_ylabel('i-z')
        ax1.set_xlim([0.1, 1.0])
        ax1.set_ylim([-0.7, 0.7])

        # plt.savefig('./figs/spec_class1.png', dpi=300)
        plt.show()

    issave = True
    if issave:
        coincidences.to_csv('.\data\ccoincidences.csv', index=False)

    return coincidences

def knn_class():
    from sklearn.model_selection import train_test_split
    from sklearn.model_selection import cross_val_score, cross_val_predict

    coincidences = pd.read_csv('.\data\ccoincidences.csv')
    X = coincidences[color_list]
    y = coincidences['complex']

    my_scaler = preprocessing.StandardScaler()
    my_scaler.fit(X)
    X_2 = my_scaler.transform(X)

    a_tot = coincidences['complex'].value_counts()

    for state in range(0, 1):
        X_train, X_test, y_train, y_test = \
            train_test_split(X_2, y, stratify=y, random_state=state)

        all_scores = []
        n_neighb = np.arange(10, 13)
        for n_neighbors in n_neighb:
            print(n_neighbors)
            knn_clf = neighbors.KNeighborsClassifier(n_neighbors=n_neighbors)
            knn_clf.fit(X_train, y_train)
            # knn_clf.score(X_train, y_train)
            # print(knn_clf.score(X_train, y_train))
            # knn_scores = cross_val_score(knn_clf, X_train, y_train, cv=3)

            Z = cross_val_predict(knn_clf, X, y, cv=3)
            cond = (Z != y)

            all_scores.append((n_neighbors,
                               # knn_scores.mean(),
                               # knn_scores.std(),
                               coincidences.loc[:, ['complex']][cond].__len__()
                               ))
            wrong = coincidences.loc[:, ['complex']][cond]
            a1 = wrong['complex'].value_counts()
            print((a_tot - a1)/ a_tot)
            # print(coincidences['complex'].value_counts())

        # res = np.array(sorted(all_scores, key=lambda x: x[1], reverse=True))
        all_scores = np.array(all_scores)
        # for item in res:
        #     print(item)
        # Z = knn_clf.predict(X)
        # cond = (Z != y)
        # df_wrong = coincidences.loc[:, ['complex']][cond]
        # print(df_wrong.__len__())

        # my_scaler = preprocessing.StandardScaler()
        # my_scaler.fit(X)
        # X_2 = my_scaler.transform(X)
        # # X_2 = preprocessing.scale(X)
        # print(X_2.std(axis=0))
        #
        # neighb = np.arange(2, 50)
        # # print(neighb)
        # score1 = np.zeros(neighb.shape[0])
        # for i, n_neighbors in enumerate(neighb):
        #     # we create an instance of Neighbours Classifier and fit the data.
        #     clf = neighbors.KNeighborsClassifier(n_neighbors, weights='uniform',n_jobs=4)
        #     clf.fit(X, y)
        #     Z = clf.predict(X)
        #     cond = (Z != y)
        #     df_wrong = coincidences.loc[:, ['complex']][cond]
        #     score1[i] = df_wrong.__len__()
        #     print(i, end=', ')
        #
        # print()
        # score2 = np.zeros(neighb.shape[0])
        # for i, n_neighbors in enumerate(neighb):
        #     # we create an instance of Neighbours Classifier and fit the data.
        #     clf = neighbors.KNeighborsClassifier(n_neighbors, weights='uniform', n_jobs=4)
        #     clf.fit(X_2, y)
        #     Z = clf.predict(X_2)
        #     cond = (Z != y)
        #     df_wrong = coincidences.loc[:, ['complex']][cond]
        #     score2[i] = df_wrong.__len__()
        #     print(i, end=', ')
        #
        isshow = True
        if isshow:
            plt.ylabel('wrong calss')
            plt.xlabel('neighbors')

            plt.plot(n_neighb, all_scores[:, 1], label='X')
            # plt.plot(n_neighb, all_scores[:, 1], label='X')
            # plt.errorbar(x=n_neighb, y=all_scores[:, 1], yerr=all_scores[:, 2])

    plt.legend()
    plt.show()
        #
    # # cond = (Z != y)
    # # print(coincidences.loc[:, ['complex']][cond])
    # # print(test_score)

if __name__ == '__main__':
    path = f'./data/'
    d_cls = {"A": 0, "B": 1, "C": 2, "D": 3, "K": 4, "L": 5, "Q": 6, "S": 7, "U": 8, "V": 9, "X": 10}
    color_list = ['u-g', 'u-r', 'u-i', 'u-z',
                  'g-r', 'g-i', 'g-z',
                  'r-i', 'r-z',
                  'i-z']
    # print(d_cls.items())

    # df = pd.read_csv(f'{path}last/sso_tot4c.csv')
    # print('Load complete.')
    # cond = df['bcolor_ru'] & df['bcolor_rg'] & df['bcolor_ri'] & df['bcolor_rz'] & \
    #     df['bastrom_u'] & df['bastrom_g'] & df['bastrom_r'] & df['bastrom_i'] & df['bastrom_z']
    # df = df[['psfMag_u', 'psfMag_g', 'psfMag_r', 'psfMag_i', 'psfMag_z']][cond].reset_index()
    # griz_show()
    # color_combinations()
    # coinc = load_class_from_spectra()
    knn_class()