# -*- coding: utf-8 -*-
"""
Created on Mon Feb 17 21:02:08 2014

@author: ventilator

Each character on a computer is assigned a unique code and the preferred standard is ASCII (American Standard Code for Information Interchange). For example, uppercase A = 65, asterisk (*) = 42, and lowercase k = 107.

A modern encryption method is to take a text file, convert the bytes to ASCII, then XOR each byte with a given value, taken from a secret key. The advantage with the XOR function is that using the same encryption key on the cipher text, restores the plain text; for example, 65 XOR 42 = 107, then 107 XOR 42 = 65.

For unbreakable encryption, the key is the same length as the plain text message, and the key is made up of random bytes. The user would keep the encrypted message and the encryption key in different locations, and without both "halves", it is impossible to decrypt the message.

Unfortunately, this method is impractical for most users, so the modified method is to use a password as a key. If the password is shorter than the message, which is likely, the key is repeated cyclically throughout the message. The balance for this method is using a sufficiently long password key for security, but short enough to be memorable.

Your task has been made easy, as the encryption key consists of three lower case characters. Using cipher.txt (right click and 'Save Link/Target As...'), a file containing the encrypted ASCII codes, and the knowledge that the plain text must contain common English words, decrypt the message and find the sum of the ASCII values in the original text.

ascii a = 97, ascii z = 122

"""

import profile

from numpy import loadtxt
from itertools import cycle, product

filename = "p059_cipher.txt"

key_alphabet = ''
for k_value in range(ord("a"),ord("z")+1):
    key_alphabet+= chr(k_value)
print(key_alphabet)

def solve_problem():
    encrypted_text = loadtxt(filename, delimiter=",", dtype = int)
    for key in product(key_alphabet, repeat = 3)     :
        decrypted_text = ''.join(chr(e^ord(k)) for e,k in zip(encrypted_text, cycle(key)))
        if (decrypted_text.find(" the ") > -1): # " the " is better than "the"
            print(decrypted_text)
            print(key)
            sum_ascii = 0
            for char in decrypted_text:
                sum_ascii += ord(char)
            print("Sum of characters")
            print(sum_ascii)
            break

    return 0

solve_problem()
# profile.run('solve_problem()')


# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from sklearn.linear_model import Lasso
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

import os
print(os.listdir("../input"))

# Any results you write to the current directory are saved as output.
#read the training data
# train_df =  pd.read_csv('../input/train.csv', nrows = 10_000_000)
train_df =  pd.read_csv('../input/train.csv', nrows = 10_000_000)

train_df['pickup_datetime'] = train_df['pickup_datetime'].str.slice(0, 16)
train_df['pickup_datetime'] = pd.to_datetime(train_df['pickup_datetime'], utc=True, format='%Y-%m-%d %H:%M')
train_df.head()
train_df.loc[:,('dropoff_longitude','pickup_longitude')].head()
print(train_df.dropoff_longitude[:2])
print(train_df.isnull().sum())
train_df = train_df[train_df.fare_amount>=0]

#Remove NaN from dataset
print('Old size %d'% len(train_df))
train_df = train_df.dropna(how = 'any', axis = 'rows')
print('New size: %d' % len(train_df))
#change add_traval_vector_features to calculate distance using  Haversine
# def add_travel_vector_features(df):
#     df['abs_diff_longitude'] = (df.dropoff_longitude - df.pickup_longitude).abs()
#     df['abs_diff_latitude'] = (df.dropoff_latitude - df.pickup_latitude).abs()

# add_travel_vector_features(train_df)
def distance(df):
    p = 0.017453292519943295     #Pi/180
    a = 0.5 - np.cos((df.dropoff_latitude - df.pickup_latitude) * p)/2 + np.cos(df.pickup_latitude * p) * np.cos(df.dropoff_latitude * p) * (1 - np.cos((df.dropoff_longitude - df.pickup_longitude) * p)) / 2
#     return 12742 * asin(sqrt(a)) #2*R*asin...
    df['distance_miles'] = 0.6213712 * 12742 * np.arcsin(np.sqrt(a))
distance(train_df)
# plot = train_df.iloc[:2000].plot.scatter('abs_diff_longitude', 'abs_diff_latitude')
# print('Old size: %d' % len(train_df))
# train_df = train_df[(train_df.abs_diff_longitude < 5.0) & (train_df.abs_diff_latitude < 5.0)]
# print('New size: %d' % len(train_df))
train_df = train_df[(train_df.distance_miles < 345)]

# Removing observations with erroneous values
mask = train_df['pickup_longitude'].between(-75, -73)
mask &= train_df['dropoff_longitude'].between(-75, -73)
mask &= train_df['pickup_latitude'].between(40, 42)
mask &= train_df['dropoff_latitude'].between(40, 42)
mask &= train_df['passenger_count'].between(0, 8)
mask &= train_df['fare_amount'].between(0, 250)

train_df = train_df[mask]

train_df['hour'] = train_df.pickup_datetime.apply(lambda t: pd.to_datetime(t).hour)
train_df['year'] = train_df.pickup_datetime.apply(lambda t: pd.to_datetime(t).year)
# Get input matrix
def get_input_matrix(df):
    return np.column_stack((df.year,df.hour,df.distance_miles,df.passenger_count, np.ones(len(df))))

#X_train= get_input_matrix(train_df)
#y_train = np.array(train_df['fare_amount'])

#print(X_train.shape)
#print(y_train.shape)
X = get_input_matrix(train_df)
y = np.array(train_df['fare_amount'])

from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score

# Create training and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state=42)
# Create the regressor: reg_all
reg_all = LinearRegression()
cv_scores = cross_val_score(reg_all,X,y,cv=5)
# Fit the regressor to the training data
reg_all.fit(X_train, y_train)
# Predict on the test data: y_pred
y_pred = reg_all.predict(X_test)

print("R^2: {}".format(reg_all.score(X_test, y_test)))
rmse = np.sqrt(mean_squared_error(y_test , y_pred))
print("Root Mean Squared Error: {}".format(rmse))

# Print the 5-fold cross-validation scores
print(cv_scores)

print("Average 5-Fold CV Score: {}".format(np.mean(cv_scores)))

train_df.columns.values
df_columns = pd.Index([  'pickup_longitude',
       'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude',
       'passenger_count', 'distance_miles','hour','year'])
df_columns
#Find the most important variable
# Instantiate a lasso regressor: lasso
X_all_feature = np.column_stack((train_df.pickup_longitude,train_df.pickup_latitude,train_df.dropoff_longitude,
                    train_df.dropoff_latitude,train_df.passenger_count,train_df.distance_miles,train_df.hour,train_df.year))
X_all_feature
lasso = Lasso(alpha=0.1)

# Fit the regressor to the data
lasso.fit(X_all_feature,y)

# Compute and print the coefficients
lasso_coef = lasso.coef_
print(lasso_coef)

# Plot the coefficients
plt.plot(range(len(df_columns)), lasso_coef)
plt.xticks(range(len(df_columns)), df_columns.values, rotation=60)
plt.margins(0.02)
plt.show()
test_df = pd.read_csv('../input/test.csv')
test_df.dtypes
# Reuse the above helper functions to add our features and generate the input matrix.
# add_travel_vector_features(test_df)
distance(test_df)
test_df['hour'] = test_df.pickup_datetime.apply(lambda t: pd.to_datetime(t).hour)
test_df['year'] = test_df.pickup_datetime.apply(lambda t: pd.to_datetime(t).year)
test_X = get_input_matrix(test_df)

# Predict fare_amount on the test set using our model (w) trained on the training set.
# test_y_predictions = np.matmul(test_X, w_OLS).round(decimals = 2)
test_y_predictions = reg_all.predict(test_X)
test_y_predictions

# Write the predictions to a CSV file which we can submit to the competition.
submission = pd.DataFrame(
    {'key': test_df.key, 'fare_amount': test_y_predictions},
    columns = ['key', 'fare_amount'])
submission.to_csv('submission.csv', index = False)

print(os.listdir('.'))
test_y_predictions[:200]
data = pd.read_csv('submission.csv')
data


import tensorflow as tf
import numpy as np
from bench.core.executer import Executer

def accuracy(pred, target, topk=1, thresh=None):
    if isinstance(topk, int):
        topk = (topk, )
        return_single = True
    else:
        return_single = False

    maxk = max(topk)
    if pred.shape[0] == 0:
        accu = [tf.zeros(tf.shape(pred)) for i in range(len(topk))]
        return accu[0] if return_single else accu
    pred_value, pred_label = tf.math.top_k(pred, maxk)
    pred_label = tf.transpose(pred_label)  # transpose to shape (maxk, N)
    correct = tf.equal(tf.cast(tf.reshape(target, (1, -1)), tf.int32), pred_label)
    if thresh is not None:
        # Only prediction values larger than thresh are counted as correct
        correct = tf.bitwise.bitwise_and(correct, tf.transpose(tf.math.greater(pred_value, thresh)))
    res = []
    for k in topk:
        correct_k = tf.reduce_sum(tf.cast(tf.reshape(correct[:k], [-1]), tf.float32), axis=0, keepdims=True)
        res.append(tf.multiply(correct_k, 100.0 / pred.shape[0]))
    return res[0] if return_single else res

def args_adaptor(np_args):
    output = tf.convert_to_tensor(np_args[0], tf.float32)
    target = tf.convert_to_tensor(np_args[1], tf.float32)

    return [output, target, 1]

def executer_creator():
    return Executer(accuracy, args_adaptor)


# -*- coding: utf-8 -*-

"""
@Author: xiezizhe
@Date: 23/6/2020 下午7:29
"""

from typing import List

class Solution:
    def maximalSquare(self, matrix: List[List[str]]) -> int:
        if not matrix:
            return 0

        m, n = len(matrix), len(matrix[0])

        ret = 0
        dp = [[0 for __ in range(n)] for _ in range(m)]

        # dp[:][0] = map(int, matrix[:][0])
        # dp[0][:] = map(int, matrix[0][:])

        for i in range(m):
            if matrix[i][0] == '1':
                dp[i][0] = 1
                ret = 1

        for i in range(n):
            if matrix[0][i] == '1':
                dp[0][i] = 1
                ret = 1

        for i in range(1, m):
            for j in range(1, n):
                if matrix[i][j] == '1':
                    dp[i][j] = min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]) + 1
                    ret = max(ret, dp[i][j])

        return ret * ret

if __name__ == "__main__":
    s = Solution()
    print(s.maximalSquare([["0", "1"], ["1", "0"]]))


from torch import nn, einsum
import torch
from functools import partial

from src.models.helpers import exists, default, PreNorm, SinusoidalPositionEmbeddings, Residual
from src.models.conv_blocks import ResnetBlock, Downsample, Upsample
from src.models.attention import Attention, LinearAttention

from src.train.util import partial_load_model

class Unet(nn.Module):
    def __init__(
        self,
        img_size,
        init_dim = None,
        dim_mults = (1, 2, 4, 8),
        time_dim = None,
        in_channels = 3,
        out_channels = 3,
        down_mode = 'avg',
        down_kern = 2,
        up_mode = 'nearest',
        up_scale = 2,
        resnet_stacks = 2,
        attn_heads = 4,
        attn_head_res = 32,
        self_condition = False,
        resnet_grnorm_groups = 4,
        classes = None
    ):
        """
        img_size             - assuming square image of (img_size, img_size)
        init_dim             - out planes from the pre-Unet layer (i.e. in_channels -> init_dim)
        dim_mults            - tuple of multipliers to multiply element-wise init_dim,
        time_dim             - dimensinality of time embeddings, if None: 4*img_size
        in_channels          - number of input channels
        out_channels         - number of output channels of the UNet
        self_condition       - legace from Hugging Face implenetation. IDK
        resnet_grnorm_groups - numbers for group norm for each  ResNet blocks
        resnet_stacks        - number of ResNet blocks
        classes              - number of classes or list of integers for number of classes for each label/type
        down_mode:           - if "conv", then strided convolution is used,
                             "avg" ot "max" - average2D or maxPool2D are used respectively
        down_kern:           - Size of the pooling kernel, has effect only if down_mode is avg or max
        up_mode:             - If 'conv", strided transposed convolution is used, otherwise, interpolation
        up_scale             - Upscale interpolation factor, default 2

        Adopted from: https://huggingface.co/blog/annotated-diffusion with modifications:
        - module lists init
        - replaced mid-attention with linear attention
        - added class conditions
        - replaced 'manual' down and up with a for-loop
        """
        super().__init__()

        # determine dimensions
        if not out_channels:
            out_channels = in_channels
        self.self_condition = self_condition
        input_channels = in_channels * (2 if self_condition else 1)
        init_dim = init_dim if init_dim else img_size
        dims = [init_dim, *map(lambda m: init_dim * m, dim_mults)]
        in_out = list(zip(dims[:-1], dims[1:]))
        self.resnet_stacks = resnet_stacks

        # time embeddings
        if not time_dim:
            time_dim = img_size * 4

        self.init_conv = nn.Conv2d(input_channels, init_dim, 1, padding = 0)
        conv_unit = partial(ResnetBlock, groups=resnet_grnorm_groups)

        self.time_mlp = nn.Sequential(
            SinusoidalPositionEmbeddings(img_size),
            nn.Linear(img_size, time_dim),
            nn.GELU(),
            nn.Linear(time_dim, time_dim),
        )

        # layers
        self.downs = nn.ModuleList([])
        self.ups = nn.ModuleList([])
        num_resolutions = len(in_out)

        # Down
        for ind, (dim_in, dim_out) in enumerate(in_out):
            is_last = ind >= (num_resolutions - 1)

            self.downs.append(
                nn.ModuleList([conv_unit(dim_in, dim_in, time_emb_dim=time_dim) for _ in range(self.resnet_stacks)])
                +
                nn.ModuleList(
                    [
                        Residual(PreNorm(dim_in, LinearAttention(dim = dim_in, heads=attn_heads, dim_head=attn_head_res))),
                        Downsample(dim_in, dim_out, down_mode, down_kern)
                        if not is_last
                        else nn.Conv2d(dim_in, dim_out, 3, padding=1),
                    ]
                )
            )

        mid_dim = dims[-1]
        self.mid_block1 = conv_unit(mid_dim, mid_dim, time_emb_dim=time_dim)
        self.mid_attn = Residual(PreNorm(mid_dim, Attention(dim=mid_dim, heads=attn_heads, dim_head=attn_head_res))) # replaced with Linear attention
        #self.mid_attn = Residual(PreNorm(mid_dim, LinearAttention(dim = mid_dim, heads=attn_heads, dim_head=attn_head_res)))
        self.mid_block2 = conv_unit(mid_dim, mid_dim, time_emb_dim=time_dim)

        # Up
        for ind, (dim_in, dim_out) in enumerate(reversed(in_out)):
            is_last = ind == (len(in_out) - 1)

            self.ups.append(
                nn.ModuleList([conv_unit(dim_out + dim_in, dim_out, time_emb_dim=time_dim) for _ in range(self.resnet_stacks)])
                +
                nn.ModuleList(
                    [
                        Residual(PreNorm(dim_out, LinearAttention(dim=dim_out, heads=attn_heads, dim_head=attn_head_res))),
                        Upsample(dim_out, dim_in, up_mode, up_scale)
                        if not is_last
                        else nn.Conv2d(dim_out, dim_in, 3, padding=1),
                    ]
                )
            )

        self.final_res_block = conv_unit(dim_in*2, img_size, time_emb_dim=time_dim) # dim * 2
        self.final_conv = nn.Conv2d(img_size, out_channels, 1)

        if classes:
            if type(classes) == int:
                self.classes = [classes]
            else:
                self.classes = classes
            # nn.Embeddings are commented out for now
            # the new approach with MLP should be flexible
            # when multi-class labels are supplied
            #self.lbl_embeds = nn.Embedding(classes, time_dim)
            self.lbl_embeds = nn.Sequential(*[
                nn.Linear(len(self.classes), time_dim),
                nn.GELU(),
                nn.Linear(time_dim, time_dim)
            ])

    def forward(self, x, time, x_self_cond=None, lbls = None):
        if self.self_condition:
            x_self_cond = default(x_self_cond, lambda: torch.zeros_like(x))
            x = torch.cat((x_self_cond, x), dim=1)

        x = self.init_conv(x)
        r = x.clone()

        t = self.time_mlp(time)

        if lbls is not None:
            t += self.lbl_embeds(lbls)

        h = []

        # downwards descent
        for i, layer in enumerate(self.downs):
            # iterate over resnet blocks
            for block in layer[0:self.resnet_stacks]:
                x = block(x,t)
                h.append(x)
            # attention
            x = layer[self.resnet_stacks](x)
            # downsampler
            x = layer[-1](x)

        x = self.mid_block1(x, t)
        x = self.mid_attn(x)
        x = self.mid_block2(x, t)

        # upwards ascent
        for i, layer in enumerate(self.ups):
            # iterate over resnet blocks
            for bi, block in enumerate(layer[0:self.resnet_stacks]):
                x_down = h.pop()
                x = torch.cat((x,  x_down), dim=1)
                x = block(x,t)
            # attention
            x = layer[self.resnet_stacks](x)
            # upsampler
            x = layer[-1](x)

        x = torch.cat((x, r), dim=1)

        x = self.final_res_block(x, t)
        return self.final_conv(x)
# ----------------------------------------------------------------------------------------

def set_unet(config_dict):
    img_size             = config_dict['img_size']
    init_dim             = config_dict.pop('init_dim', None)
    dim_mults            = config_dict.pop('dim_mults', (1, 2, 4, 8))
    time_dim             = config_dict.pop('time_dim', None)
    in_channels          = config_dict.pop('in_channels', 3)
    out_channels         = config_dict.pop('out_channels', 3)
    self_condition       = config_dict.pop('self_condition', False)
    resnet_grnorm_groups = config_dict.pop('resnet_grnorm_groups', 4)
    resnet_stacks        = config_dict.pop('resnet_stacks', 2)
    classes              = config_dict.pop('classes', None)
    down_mode            = config_dict.pop('classes', 'avg')
    down_kern            = config_dict.pop('down_kern', 2)
    up_mode              = config_dict.pop('up_mode', 'bilinear')
    up_scale             = config_dict.pop('up_scale', 2)
    attn_heads           = config_dict.pop('attn_heads', 4)
    attn_head_res        = config_dict.pop('attn_head_res', 16)

    model = Unet(
        img_size             = img_size,
        init_dim             = init_dim,
        dim_mults            = dim_mults,
        time_dim             = time_dim,
        in_channels          = in_channels,
        out_channels         = out_channels,
        self_condition       = self_condition,
        resnet_grnorm_groups = resnet_grnorm_groups,
        resnet_stacks        = resnet_stacks,
        classes              = classes,
        down_mode            = down_mode,
        down_kern            = down_kern,
        up_mode              = up_mode,
        up_scale             = up_scale,
        attn_heads           = attn_heads,
        attn_head_res        = attn_head_res
    )

    if 'load_name' in config_dict:
        weights = config_dict['load_name']
    else:
        weights = None

    if weights:
        print(f'Loading model weights from\n\t{weights}')
        model = partial_load_model(model, weights)

    return model


import os

import pandas as pd
import numpy as np
import tensorflow as tf

from os.path import normpath

from tensorflow.math import reduce_sum

from tensorflow.keras import (
    Input,
    Model
)

from tqdm.auto import tqdm

from joblib import Parallel, delayed

from statsmodels.stats.multitest import multipletests

from .onehot_dna import (
    nuc_to_vec,
    convert_masked_seq_dict,
    seq_to_mat
)

from .onehot_dna import (
    get_degenerate_pct
)

def force_cpu_only():
    print('Switching to CPU-only.')
    tf.config.set_visible_devices([], 'GPU')
    tf.config.threading.set_intra_op_parallelism_threads(2)
    print(f'Intra-op threads: {tf.config.threading.get_intra_op_parallelism_threads()}')
    tf.config.threading.set_inter_op_parallelism_threads(1)
    print(f'Inter-op threads: {tf.config.threading.get_inter_op_parallelism_threads()}')

def manage_gpu_memory(device_index = None):
    physical_devices = tf.config.list_physical_devices('GPU')
    if device_index is None:
        device_indices = range(len(physical_devices))
    else:
        device_indices = [device_index]
    for idx in device_indices:
        try:
            tf.config.experimental.set_memory_growth(physical_devices[idx], True)
        except:
            pass

def scored_fasta_dicts_to_df(
    sequence_dict,
    score_dict,
    description_dict
):
    return pd.DataFrame(
        data = [
            (
                sequence_id,
                sequence,
                score_dict[sequence_id],
                description_dict[sequence_id]
            )
            for sequence_id, sequence
            in sequence_dict.items()
        ],
        columns = [
            'sequence_id',
            'sequence',
            'score',
            'description'
        ]
    )

def filter_scored_fasta_df(
    scored_fasta_df,
    degenerate_pct_thresh = 100,
    sequence_length = 'max'
):
    df = scored_fasta_df.copy()
    df['length'] = df['sequence'].map(len)

    if sequence_length == 'max':
        sequence_length = df['length'].max()
    if sequence_length is not None:
        df = df[df['length'] == sequence_length].copy()

    df['degenerate_pct'] = df['sequence'].map(get_degenerate_pct)
    df = (
        df[
            df['degenerate_pct'] <= degenerate_pct_thresh
        ]
        .copy()
        .reset_index(drop = True)
    )

    return df

def order_scored_fasta_df(
    scored_fasta_df,
    shuffle = True,
    sort = True,
    ascending = True,
    sort_column = 'score'
):
    df = scored_fasta_df.copy()

    if shuffle:
        df = (
            df
            .sample(frac = 1)
            .copy()
            .reset_index(drop = True)
        )
    if sort:
        df = (
            df.sort_values(
                by = sort_column,
                ascending = ascending
            )
            .copy()
            .reset_index(drop = True)
        )
        df['rank'] = np.argsort(df[sort_column])

    return df

def onehot(
    sequence,
    nuc_to_vec,
    convert_masked_seq
):
    return np.array(seq_to_mat(
        sequence,
        nuc_to_vec = nuc_to_vec,
        convert_masked_seq = convert_masked_seq
    ))

def scored_fasta_df_to_dataset(
    scored_fasta_df,
    batch_size = 1000,
    nuc_to_vec = nuc_to_vec,
    convert_masked_seq = convert_masked_seq_dict['n'],
    sequence_column = 'sequence',
    onehot_column = 'onehot_sequence',
    score_column = 'score',
    n_jobs = 1,
    progress_wrapper = tqdm
):

    # def onehot(sequence):
    #     return np.array(seq_to_mat(
    #         sequence,
    #         nuc_to_vec = nuc_to_vec,
    #         convert_masked_seq = convert_masked_seq
    #     ))

    df = scored_fasta_df.copy()
    if n_jobs > 1:
        df[onehot_column] = Parallel(n_jobs = n_jobs)(
            delayed(onehot)(
                sequence,
                nuc_to_vec,
                convert_masked_seq
            )
            for sequence
            in progress_wrapper(df[sequence_column])
        )
    else:
        df[onehot_column] = [
            onehot(
                sequence,
                nuc_to_vec,
                convert_masked_seq
            )
            for sequence
            in progress_wrapper(df[sequence_column])
        ]

    onehot_sequences = np.stack(list(df[onehot_column]))
    scores = df[score_column].astype(float)
    data = [
        onehot_sequences,
        list(scores)
    ]

    return tf.data.Dataset.from_tensor_slices(tuple(data)).map(
        lambda onehot_sequences, scores: (
            tf.cast(onehot_sequences, tf.uint8),
            tf.cast(scores, tf.as_dtype('float'))
        )
    ).batch(batch_size)

def get_input_shape_from_dataset(dataset):
    return tuple(dataset.element_spec[0].shape)[1:]

def get_sequence_length_from_dataset(dataset):
    return get_input_shape_from_dataset(dataset)[0]

def get_normalize_onehot_model(model_input):
    normalize_onehot_model_output = (
        model_input /
        reduce_sum(model_input, axis = -1, keepdims = True)
    )
    normalize_onehot_model = Model(
        inputs = model_input,
        outputs = normalize_onehot_model_output
    )
    return normalize_onehot_model

def get_local_gc_model(
    model_input,
    normalize_onehot_model = None
):
    if normalize_onehot_model is None:
        normalize_onehot_model = get_normalize_onehot_model(model_input)
    local_gc_model_output = reduce_sum(
        normalize_onehot_model(model_input)[:,:,1:-1],
        axis = -1,
        keepdims = True
    )
    local_gc_model = Model(
        inputs = model_input,
        outputs = local_gc_model_output
    )
    return local_gc_model

def get_local_pooled_gc_model(
    model_input,
    pool_size = 1,
    strides = 1,
    padding = 'same',
    local_gc_model = None
):
    if local_gc_model is None:
        local_gc_model = get_local_gc_model(model_input)
    model_input_layer = InputLayer(
        input_shape = model_input.shape[1:]
    )
    pool = AveragePooling1D(
        pool_size = pool_size,
        strides = strides,
        padding = padding
    )
    local_pooled_gc_model = Sequential([
        model_input_layer,
        local_gc_model,
        pool
    ])

    return local_pooled_gc_model

def get_global_gc_model(
    model_input,
    local_gc_model = None
):
    if local_gc_model is None:
        local_gc_model = get_local_gc_model(model_input)
    global_gc_model_output = (
        reduce_sum(
            local_gc_model(model_input),
            axis = -2,
            keepdims = True
        ) /
        model_input.shape[-2])
    global_gc_model = Model(
        inputs = model_input,
        outputs = global_gc_model_output
    )
    return global_gc_model

def append_gc_ratios_to_dataset(dataset, gc_ratio_type = 'global'):

    model_input = Input(shape = get_input_shape_from_dataset(dataset))

    normalize_onehot_model = get_normalize_onehot_model(
        model_input
    )

    local_gc_model = get_local_gc_model(
        model_input,
        normalize_onehot_model = normalize_onehot_model
    )

    if gc_ratio_type is 'global':
        gc_model = get_global_gc_model(
            model_input,
            local_gc_model = local_gc_model
        )
    elif gc_ratio_type is 'local':
        gc_model = local_gc_model

    def append_gc_ratio_to_data(onehot_sequences, scores):
        return (
            onehot_sequences,
            scores,
            gc_model(
                tf.cast(
                    onehot_sequences,
                    tf.as_dtype('float')
                )
            )
        )

    return dataset.map(append_gc_ratio_to_data)

def filepaths_df_to_profile_dicts(
    df,
    motif_orientation = '+',
    mepp_plot_format = 'png'
):
    filepaths_df = df.copy()

    filepaths_df['positions_df_filepath'] = (
        filepaths_df['outdir'] +
        '/positions_df.pkl'
    )
    filepaths_df['mepp_plot_filepath'] = (
        filepaths_df['outdir'] +
        f'/mepp_plot.{mepp_plot_format}'
    )
    filepaths_df['motif_matrix_filepath'] = (
        filepaths_df['outdir'] +
        f'/motif_matrix.npy'
    )
    filepaths_df['positions_df_exists'] = (
        filepaths_df['positions_df_filepath']
        .map(normpath)
        .map(os.path.exists)
    )

    filepaths_df = filepaths_df[
        (filepaths_df['orientation'] == motif_orientation) &
        (filepaths_df['positions_df_exists'] == True) &
        (filepaths_df['retval'] == 0)
    ].copy()

    motif_id_to_profile_df_ = (
        filepaths_df
        .copy()
        .set_index('motif_id')['positions_df_filepath']
        .map(normpath)
        .map(pd.read_pickle)
    ).to_dict()

    motif_id_to_profile_df = {
        k:v
        for k,v
        in motif_id_to_profile_df_.items()
        if v['positional_r'].fillna(0.0).abs().sum()>0.0
    }

    motif_id_to_profile = {
        motif_id: list(df['positional_r'])
        for motif_id, df
        in motif_id_to_profile_df.items()
    }

    motif_id_to_mepp_plot = (
        filepaths_df
        .set_index('motif_id')
        ['mepp_plot_filepath']
        .to_dict()
    )

    motif_id_to_motif_matrix = {
        k: np.load(normpath(v), allow_pickle = True)
        for k,v
        in (
            filepaths_df
            .set_index('motif_id')[
                'motif_matrix_filepath'
            ]
            .to_dict()
            .items()
        )
    }

    return dict(
        motif_id_to_profile = motif_id_to_profile,
        motif_id_to_profile_df = motif_id_to_profile_df,
        motif_id_to_mepp_plot = motif_id_to_mepp_plot,
        motif_id_to_motif_matrix = motif_id_to_motif_matrix,
        filepaths_df = filepaths_df
    )

def get_minmax_stats(positional_r_df):
    df = positional_r_df.copy()
    has_permutation = (
        ('permutation_positional_r_lower' in list(df.columns)) and
        ('permutation_positional_r_upper' in list(df.columns))
    )
    col_prefix = ''
    if has_permutation:
        col_prefix = 'permutation_'
    min_df = df[df['positional_r'] == df['positional_r'].min()].head(1)
    max_df = df[df['positional_r'] == df['positional_r'].max()].head(1)
    min_r = list(min_df['positional_r'])[0]
    min_r_pos = list(min_df['position'])[0]
    min_r_pval = list(min_df[f'{col_prefix}positional_r_pval'])[0]
    max_r = list(max_df['positional_r'])[0]
    max_r_pval = list(max_df[f'{col_prefix}positional_r_pval'])[0]
    max_r_pos = list(max_df['position'])[0]
    if np.abs(max_r) >= np.abs(min_r):
        extreme_r = max_r
        extreme_r_pval = max_r_pval
        extreme_r_pos = max_r_pos
    else:
        extreme_r = min_r
        extreme_r_pval = min_r_pval
        extreme_r_pos = min_r_pos
    r_range = max_r - min_r

    has_integral = 'integral_r' in list(positional_r_df.columns)
    if has_integral:
        integral_r = positional_r_df['integral_r'].mean()
        integral_r_lower = positional_r_df['permutation_integral_r_lower'].mean()
        integral_r_upper = positional_r_df['permutation_integral_r_upper'].mean()
        integral_r_pval = positional_r_df['permutation_integral_r_pval'].mean()
    else:
        integral_r = None
        integral_r_lower = None
        integral_r_upper = None
        integral_r_pval = None

    return dict(
        integral_r = integral_r,
        integral_r_lower = integral_r_lower,
        integral_r_upper = integral_r_upper,
        integral_r_pval = integral_r_pval,
        extreme_r = extreme_r,
        abs_extreme_r = np.abs(extreme_r),
        extreme_r_pval = extreme_r_pval,
        extreme_r_pos = extreme_r_pos,
        max_r = max_r,
        max_r_pval = max_r_pval,
        max_r_pos = max_r_pos,
        min_r = min_r,
        min_r_pval = min_r_pval,
        min_r_pos = min_r_pos,
        r_range = r_range
    )

def get_minmax_stats_df(
    motif_id_to_profile_df,
    mt_method = 'fdr_tsbky',
    mt_alpha = 0.01,
    thorough_mt = True
):
    minmax_stats_df = pd.DataFrame.from_records([
        dict(
            **{'motif_id': k},
            **get_minmax_stats(df)
        )
        for k, df
        in motif_id_to_profile_df.items()
    ])

    if thorough_mt:
        if 'permutation_positional_r_pval' in list(list(motif_id_to_profile_df.values())[0].columns):
            pval_col = 'permutation_positional_r_pval'
        else:
            pval_col = 'positional_r_pval'
        df = pd.concat({k: df[['position', pval_col]].copy() for k, df in motif_id_to_profile_df.items()})
        df = df.reset_index().rename(columns={'level_0':'motif_id'}).drop(columns = ['level_1'])
        df['padj'] = multipletests(
            df[pval_col],
            method = mt_method,
            alpha = mt_alpha)[1]
        df['sig'] = df['padj'] <= mt_alpha

        permutation_positional_r_padj_lu = df.set_index(['motif_id','position'])['padj'].to_dict()
        permutation_positional_r_sig_lu = df.set_index(['motif_id','position'])['sig'].to_dict()

    for prefix in ['extreme', 'max', 'min']:
        pval_col = f'{prefix}_r_pval'
        padj_col = pval_col.replace('pval', 'padj')
        sig_col = padj_col.replace('padj', 'sig')

        if thorough_mt == False:
            padjs = multipletests(
                minmax_stats_df[pval_col],
                method = mt_method,
                alpha = mt_alpha
            )[1]
            sigs = padjs <= mt_alpha
        else:
            padjs = np.nan
            sigs = np.nan

        pval_col_idx = list(minmax_stats_df.columns).index(pval_col)
        minmax_stats_df.drop(
            columns=[
                col
                for col
                in [padj_col, sig_col]
                if col
                in list(minmax_stats_df.columns)
            ],
            inplace = True
        )
        minmax_stats_df.insert(pval_col_idx+1, sig_col, sigs)
        minmax_stats_df.insert(pval_col_idx+1, padj_col, padjs)

        if thorough_mt:
            lu_keys = minmax_stats_df[['motif_id', f'{prefix}_r_pos']].apply(tuple, axis = 1)
            minmax_stats_df[f'{prefix}_r_padj'] = lu_keys.map(permutation_positional_r_padj_lu)
            minmax_stats_df[f'{prefix}_r_sig'] = lu_keys.map(permutation_positional_r_sig_lu)

    has_integral = (minmax_stats_df['integral_r'].isna().any() == False)
    if has_integral:
        pval_col = 'integral_r_pval'
        pval_col_idx = list(minmax_stats_df.columns).index(pval_col)
        padj_col = 'integral_r_padj'
        sig_col = 'integral_r_sig'
        padjs = multipletests(
            minmax_stats_df[pval_col],
            method = mt_method,
            alpha = mt_alpha
        )[1]
        sigs = padjs <= mt_alpha
        minmax_stats_df.insert(pval_col_idx+1, sig_col, sigs)
        minmax_stats_df.insert(pval_col_idx+1, padj_col, padjs)

    return minmax_stats_df

def re_prefix_filepath_dict(filepath_dict, old_prefix, new_prefix):
    return {
        k: new_prefix+v[len(old_prefix):]
        for k, v
        in filepath_dict.items()
    }


from __future__ import absolute_import
from __future__ import print_function

import io
import re
from glob import glob
from os.path import basename
from os.path import dirname
from os.path import join
from os.path import splitext

from setuptools import find_packages
from setuptools import setup

def read(*names, **kwargs):
    with io.open(
        join(dirname(__file__), *names),
        encoding=kwargs.get('encoding', 'utf8')
    ) as fh:
        return fh.read()

with open("README.md", "r", encoding="utf-8") as fh:
    long_description = fh.read()

setup(
    name="swat-pytools",
    version="0.0.1",
    author="J. Sebastian Hernandez-Suarez",
    author_email="herna505@msu.edu",
    description="A Python wrapper for executing and calibrating SWAT",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/jshernandezs/swat-pytools",
    project_urls={
        "Bug Tracker": "https://github.com/jshernandezs/swat-pytools/issues",
    },
    install_requires=['autograd>=1.3', 'hvwfg>=1.0.2', 'matplotlib>=3.4',
    'numpy>=1.15', 'pandas>=1.1', 'pymoo==0.4.2', 'scipy>=1.1'],
    classifiers=[
        "Intended Audience :: Science/Research",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
        "Topic :: Scientific/Engineering",
        "License :: OSI Approved :: GNU General Public License",
        "Operating System :: Unix/macOS",
    ],
    package_dir={"": "src"},
    packages=find_packages("src"),
    py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')],
    include_package_data=True,
    zip_safe=False,
    python_requires=">=3.7",
    entry_points={
        'console_scripts': [
            'api = api.api:main',
        ]
    },
)


from ultralytics import YOLO
from PIL import Image

model = YOLO("Models/YoloV8_own/best.pt")

results = model.predict("Pictures/germanShepherd.jpg")
result = results[0]
len(result.boxes)
box = result.boxes[0]

# print("Object type:",box.cls[0])
# print("Coordinates:",box.xyxy[0])
# print("Probability:",box.conf[0])
# print(result.names)

for box in result.boxes:
    class_id = result.names[box.cls[0].item()]
    cords = box.xyxy[0].tolist()
    cords = [round(x) for x in cords]
    conf = round(box.conf[0].item(), 2)
    print("Object type:", class_id)
    print("Coordinates:", cords)
    print("Probability:", conf)
    print("---")

Image.fromarray(result.plot()[:, :, ::-1]).show()


from typing import List
import pandas as pd
import pytest
from numpy.testing import assert_allclose

from make_dataset import read_data
from entities.feature_params import FeatureParams
from src.features.build_features import process_features, extract_target, build_transformer

@pytest.fixture
def feature_params(
        categorical_features: List[str],
        numerical_features: List[str],
        target_col: str,
) -> FeatureParams:
    params = FeatureParams(
        categorical_features=categorical_features,
        numerical_features=numerical_features,
        target_col=target_col,
    )
    return params

def test_make_features(
        feature_params: FeatureParams, dataset_path: str,
):
    data = read_data(dataset_path)
    transformer = build_transformer(feature_params)
    transformer.fit(data)
    features = process_features(transformer, data)
    assert not pd.isnull(features).any().any()
    assert len(features) == len(data)

def test_extract_features(
        feature_params: FeatureParams,
        dataset_path: str,
):
    data = read_data(dataset_path)
    target = extract_target(data, feature_params)
    assert_allclose(
        data[feature_params.target_col].to_numpy(), target.to_numpy()
    )

def test_target(
        feature_params: FeatureParams,
        dataset_path: str,
):
    data = read_data(dataset_path)
    assert len(data[feature_params.target_col]) > 0
    assert len(data[feature_params.numerical_features]) == len(data[feature_params.target_col])


from __future__ import division,print_function
import sys,random
sys.dont_write_bytecode = True

import housing,ranges
from rangesLib import *

# add in scorerd. abcd mre, lift

def g(x): return ('%g' % x)

def _rangeLib(seed=1):
  random.seed(seed)
  def show(n,rule):
    print(n,g(rule.score),rule)
    #print(n,sorted(map(lambda z:z.id,rule.rows)))
  t=housing.housing()
  b4=Counts(map(lambda l:l.score,t.data))
  print(b4.mu)
  r=[]
  for i  in t.indep:
    more=ranges.sdiv(t.data,attr=t.names[i],tiny=20,
              x=lambda z:z[i],
              y=lambda z:z.score,
              small=0.01)
    print(t.names[i],len(more) )
    if len(more) > 1:
        r += more
  #r = sorted(r,key=lambda z : z.y.mu*-1)
  for one in r:
    if one.y.mu > b4.mu:
      print(one,o(mu=g(one.y.mu),sd=g(one.y.sd)), '<==',o(b4=g(b4.mu),sd0=g(b4.sd())))
  #exit()
  rules =map(lambda z:Rule([z],z.rows),r)
  most=b4.mu
  n=0
  for _ in range(32):
    rules = sorted(rules,key=lambda z: z.score)[-16:]
    for i in range(32):
      n+=1
      rule1  = random.choice(rules)
      rule2  = random.choice(rules)
      rule3 = rule1+rule2
      if rule3 and len(rule3.rows) > len(t.data)**0.5  and rule3.score > most:
        most = rule3.score
        print("")
        print(n,most,int(100*most/b4.mu))
        show(1,rule1)
        show(2,rule2)
        show(3,rule3)
        rules += [rule3]
  print("\n",n,sorted(rules,key=lambda z: z.score)[-1])

if __name__ == "__main__":
    _rangeLib()


from multiprocessing.sharedctypes import Value
import atexit
import os
import wandb

from pathlib import Path

from minerl3161.agents import BaseAgent

class Checkpointer:
    def __init__(
        self,
        agent: BaseAgent,
        checkpoint_every: int = None,
        use_wandb: bool = True,
        use_atexit: bool = True
    ) -> None:
        """
        Checkpointer class manager agent checkpointing. Currently only checkpointing with wandb is supported.

        Args:
            agent (BaseAgent): The agent to checkpoint
            checkpoint_every (int): checkpoint every n timesteps. Defaults to None.
            use_wandb (bool): whether or not to checkpoint with wandb. Defaults to True.
            use_atexit (bool): whether to make a checkpoint at exit

        Raises:
            ValueError: Throws error if checkpoint_every is set, but use_wandb == False.
        """
        self.agent = agent

        self.checkpoint_every = checkpoint_every
        self.use_wandb = use_wandb

        if checkpoint_every is not None:
            self.active = True
        else:
            self.active = False

        if self.active and not use_wandb:
            print("Currently, checkpointing only supports checkpointing with wandb")
            self.active = False

        if self.active and use_atexit:
            atexit.register(self.make_checkpoint, "final.pth")

    def step(self, timestep: int) -> dict:
        """
        Decides whether or not to checkpoint, called every timestep

        Args:
            timestep (int): the current timestep

        Returns:
            dict: any information to log
        """
        log_dict = {}
        log_dict["checkpointed"] = False

        if not self.active:
            return log_dict

        if timestep % self.checkpoint_every == 0:
            pth = os.path.join(wandb.run.dir, "checkpoints")
            Path(pth).mkdir(parents=True, exist_ok=True)
            pth = os.path.join(pth, f"ckpt_{timestep}.zip")
            self.make_checkpoint(pth)

            log_dict["checkpointed"] = True

        return log_dict

    def make_checkpoint(self, pth: str) -> dict:
        """
        Creates a checkpoint at the specified location

        Args:
            pth (str): where to make the checkpoint (full path, not just folder)

        Returns:
            dict: any information to log
        """
        self.agent.save(pth)

        return {}


import sys;
sys.path.extend(['../crypto_trading'])
import pandas as pd

import os
import asyncio

from pathlib import Path

from market.tardis import Tardis, TardisType, BitMEX, instrument_info, download_book_change_tasks, download_task

from calendar import monthrange, isleap
import datetime as dt

# https://stackoverflow.com/questions/7045920/how-to-calculate-the-number-of-days-in-the-years-between-2-dates-in-python
def days_in_year(year):
    return 365 + isleap(year)

def days_in_months(year, nof_months=7):
    total_days = 0
    for m in range(1, nof_months+1):
        _, temp = monthrange(year, m)
        total_days += temp
    return (total_days)

def months_in_year(year):
    now = dt.datetime.now()
    if year == now.year:
        return (now.month - 1)
    elif year < now.year:
        return (12)
    else:
        return (None)

async def download_year (symbol, dyear = 2021, dmonths = 7, exchange = "binance-futures", dataType = "trade_bar_1m"):

    if dyear is None or dmonths is None:
        return

    start_date = f'{dyear}-01-01T00:00:00.000Z'

    if dmonths == 12:
        num_days = days_in_year(dyear)
    else:
        num_days = days_in_months(dyear, dmonths)

    task = download_task(exchange, symbol, dataType, start_date, num_days)
    # task = [download_task(exchange, s, dataType, start_date, num_days) for s in symbols]
    await asyncio.gather(task)

def make_new_name(old_name):
    try:
        _, _, ticker, year, _ = old_name.split("_")
        return (("_").join([ticker, year]))
    except:
        print (f'\nThere were problems in renaming the file "{old_name}" !')
        return (old_name)

def rename_files(folder, f_new_name):
    for count, f in enumerate(os.listdir(folder)):
        f_name, f_ext = os.path.splitext(f)

        f_name = f_new_name(f_name)

        new_name = f'{f_name}{f_ext}'
        os.rename(Path(folder).joinpath(f), Path(folder).joinpath(new_name))

def to_zorro_t6_compatible_csv(df: pd.DataFrame) -> pd.DataFrame:
    date = [int(d) for d in df.index.strftime("%Y%m%d").values]
    time = df.index.strftime("%H:%M:%S")
    df.columns = list(map(lambda x: x.lower(),  list(df.columns)))
    res = pd.DataFrame({"Date": date, "Time": time, "Open": df.open, "High": df.high, "Low": df.low, "Close": df.close, "Volume": df.volume})
    return res.reset_index(drop=True)

def extended_in(el_set, target):
    res = False
    for el in el_set:
        if el in target:
            res = True
            break
    return (res)


# -*- coding:utf-8 -*-
from sklearn.datasets import load_digits
from scipy import optimize
import numpy as np
import matplotlib.pyplot as plt
import scipy.io as sio

# 这个数据集中并没有图片，而是经过提取得到的手写数字特征和标记，就免去了我们的提取数据的麻烦，
# 但是在实际的应用中是需要我们对图片中的数据进行提取
digits = load_digits()  # 1797*64 共有1797个数据，0 ~9中的某个，均为8*8像素，故有64列，每列代表其像素的灰度值（0~255之间）
print(digits.keys())  # 属性查看
data = digits.data  # 训练集
target = digits.target  # 标记，特征向量对应的标记，每一个元素都是自然数是0-9的数字，label
print(data.shape)
print(target.shape)
print('the images 15 is', target[15])
# 随机选50个数据进行识别
classes = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
num_classes = len(classes)
samples_per_class = 5  # 每个类别采样个数

for y, cla in enumerate(classes):  # 既要遍历索引又要遍历元素，对列表元素位置和元素进行循环，y表示元素位置（0,num_class），cla元素本身
    idxs = np.flatnonzero(target == y)  # 找出标签中y类的位置 (该函数输入一个矩阵，返回扁平化后矩阵中非零元素的位置（index）,这里用来返回某个特定元素的位置)
    idxs = np.random.choice(idxs, samples_per_class, replace=False)  # # 从上述找到的位置中随机选出我们所需的5个样本，没有重复出现的数值
    for i, idx in enumerate(idxs):  # 对所选的样本的位置和样本所对应的图片在训练集中的位置进行循环
        plt_idx = i * num_classes + y + 1  # 在子图中所占位置的计算
        plt.subplot(samples_per_class, num_classes, plt_idx)  # 说明要画的子图的编号，subplot绘制多个子图
        plt.imshow(digits.images[idx].astype('uint8'))  # 画图,为图像提供了特殊的数据类型uint8(8位无符号整数),存入一个8位矩阵
        plt.axis('off')
        if i == 0:
            plt.title(cla)
plt.show()

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def out(x, w):
    return sigmoid(np.dot(x, w))

# 优化算法，稍微改写了cost函数（代价函数）和gradient函数,分别为f(),gradf()
def f(params, *args):
    X_train, y_train, reg = args  # reg相当于lamba ，正则化参数
    m, n = X_train.shape
    J = 0
    theta = params.reshape((n, 1))
    h = out(X_train, theta)
    theta_1 = theta[1:, :]
    J = -1 * np.sum(y_train * np.log(h) + (1 - y_train) * np.log((1 - h))) / m + 0.5 * reg * theta_1.T.dot(theta_1) / m
    return J

def gradf(params, *args):
    X_train, y_train, reg = args
    m, n = X_train.shape
    theta = params.reshape(-1, 1)
    h = out(X_train, theta)
    theta_1 = theta[1:, :]
    grad = X_train.T.dot((h - y_train)) / m
    grad[1:, :] += reg * theta_1 / m
    g = grad.ravel()
    return g

def oneVsAll(x, y, num_class, reg):
    m, n = x.shape
    thetas = np.zeros((n, num_class))  # 调用onevsall函数时，此时thetas=65*10
    for i in range(num_class):
        params = np.zeros((x.shape[1], 1)).ravel()
        args = (x, y == i, reg)
        # 使用高级优化算法（fmin_cg）去训练
        res = optimize.fmin_cg(f, x0=params, fprime=gradf, args=args, maxiter=500)
        thetas[:, i] = res  # 就是取所有行的第几个数据
    return thetas

X = data
y = target
y = y.reshape((-1, 1))
X_mean = np.mean(X, axis=0)  # 对各列求均值，返回 1* n 矩阵 # 1*64
X -= X_mean
m = X.shape[0]
X = np.hstack((np.ones((m, 1)), X))
# hstack会将多个value(value_one, value_two)的相同维度的数值组合在一起，并以同value同样的数据结构返回numpy数组
print(X.shape)
thetas = oneVsAll(X, y, 10, 1.0)
print(thetas.shape)

def predict(x, thetas):
    h = out(x, thetas)
    print(h.shape)
    a = sigmoid(h)
    print(a.shape)
    pred = np.argmax(a, axis=1)  # 找出每行的最大值（即概率）,返回列号即预测的数字。返回后，实际组成的数列是按照扁平的形式排列
    return pred

y_pred = predict(X, thetas)
print(y_pred.shape)
print("train accuracy is :", np.mean(y.ravel() == y_pred))  # 首先这两个矩阵进行比较，返回一个bool矩阵，当对应的数字一致，返回True(或1)，不同False（0）
                                                              #  形成bool矩阵后，通过mean函数，求[0,1,1,0,1,0,1......]的均值，即准确率
# 随机选出几个图片，测试一下
m, n = data.shape
example_size = 10
example_index = np.random.choice(m, (example_size))  # 随机选出几个图片
print(example_index)
for i, idx in enumerate(example_index):
    print("%d example is number %d,we predict it as %d" % (i, target[idx], predict(X[idx, :].reshape(1, -1), thetas)))

# 从mat文件导入数据，这里采用前向神经网络，不再用原有的数据集
weight = sio.loadmat('ex3weights')  # 5000*400
Theta1 = weight["Theta1"]
Theta2 = weight["Theta2"]

data = sio.loadmat('ex3data1')
X = data["X"]
y = data["y"]
print(X.shape)
print(y.shape)

m = X.shape[0]
X1 = np.hstack((np.ones((m, 1)), X))
print(X1.shape)

# 模拟实现前向传播算法
def nn_predict(x, theta1, theta2):
    m = x.shape[0]
    a1 = sigmoid(x.dot(theta1.T))
    a1 = np.hstack((np.ones((m, 1)), a1))
    a2 = sigmoid(a1.dot(theta2.T))
    print(a2.shape)
    pred = np.argmax(a2, axis=1) + 1  # 这里的数字是1-10，而不是0-9，这里的a2相当于h（x）
    return pred

pred = nn_predict(X1, Theta1, Theta2)
print(pred[-10:])
print(y[-10:])
print("train accuracy is :", np.mean(y == pred.reshape(-1, 1)) * 100)

# 随机选择样本，测试一下
m, n = X1.shape
example_size = 10
example_index = np.random.choice(m, example_size)
print(example_index)
for i, idx in enumerate(example_index):
    print("%d example is number %d,we predict it as %d" % (i, y[idx], nn_predict(X1[idx, :].reshape(1, -1), Theta1, Theta2)))


'''
Code to plot graph using data file

Alexandre Mascarenhas
'''
import json
import shutil
import itertools
import operator
import random
import numpy as np
import pandas as pd
import math
import matplotlib.pyplot as plt
import datetime
import os
import csv
import sys

cDate = datetime.datetime.now()
year = cDate.year
month = cDate.month
day = cDate.day
hour = cDate.hour
minute = cDate.minute

def writeTXT(data, name, path):
    line = f"- {name}= {data[0]:.4f}({data[1]:.4f})\n"
    print(line)
    f = open(f"{path}/results.txt","a")
    f.write(line)
    f.close()

def bestErrorBeforeChange(path, std=1):
    df = pd.read_csv(f"{path}/data.csv")
    data = [[] for i in range( len(pd.unique(df["run"])) )]
    for i in range(len(pd.unique(df["run"])) ): # Get the number of runs
        data[i] = df[df["run"] == i+1]
        data[i] = data[i].drop_duplicates(subset=["env"], keep="last")[["gen", "nevals", "swarmId", "bestError", "env"]]
        data[i].reset_index(inplace=True)
        del data[i]["index"]
        #data[i].to_csv(f"{path}/run{i+1}.csv", index = True)

    NCHANGES = len(data[0]["env"])
    NRUNS = len(data)
    luffy = [0 for _ in range(NRUNS)]

    for j in range(NRUNS):
        luffy[j] = np.sum(data[j]['bestError'])/NCHANGES

    bobc = [np.mean(luffy), np.std(luffy)]
    if(std):
        return bobc
    else:
        return bobc[0]

def offlineError(path, std=1):
    df = pd.read_csv(f"{path}/data.csv")
    data = [[] for i in range( len(pd.unique(df["run"])) )]
    for i in range(len(pd.unique(df["run"])) ): # Get the number of runs
        data[i] = df[df["run"] == i+1]
        data[i] = data[i].drop_duplicates(subset=["gen"], keep="last")[["gen", "nevals", "swarmId", "bestError", "env"]]
        data[i].reset_index(inplace=True)
        del data[i]["index"]
        #data[i].to_csv(f"{path}/run{i+1}.csv", index = True)

    NGEN = len(data[0]["gen"])
    NRUNS = len(data)
    luffy = [0 for _ in range(NRUNS)]

    for j in range(NRUNS):
        luffy[j] = np.sum(data[j]['bestError'])/NGEN

    eo = [np.mean(luffy), np.std(luffy)]

    if(std):
        return eo
    else:
        return eo[0]

def main():
    # reading the parameters from the config file
    try:
        path = sys.argv[1]
    except IndexError:
        with open("./config.ini") as f:
            parameters = json.loads(f.read())
        debug = parameters["DEBUG"]
        if(debug):
            print("Parameters:")
            print(parameters)
        #path = f"{parameters['PATH']}/{parameters['ALGORITHM']}/{sys.argv[1]}/{sys.argv[2]}"
    try:
        std = int(sys.argv[2])
    except IndexError:
        std = 0

    eo = offlineError(path, std)
    bobc = bestErrorBeforeChange(path, std)

    if(std):
        print(f"[Offline Error]       Eo:  {eo[0]:.4f}({eo[1]:.4f})")
        print(f"[Best before change]  Ebc: {bobc[0]:.4f}({bobc[1]:.4f})")
    else:
        print(f"[Offline Error]       Eo:  {eo:.4f}")
        print(f"[Best before change]  Ebc: {bobc:.4f}")

if __name__ == "__main__":
    main()


"""
Created on Dec 13, 2018
@author: Yuedong Chen
"""

from data import create_dataloader
from model import create_model
from visualizer import Visualizer
import copy

class BaseSolver(object):
    """docstring for BaseSolver"""
    def __init__(self):
        super(BaseSolver, self).__init__()

    def initialize(self, opt):
        self.opt = opt
        self.visual = Visualizer()
        self.visual.initialize(self.opt)

        self.CK_FACIAL_EXPRESSION = ['anger', 'contempt', 'disgust', 'fear', 'happy', 'sadness', 'surprise']
        self.OC_FACIAL_EXPRESSION = ['Anger', 'Disgust', 'Fear', 'Happiness', 'Sadness', 'Surprise']

    def run_solver(self):
        if self.opt.mode == "train":
            self.train_networks()
        else:
            self.test_networks(self.opt)

    def train_networks(self):
        # init train setting
        self.init_train_setting()

        # for every epoch
        for epoch in range(self.opt.epoch_count, self.epoch_len + 1):
            # train network
            self.train_epoch(epoch)
            # update learning rate
            self.cur_lr = self.train_model.update_learning_rate()
            # save checkpoint if needed
            if epoch % self.opt.save_epoch_freq == 0:
                self.train_model.save_ckpt(epoch)

        # save the last epoch
        self.train_model.save_ckpt(self.epoch_len)

    def init_train_setting(self):
        self.train_dataset = create_dataloader(self.opt)
        self.train_model = create_model(self.opt)
        if 'CK' in self.train_dataset.name() or 'Affect' in self.train_dataset.name():
            self.FACIAL_EXPRESSION = self.CK_FACIAL_EXPRESSION
        else:
            self.FACIAL_EXPRESSION = self.OC_FACIAL_EXPRESSION

        self.train_total_steps = 0
        self.epoch_len = self.opt.niter + self.opt.niter_decay
        self.cur_lr = self.opt.lr

    def train_epoch(self, epoch):
        pass

    def test_networks(self, opt):
        pass

    def init_test_setting(self, opt):
        # hard code some params
        opt.visdom_display_id = 0
        opt.serial_batches = True

        dataset = create_dataloader(opt)
        model = create_model(opt)
        model.set_eval()
        return dataset, model


import logging
from datetime import datetime, timedelta

from aiogram import Bot, Dispatcher, executor, types
from aiogram.contrib.fsm_storage.memory import MemoryStorage
from aiogram.dispatcher import FSMContext
from aiogram.types import ReplyKeyboardRemove

from config import API_TOKEN, EMAIL_DOMAIN
from custom_funcs import set_timer
import keyboard as kb
from onesec_api import Mailbox
import asyncio
from bs4 import BeautifulSoup

logging.basicConfig(level=logging.INFO)
bot = Bot(token=API_TOKEN)
dp = Dispatcher(bot, storage=MemoryStorage())

@dp.message_handler(content_types=['text'], state='*')
async def texthandler(m: types.Message, state: FSMContext):
    if m.text != '✉ Получить почту':
        await m.answer(
            f'Приветствую вас, {m.from_user.mention}\nЭтот бот создан для быстрого получения временной почты.\nНажмите '
            f'на кнопу ниже 👇',
            reply_markup=kb.menu)
    elif m.text == '✉ Получить почту':
        has_mail = await state.get_data()
        try:
            if has_mail['has_mail']:
                await m.answer(f"✉ У вас уже есть активная почта - {has_mail['email']}\nЖдите окончания лимитов...")
        except Exception:
            mail = Mailbox('')
            email = f'{mail.mailbox_}@{EMAIL_DOMAIN}'
            await m.answer(
                f'📫 Вот ваша почта: {email}\nОтправляйте письмо.\n'
                f'Почта проверяется автоматически, каждые 5 секунд, '
                f'если придет новое письмо, мы вас об этом оповестим!\n'
                f'На 1 почту можно получить только - 5 писем за 3 '
                f'минуты времени.\n\n '
                f'@mal1kov', reply_markup=ReplyKeyboardRemove())
            await state.update_data(has_mail=True)
            await state.update_data(email=email)

            last_length = 0
            now = datetime.now()
            end_time = now + timedelta(minutes=3)
            while True:
                now = datetime.now()
                timed_out = await set_timer(now, end_time)
                if last_length >= 5 or timed_out:
                    await m.answer('Лимиты этой почты закончились. ⌛ \nСпасибо за то, что вы с нами! 👍 \n\nСказать '
                                   'спасибо 👉 Донат '
                                   '\n\nНажмите на кнопу ниже, чтобы получить новую временную '
                                   'почту 👇', reply_markup=kb.menu, parse_mode='HTML')
                    await state.finish()
                    break

                filtered_mail = mail.filtered_mail()
                if isinstance(filtered_mail, list):
                    if last_length == len(filtered_mail):
                        pass
                    else:
                        last_length = len(filtered_mail)

                        new_mail = mail.mail_jobs('read', filtered_mail[0])

                        js = new_mail.json()
                        sender = js['from']
                        theme = js['subject']
                        mes = js['body']
                        cleantext = BeautifulSoup(mes, "html.parser")
                        cleantext = "\n".join(item.strip() for item in cleantext.find_all(text=True))
                        await m.answer(
                            f'📩 Новое письмо:\n'
                            f'От: {sender}\n'
                            f'Тема: {theme}\n'
                            f'Сообщение: \n\n{cleantext}', parse_mode='HTML')

                await asyncio.sleep(5)

if __name__ == '__main__':
    executor.start_polling(dp, skip_updates=True)


from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import *
import BackGround
import sqlite3

class Ui_QueryWindow(object):
    def setupUi(self, QueryWindow):
        QueryWindow.setObjectName("QueryWindow")
        QueryWindow.resize(911, 800)
        self.centralwidget = QtWidgets.QWidget(QueryWindow)
        self.centralwidget.setObjectName("centralwidget")
        self.titleFind = QtWidgets.QLabel(self.centralwidget)
        self.titleFind.setGeometry(QtCore.QRect(-1, 10, 901, 100))
        font = QtGui.QFont()
        font.setFamily("Consolas")
        font.setPointSize(28)
        font.setBold(True)
        font.setWeight(75)
        self.titleFind.setFont(font)
        self.titleFind.setStyleSheet("color: rgb(255, 255, 255);")
        self.titleFind.setAlignment(QtCore.Qt.AlignCenter)
        self.titleFind.setObjectName("titleFind")
        self.Background = QtWidgets.QLabel(self.centralwidget)
        self.Background.setGeometry(QtCore.QRect(0, 0, 900, 770))
        self.Background.setStyleSheet("background-image: url(:/Main/background.png);")
        self.Background.setText("")
        self.Background.setObjectName("Background")
        self.LabelStudentID = QtWidgets.QLabel(self.centralwidget)
        self.LabelStudentID.setGeometry(QtCore.QRect(50, 90, 160, 50))
        font = QtGui.QFont()
        font.setFamily("Consolas")
        font.setPointSize(16)
        font.setBold(True)
        font.setWeight(75)
        self.LabelStudentID.setFont(font)
        self.LabelStudentID.setStyleSheet("color: rgb(255, 255, 255);")
        self.LabelStudentID.setObjectName("LabelStudentID")
        self.TextQuery = QtWidgets.QTextEdit(self.centralwidget)
        self.TextQuery.setGeometry(QtCore.QRect(50, 150, 601, 151))
        font = QtGui.QFont()
        font.setFamily("Consolas")
        font.setPointSize(16)
        self.TextQuery.setFont(font)
        self.TextQuery.setObjectName("TextQuery")
        self.ButtonExecute = QtWidgets.QPushButton(self.centralwidget)
        self.ButtonExecute.setGeometry(QtCore.QRect(700, 200, 150, 50))
        font = QtGui.QFont()
        font.setFamily("Consolas")
        font.setPointSize(16)
        font.setBold(True)
        font.setWeight(75)
        self.ButtonExecute.setFont(font)
        self.ButtonExecute.setObjectName("ButtonExecute")
        self.ButtonQuit = QtWidgets.QPushButton(self.centralwidget)
        self.ButtonQuit.setGeometry(QtCore.QRect(200, 680, 150, 50))
        font = QtGui.QFont()
        font.setFamily("Consolas")
        font.setPointSize(16)
        font.setBold(True)
        font.setWeight(75)
        self.ButtonQuit.setFont(font)
        self.ButtonQuit.setObjectName("ButtonQuit")

        self.ButtonClear = QtWidgets.QPushButton(self.centralwidget)
        self.ButtonClear.setGeometry(QtCore.QRect(550, 680, 150, 50))
        font = QtGui.QFont()
        font.setFamily("Consolas")
        font.setPointSize(16)
        font.setBold(True)
        font.setWeight(75)
        self.ButtonClear.setFont(font)
        self.ButtonClear.setObjectName("ButtonClear")

        self.TableData = QtWidgets.QTableWidget(self.centralwidget)
        self.TableData.setGeometry(QtCore.QRect(50, 320, 800, 320))
        self.TableData.setObjectName("TableData")
        self.TableData.setColumnCount(0)
        self.TableData.setRowCount(0)
        self.Background.raise_()
        self.titleFind.raise_()
        self.LabelStudentID.raise_()
        self.TextQuery.raise_()
        self.ButtonExecute.raise_()
        self.ButtonQuit.raise_()
        self.TableData.raise_()
        self.ButtonClear.raise_()
        QueryWindow.setCentralWidget(self.centralwidget)
        self.menubar = QtWidgets.QMenuBar(QueryWindow)
        self.menubar.setGeometry(QtCore.QRect(0, 0, 911, 26))
        self.menubar.setObjectName("menubar")
        QueryWindow.setMenuBar(self.menubar)
        self.statusbar = QtWidgets.QStatusBar(QueryWindow)
        self.statusbar.setObjectName("statusbar")
        QueryWindow.setStatusBar(self.statusbar)

        self.retranslateUi(QueryWindow)
        QtCore.QMetaObject.connectSlotsByName(QueryWindow)

        self.ButtonQuit.clicked.connect(lambda:self.close(QueryWindow))
        self.ButtonExecute.clicked.connect(self.query_execute)
        self.ButtonClear.clicked.connect(lambda: self.clear(self.TableData))
        # ? set title
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap("icon.png"), QtGui.QIcon.Selected, QtGui.QIcon.On)
        QueryWindow.setWindowIcon(icon)

    def close(self,QueryWindow):
        mess = QMessageBox()
        mess.setIcon(QMessageBox.Warning)
        mess.setWindowTitle("Window Close")
        mess.setWindowIcon(QtGui.QIcon('warning.png'))
        mess.setStyleSheet("QLabel{min-height:30 px; font-size: 24px;}")
        mess.setText("Are you sure you want to close the windodw ?")
        mess.setStandardButtons(QMessageBox.Yes | QMessageBox.No )
        question = mess.exec_()
        if question == QMessageBox.Yes :
            QueryWindow.hide()

    def query_execute(self):
        try:
            connection = sqlite3.connect('StudentManagement.db')
            cur = connection.cursor()
            query = self.TextQuery.toPlainText()
            res = cur.execute(query)

            self.TableData.horizontalHeader().setStretchLastSection(True)
            self.TableData.horizontalHeader().setSectionResizeMode(QtWidgets.QHeaderView.Stretch)

            columnName =[]
            for column in res.description:
                columnName.append(column[0])

    # ?Set rowcount vs columnCount
            for row_number, row_data in enumerate(res):
                self.TableData.setRowCount(row_number)
                self.TableData.setColumnCount(len(row_data))
                self.TableData.insertRow(row_number)

                # St columns name
                self.TableData.setHorizontalHeaderLabels(columnName)

                for column_number, data in enumerate(row_data):
                    self.TableData.setItem(row_number, column_number, QtWidgets.QTableWidgetItem(str(data)))
        except:
            mess = QMessageBox()
            mess.setIcon(QMessageBox.Critical)
            mess.setWindowIcon(QtGui.QIcon('cancel.png'))
            mess.setStyleSheet("QLabel{min-height:30 px; font-size: 24px;}")
            mess.setWindowIcon(QtGui.QIcon('cancel.png'))
            mess.setWindowTitle("Error")
            mess.setText(" Error ")
            mess.exec_()

    def clear(self, TableData):
        TableData.horizontalHeader().hide()
        while TableData.rowCount() > 0:
            TableData.removeRow(0)
        pass

    def retranslateUi(self, QueryWindow):
        _translate = QtCore.QCoreApplication.translate
        QueryWindow.setWindowTitle(_translate("QueryWindow", "Query SQL"))
        self.titleFind.setText(_translate("QueryWindow", "Query"))
        self.Background.setWhatsThis(_translate("QueryWindow", "

"))
        self.LabelStudentID.setText(_translate("QueryWindow", " Query"))
        self.ButtonExecute.setText(_translate("QueryWindow", "Execute"))
        self.ButtonQuit.setText(_translate("QueryWindow", "Quit"))
        self.ButtonClear.setText(_translate("QueryWindow", "Clear"))

if __name__ == "__main__":
    import sys
    app = QtWidgets.QApplication(sys.argv)
    QueryWindow = QtWidgets.QMainWindow()
    ui = Ui_QueryWindow()
    ui.setupUi(QueryWindow)
    QueryWindow.show()
    sys.exit(app.exec_())


import time
import pytest
from fastapi.testclient import TestClient
from productClass import Product
from main import app

client = TestClient(app)

test_product = Product(
    product_name="pytest",
    description="new test product",
    quantity_available=4,
    contact_info="mail.com"

)

test_product_json_data = {
    "id": f"{test_product.id}",
    "product_name": f"{test_product.product_name}",
    "description": f"{test_product.description}",
    "quantity_available": test_product.quantity_available,
    "upload_time": test_product.upload_time,
    "contact_info": f"{test_product.contact_info}"
}

def test_new_product():
    assert test_product.id

def test_get_home():
    response = client.get("/")
    assert response.status_code == 200

@pytest.mark.asyncio
async def test_post_product():
    response = client.post("http://Treasurejunk:8001/product", json=test_product_json_data)
    assert response.status_code == 200
    assert response.json() == test_product_json_data
    time.sleep(1)

@pytest.mark.asyncio
async def test_get_product_by_name():
    response = client.get(f"http://Treasurejunk:8001/product/name/{test_product.product_name}")
    assert response.status_code == 200

@pytest.mark.asyncio
async def test_get_product_by_id():
    response = client.get(f"http://Treasurejunk:8001/product/id/{test_product.id}")
    assert response.status_code == 200

@pytest.mark.asyncio
async def test_get_all_products():
    response = client.get("/products")
    assert response.status_code == 200
    assert test_product_json_data in response.json()

@pytest.mark.asyncio
async def test_update_product_description():
    response = client.put(f"http://Treasurejunk:8001/product/update/{test_product.product_name}/description?i_product_description=Updated")
    assert response.status_code == 200
    assert response.json() == f"{test_product.product_name} description updated successfully"

@pytest.mark.asyncio
async def test_update_product_quantity():
    response = client.put(f"http://Treasurejunk:8001/product/update/{test_product.product_name}/quantity?i_product_quantity=10")
    assert response.status_code == 200
    assert response.json() == f"{test_product.product_name} quantity updated successfully"

@pytest.mark.asyncio
async def test_delete_product():
    response = client.delete(f"http://Treasurejunk:8001/product/remove/{test_product.product_name}")
    assert response.status_code == 200
    assert response.json() == f"{test_product.product_name} removed successfully"


import re
import glob
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy.ndimage import gaussian_filter1d

plt.rcParams.update({
    'text.usetex': True,
    'text.latex.preamble': r'\usepackage{siunitx}'
})

plt.style.use('ggplot')

# importing files into dataframes
files = glob.glob('data/T_15*')
voltages = [int(re.search('VR_(.+?)mV', file).group(1)) for file in files]
dfs = [pd.read_csv(file, sep='\s+', skiprows=(0, 1), index_col=0, decimal=',')
       for file in files]

# relabeling columns and index
for df in dfs:
    df.columns = ['I', 'U']
    df.index.names = ['Time']
    df.loc[:, 'U'] *= 2

# plot of current in terms of drive voltage
fig, ax = plt.subplots()

ax.set(xlim=(dfs[0].iloc[0, 1], dfs[0].iloc[-1, 1]),
       xlabel=r'$ U_a $ (\si{\volt})',
       ylabel=r'$ I_s $ (\si{\ampere})')

minima_dfs = []
for i in range(len(dfs)):
    x = np.array(gaussian_filter1d(dfs[i].loc[:, 'U'], sigma=3))
    y = np.array(gaussian_filter1d(dfs[i].loc[:, 'I'], sigma=3))

    minima_indexes = (np.diff(np.sign(np.diff(y))) > 0).nonzero()[0][-2:] + 1
    minima_df = dfs[i].iloc[minima_indexes, :]
    minima_dfs.append(minima_df)

    x_minima = minima_df.loc[:, 'U']
    y_minima = minima_df.loc[:, 'I']

    color = ax.plot(x_minima, y_minima, 'o', ms=3.0)[-1].get_color()
    ax.plot(x, y, color=color, label=rf'$ U_s = {voltages[i]} \si{{\milli\volt}} $')

# sorting legend labels by stop voltage value
handles, labels = ax.get_legend_handles_labels()

handles_labels_voltages = [[h, l, v] for h, l, v in zip(handles, labels, voltages)]
handles_labels_voltages.sort(key=lambda list: list[2])

handles = [hv[0] for hv in handles_labels_voltages]
labels = [hv[1] for hv in handles_labels_voltages]

ax.legend(handles, labels)

# plot of current minima in terms of stop voltages
fig_min, ax_min = plt.subplots()

ax_min.set(xlabel=r'$ U_s $ (\si{\volt})',
           ylabel=r'$ I_s $ (\si{\ampere})')

current_mins = [[df.iloc[i, 0] for df in minima_dfs] for i in range(2)]
voltage_mins = [[df.iloc[i, 1] for df in minima_dfs] for i in range(2)]

final_dfs = []
for current_min, voltage_min in zip(current_mins, voltage_mins):
    data = [[v, i, u] for v, i, u in zip(voltages, current_min, voltage_min)]
    sorted_data = sorted(data, key=lambda list: list[0])

    x = [data[0] for data in sorted_data]
    y = [data[1] for data in sorted_data]
    u = [data[2] for data in sorted_data]

    ax_min.plot(x, y, 'o-', ms=3.0)

    df = pd.DataFrame(np.array([x, y, u]).T, columns=('U_stop', 'I', 'U_drive'))
    final_dfs.append(df)

# export data for each minimum as csv
filenames = ('first_minimum.csv', 'second_minimum.csv')
for df, filename in zip(final_dfs, filenames):
    df.to_csv(filename, decimal=',', index=None)

fig.savefig('plots/constant_temp.png', dpi=300)
fig_min.savefig('plots/current_minima.png', dpi=300)
plt.show()


# -*- coding = utf-8 -*-
# @time:2020/9/22 23:51
# Author:TC
# @File:网格搜索+xgboost.py
# @Software:PyCharm

from sklearn.datasets import load_iris
from xgboost import plot_importance
from matplotlib import pyplot as plt
from sklearn.metrics import accuracy_score
import xgboost as xgb
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import roc_auc_score

iris = load_iris()
X,y = iris.data,iris.target
col = iris.target_names
train_x, valid_x, train_y, valid_y = train_test_split(X, y, test_size=0.3, random_state=1)   # 分训练集和验证集
parameters = {
              'max_depth': [5, 10, 15, 20, 25],
              'learning_rate': [0.01, 0.02, 0.05, 0.1, 0.15],
              'n_estimators': [500, 1000, 2000, 3000, 5000],
              'min_child_weight': [0, 2, 5, 10, 20],
              'max_delta_step': [0, 0.2, 0.6, 1, 2],
              'subsample': [0.6, 0.7, 0.8, 0.85, 0.95],
              'colsample_bytree': [0.5, 0.6, 0.7, 0.8, 0.9],
              'reg_alpha': [0, 0.25, 0.5, 0.75, 1],
              'reg_lambda': [0.2, 0.4, 0.6, 0.8, 1],
              'scale_pos_weight': [0.2, 0.4, 0.6, 0.8, 1]

}

xlf = xgb.XGBClassifier(max_depth=10,
            learning_rate=0.01,
            n_estimators=2000,
            silent=True,
            objective='multi:softmax',
            num_class=3 ,
            nthread=-1,
            gamma=0,
            min_child_weight=1,
            max_delta_step=0,
            subsample=0.85,
            colsample_bytree=0.7,
            colsample_bylevel=1,
            reg_alpha=0,
            reg_lambda=1,
            scale_pos_weight=1,
            seed=0,
            missing=None)

gs = GridSearchCV(xlf, param_grid=parameters, scoring='accuracy', cv=3)
gs.fit(train_x, train_y)

print("Best score: %0.3f" % gs.best_score_)
print("Best parameters set: %s" % gs.best_params_ )


# -*- coding: utf-8 -*-

import numpy as np
from collections import Counter

def play_round(player, board, s, win_size):
    print('\n--------------------------------')
    print(f"Turn {turn} :: {symbol[player]}'s move")
    print('--------------------------------\n')
    while True:
        print_board(board, s)
        try:
            move = int(input("Your move:")) - 1
            if move not in range(0, s ** 2):
                raise ValueError
            if int(board[move // s, move % s]) != 0:
                raise LookupError
        except ValueError:
            print(f'Your input was invalid, please try again. Must be an integer within range of 1 and {s ** 2}')
        except LookupError:
            print('That space was already picked, please choose another number')
        else:
            board[move // s, move % s] = player
            win_check_temp = win_check(player, win_size, get_rcd(s, board, win_size))
            if win_check_temp == 'Win':
                print_end(f'{symbol[player]} wins!', board, s)
                return False
            elif win_check_temp == 'Tie':
                print_end('Tied game, no more moves!', board, s)
                return False
            else:
                return True

def get_rcd(s, board, win_size):
    """
    Parameters
    ----------
    s : int
         number that acts as the main parameter for calculating the two axes of the game board.
    board : numpy array
        Allows for parsing possible win-cases
    win_size : int
        win condition, ie how many contiguous spaces a player needs to win.

    Returns
    -------
    rcd : list
    Row Columns Diagonals, list of all possible axes that could meet win condition
    """

    # rows, columns, diagonals: for win condition
    rcd = []
    for ii in range(s):
        rcd.append(board[:, ii].tolist())
        rcd.append(board[ii].tolist())
        for i in range(s):
            rcd.append(np.diag(board, k=i).tolist())
            rcd.append(np.diag(np.fliplr(board), k=i).tolist())
            if i > 0:
                rcd.append(np.diag(board, k=-i).tolist())
                rcd.append(np.diag(np.fliplr(board), k=-i).tolist())
    for lst in rcd:
        if len(lst) > win_size:
            for i, item in enumerate(lst):
                if i + win_size <= s + 1:
                    rcd.append(lst[i:i + win_size])

    # list comprehension to omit any axis list that is not equal to win condition(win_size)
    rcd[:] = [lst for lst in rcd if not len(lst) < win_size] and [lst for lst in rcd if not len(lst) > win_size]

    return rcd

def win_check(player, win_size, rcd):
    tie_check = 0
    for lst in rcd:
        temp_collection = Counter(lst)
        if temp_collection.get(player) == win_size:
            return 'Win'
        elif temp_collection.get(0) is None:
            tie_check += 0
        else:
            tie_check += temp_collection.get(0)
    if tie_check == 0:
        return 'Tie'
    return 'None'

def print_board(board, s):
    """
    Parameters
    ----------
    board : numpy array
    s : int
        Board size, (s*s = board dimensions)

    Returns
    -------
    terminal printout of current game board.
    """
    padding = len(str(s ** 2))
    counter = 0
    for row in range(s):
        temp_string = "|"
        for i in board[row]:
            padding_mod = 1
            temp_i = str(int(i))
            temp_val = ''
            counter += 1
            if temp_i == '0':
                temp_val = str(counter)
            if temp_i == '1':
                temp_val = 'X'
            if temp_i == '2':
                temp_val = 'O'
            while len(temp_val) != padding:
                if padding_mod % 2 == 1:
                    temp_val = ' ' + temp_val
                    padding_mod += 1
                else:
                    temp_val = temp_val + ' '
                    padding_mod += 1
            temp_string += temp_val + "|"
        print(temp_string)

def print_end(message, board, s):
    """
    Parameters
    ----------
    message : string
        provided by function call, either game win message or tie.
    board : numpy array
    s : int

    Returns
    -------
    end message depending on board state, win vs tie.
    """
    print('\n')
    print_board(board, s)
    print('\n--------------------------------')
    print(message)
    print('--------------------------------\n')

def main():
    print(
        '''
    Welcome to advanced tic-tac-toe!
    You will be asked to select two parameters for the game: board size and win condition.
    For board size: Input a single number that will act as both the X and Y axis of the board.
    ~~This number should not be smaller than 3!~~
    For win condition: Input a single number that represents how many contiguous spaces in a straight line you need to win.
    ~~This number should be no smaller than 2 and no larger than your selected board size!~~
    '''
    )

    while True:
        # Loop to establish board size. s*s = np.zeros array. Must be > 3
        try:
            s = int(input('Please enter board size: '))
            if s < 3:
                raise ValueError
            break
        except ValueError:
            print('Your input was invalid or too small, please use an integer')
    while True:
        # Loop to establish a valid win condition. Must be <= s and >= 1
        try:
            win_size = int(input('Please enter win condition: '))
            if win_size > s or win_size <= 1:
                raise ValueError
            break
        except ValueError:
            print(
                "Your input was invalid, please use an integer. "
                f"Make sure it's not larger than your selected board size: {s}")

    board = np.zeros((s, s))
    game_state = True

    # global vars
    global turn
    global symbol

    turn = 0
    symbol = {1: 'X', 2: 'O'}

    while game_state is True:
        turn += 1
        player = (turn % 2) + 1
        game_state = play_round(player, board, s, win_size)

game = True
game_number = 1

while game is True:
    print(f'Starting Game {game_number}!')
    main()
    while True:
        try:
            game_bool = input('Game over. Would you like to play again? Y/N: ')
            if game_bool.upper() in ['Y', 'YES']:
                game_number += 1
                game = True
                break
            elif game_bool.upper() in ['N', 'NO']:
                game = False
                break
            else:
                raise ValueError
        except ValueError:
            print('Invalid input, please type either Y for Yes, or N for No.')


import arxiv
# max_results = float('inf'),
search = arxiv.Search(
  query = "Word Embedding in Bangla",
  id_list = [],
  max_results = 10,
  sort_by = arxiv.SortCriterion.Relevance,
  sort_order = arxiv.SortOrder.Descending
)
for result in search.results():
    print(result.title)
    result.download_pdf()


import logging
import subprocess
import os
import time
import shlex
from ..config import sim_config

class Sim():
	def __init__(self, port, time_till_timeout = sim_config.time_till_timeout):
		'''
			A Sim object starts and holds the donkeysim subprocess.
			It must be pinged every time_till_timeout seconds with self.keep_alive_ping() or self.is_timeout() will return True

			WARNING: IF THE PORT IS UNAVAILABLE NO ERROR IS THROWN AND A DONKEYSIM SUBPROCESS WILL START WITH UNKNOWN PORT

			Args:
				port ([int]): port number the sim will listen to. Must be available (use the PortHandler class)
				time_till_timeout (int, optional): Time until timeout without pings. Defaults to sim_config.time_till_timeout.
		'''
		self.port = int(port)
		self.SimLogger = logging.getLogger(f"SimProcess {self.port}")
		self.SimLogger.setLevel(logging.DEBUG)
		self.start_time = time.time()
		self.last_ping_time = time.time()
		self.time_till_timeout = time_till_timeout
		self.process = self.start_sim_proc()
		self.is_in_use = True

	def start_sim_proc(self):
		'''
			Starts the donkeysim subprocess using the self.port argument and returns the subprocess instance
		'''
		self.SimLogger.debug(f"Launching sim at port {self.port}")
		cmd = os.environ["SIM_PATH"] + " --port " + str(self.port)
		print(shlex.split(cmd))
		proc = subprocess.Popen(shlex.split(cmd))
		self.SimLogger.debug(f"Launched sim at port {self.port}")
		return proc

	def ping(self):
		'''
			The Sim instance must be pinged at least once every self.time_till_timeout seconds to stay alive.
		'''
		self.SimLogger.debug(f"Pinging sim at port {self.port}. Last ping time = {self.last_ping_time}")
		self.last_ping_time = time.time()

	def is_timeout(self):
		'''
			Checks if the process has been pinged to stay alive less that self.time_till_timeout seconds ago
		'''
		self.SimLogger.debug(f"Checking timeout for sim {self.port}")
		if (time.time() - self.last_ping_time) > self.time_till_timeout:
			self.SimLogger.debug(f"Sim found to be timed out with current time : {time.time()} and last_ping_time = {self.last_ping_time}")
			return True
		self.SimLogger.debug(f"not timed out")
		return False

	def is_alive(self):
		'''
			Returns True if the donkeysim process is still running.
		'''
		return (self.process.poll() == None)

	def kill(self):
		'''
			Kill the donkeysim subprocess and set self.process to None.
		'''
		self.SimLogger.debug(f"Killing {self}")
		self.process.kill()
		self.process = None

	def safety_check_port_arg():
		pass
		# TODO: Inquire as to the safety of this

	def __str__(self):
		return (f"Sim instance at port {self.port}")

	def __repr__(self):
		return str(self)

# if __name__ == "__main__":
#     sim = Sim(8889, 5)
#     # sim.permapoll()
#     while True:
#         time.sleep(1)
#         print(sim.is_alive())
#         if (not sim.should_be_alive()):
#             sim.kill_sim()


import requests
from bs4 import BeautifulSoup

url = 'https://finance.yahoo.com/quote/MSFT/balance-sheet?p=MSFT'
page = requests.get(url)
page_content = page.content
soup = BeautifulSoup(page_content, 'html.parser')
table = soup.find_all('div', {'class': "M(0) Whs(n) BdEnd Bdc($seperatorColor) D(itb)"})
for t in table:
    rows = t.find_all('div', {'class': 'D(tbr) fi-row Bgc($hoverBgColor):h'})
    for row in rows:
        print(row.get_text())


"""
867. Transpose Matrix
Given a 2D integer array matrix, return the transpose of matrix.

The transpose of a matrix is the matrix flipped over its main diagonal, switching the matrix's row and column indices.

Example1:
Input: matrix = [[1,2,3],[4,5,6],[7,8,9]]
Output: [[1,4,7],[2,5,8],[3,6,9]]

Example2:
Input: matrix = [[1,2,3],[4,5,6]]
Output: [[1,4],[2,5],[3,6]]

Constraints:
m == matrix.length
n == matrix[i].length
1 <= m, n <= 1000
-10^9 <= matrix[i][j] <= 10^9
"""

"""
Note:
1. brute-force: O(mn) time | O(1) space - where m is len(matrix) and n is len(matrix[i])
"""

from typing import List
class Solution:
    def transpose(self, matrix: List[List[int]]) -> List[List[int]]:
        m, n = len(matrix), len(matrix[0])
        newMatrix = [[0] * m for _ in range(n)]
        for row in range(m):
            for col in range(n):
                newMatrix[col][row] = matrix[row][col]
        return newMatrix

# Unit Tests
import unittest
funcs = [Solution().transpose]

class TestTranspose(unittest.TestCase):
    def testTranspose1(self):
        for func in funcs:
            matrix = [[1,2,3],[4,5,6],[7,8,9]]
            self.assertEqual(func(matrix=matrix), [[1,4,7],[2,5,8],[3,6,9]])

    def testTranspose2(self):
        for func in funcs:
            matrix = [[1,2,3],[4,5,6]]
            self.assertEqual(func(matrix=matrix), [[1,4],[2,5],[3,6]])

if __name__ == "__main__":
    unittest.main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import cv2
import numpy as np
import glob
import filter

def remove_objects(img, lower_size=None, upper_size=None):
    # find all objects
    nlabels, labels, stats, centroids = cv2.connectedComponentsWithStats(img)

    sizes = stats[1:, -1]
    _img = np.zeros((labels.shape))

    # process all objects, label=0 is background, objects are started from 1
    for i in range(1, nlabels):

        # remove small objects
        if (lower_size is not None) and (upper_size is not None):
            if lower_size < sizes[i - 1] and sizes[i - 1] < upper_size:
                _img[labels == i] = 255

        elif (lower_size is not None) and (upper_size is None):
            if lower_size < sizes[i - 1]:
                _img[labels == i] = 255

        elif (lower_size is None) and (upper_size is not None):
            if sizes[i - 1] < upper_size:
                _img[labels == i] = 255

    return _img

def filter_loss(input,test):

    input = input.astype(int)
    test = test.astype(int)

    miss_mask = cv2.bitwise_xor(input,test)

    loss = cv2.countNonZero(miss_mask)

    return loss

#学習データ
train_data = sorted(glob.glob('./output/*.jpg'))
test_data = sorted(glob.glob('./test_data/*.jpg'))

print(train_data)

#画像サイズ
img_size = 768*960
#学習データの量
file_num = len(train_data)

fin_loss = 1

for remove_area in range(5000,20000+1,1000):#面積による除去の許容面積の全探索
    print(remove_area)
    loss = 0
    for train_path,test_path in zip(train_data,test_data):

        train_img = cv2.imread(train_path, cv2.IMREAD_GRAYSCALE)
        test_img = cv2.imread(test_path, cv2.IMREAD_GRAYSCALE)

        fin_mask = remove_objects(train_img,lower_size=remove_area)

        loss += filter_loss(fin_mask,test_img)/img_size

    loss = loss/file_num
    if loss < fin_loss:
        output = remove_area
        fin_loss = loss

print('final output:')
print(output)

for i,path in enumerate(train_data):

    img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)

    output_img = remove_objects(img,lower_size=output)

    path = './final_output/sample'+str(i)+'.jpg'
    cv2.imwrite(path,output_img)


from typing import List, Optional, Tuple

class Node:
    def __init__(self, key=None, value=None):
        self.key = key
        self.value = value
        self.left = None
        self.right = None
        self.parent = None

class TreeMap:
    def __init__(self):
        self.root = None
        self.length = 0

    def __delitem__(self, key):
        self.remove(key)

    def __getitem__(self, key):
        return self.get(key)

    def __setitem__(self, key, value):
        self.set(key, value)

    def __len__(self):
        return self.length

    def __iter__(self):
        return self._inorder_generator(self.root)

    def _inorder_generator(self, node):
        if node is not None:
            yield from self._inorder_generator(node.left)
            yield node
            yield from self._inorder_generator(node.right)

    def _insert(self, new_node: Node):
        parent_node = None
        current_node = self.root

        while current_node is not None:
            parent_node = current_node
            if new_node.key < current_node.key:
                current_node = current_node.left
            else:
                current_node = current_node.right

        new_node.parent = parent_node
        if parent_node is None:
            self.root = new_node
        elif new_node.key < parent_node.key:
            parent_node.left = new_node
        else:
            parent_node.right = new_node

        new_node.left = None
        new_node.right = None

    def _transplant(self, x: Node, y: Node):
        if x.parent is None:
            self.root = y
        elif x == x.parent.left:
            x.parent.left = y
        else:
            x.parent.right = y
        if y is not None:
            y.parent = x.parent

    def _minimum(self, node: Node):
        while node.left is not None:
            node = node.left
        return node

    def _delete(self, target_node: Node):
        if target_node.left is None:
            self._transplant(target_node, target_node.right)

        elif target_node.right is None:
            self._transplant(target_node, target_node.left)

        else:
            min_node = self._minimum(target_node.right)
            min_node_right = min_node.right
            if min_node.parent == target_node:
                min_node_right.parent = target_node

            else:
                self._transplant(min_node, min_node.right)
                min_node.right = target_node.right
                min_node.right.parent = min_node

            self._transplant(target_node, min_node)
            min_node.left = target_node.left
            min_node.left.parent = min_node

    def _find(self, node: Node, key) -> Tuple[Optional[Node], Optional[Node]]:
        if node is None:
            return None, None

        if key == node.key:
            return node, node.parent
        elif key < node.key:
            return self._find(node.left, key)
        else:
            return self._find(node.right, key)

    def get(self, key):
        node, _ = self._find(self.root, key)
        if node is not None:
            return node.value
        return None

    def set(self, key, value):
        node, parent = self._find(self.root, key)
        if node is not None:
            node.value = value
        else:
            node = Node(key, value)
            if parent is None:
                self._insert(node)
            elif key < parent.key:
                parent.left = node
            else:
                parent.right = node
            self.length += 1

    def remove(self, key):
        node, _ = self._find(self.root, key)
        if node is not None:
            self._delete(node)
            self.length -= 1
        return None

    def keys(self):
        for node in self:
            yield node.key

    def values(self):
        for node in self:
            yield node.value

    def items(self):
        for node in self:
            yield node.key, node.value

class Solution:
    def run_commands(self, commands: List[str], arguments: List[Optional[List]]):
        tree_map = None
        commands_output = list()
        for index in range(len(commands)):
            if commands[index] == "TreeMap":
                tree_map = TreeMap()
                commands_output.append(None)
                continue
            if commands[index] == "put":
                commands_output.append(tree_map.set(arguments[index][0], arguments[index][1]))
                continue
            if commands[index] == "remove":
                commands_output.append(tree_map.remove(arguments[index][0]))
                continue
            if commands[index] == "get":
                commands_output.append(tree_map.get(arguments[index][0]))
                continue
        return commands_output


import pandas as pd
df= pd.read_csv("Seed_Data.csv")
print(df.describe)
df.info()

X = df.iloc[:,0:7]
X.info()

y = df.iloc[:,7]
print(y.describe)

import sklearn
from sklearn import svm
from sklearn.svm import SVC

from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test= train_test_split(X,y, test_size=0.2,random_state=13)
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train=sc.fit_transform(X_train)
X_test=sc.fit(X_test)
clf = svm.SVC()
clf.fit(X_train,y_train)

pred = clf.predict(X_test)
print(pred)
sklearn.metrics.accuracy_score(y_test,pred)
print(sklearn.metrics.classification_report(y_test,pred))


from typing import Union

from django.shortcuts import get_object_or_404
from rest_framework import permissions
from rest_framework.exceptions import PermissionDenied

from paasng.platform.applications.constants import ApplicationRole
from paasng.platform.applications.models import Application, ApplicationMembership
from paasng.platform.modules.models import Module

from .base import ProtectedResource
from .tools import user_has_perm, user_has_perms

class ApplicationResource(ProtectedResource):

    permissions = [
        ('view_application', 'Can view application'),
        ('checkout_source', 'Can checkout source'),
        ('commit_source', 'Can checkout source'),
        ('manage_services', 'Can checkout source'),
        ('manage_processes', 'Can manage process'),
        ('view_logs', 'Can view logs'),
        ('manage_deploy', 'Can manage deployments'),
        ('manage_env_protection', 'Can manage env protection'),
        ('manage_members', 'Can manage memberships'),
        ('manage_product', 'Can manage product which is related to this application'),
        ('review_app', 'Can review app when it will be online'),
        ('edit_app', 'Can edit app, such as name'),
        ('delete_app', 'Can delete app'),
        ('manage_cloud_api', 'Can manage cloud api'),
        ('manage_access_control', 'Can manage access control'),
    ]

    def _get_role_of_user(self, user, obj):
        """Get the role of application"""
        role = get_user_app_role(user, obj)
        return ApplicationRole(role).name.lower()

application_resource = ApplicationResource()

application_resource.add_nobody_role()
application_resource.add_role(
    'administrator',
    {
        'view_application': True,
        'checkout_source': True,
        'commit_source': True,
        'manage_services': True,
        'manage_processes': True,
        'view_logs': True,
        'manage_deploy': True,
        'manage_env_protection': True,
        'manage_members': True,
        'manage_product': True,
        'review_app': True,
        'edit_app': True,
        'delete_app': True,
        'manage_cloud_api': True,
        'manage_access_control': True,
    },
)
application_resource.add_role(
    'developer',
    {
        'view_application': True,
        'checkout_source': True,
        'commit_source': True,
        'manage_services': True,
        'manage_processes': True,
        'view_logs': True,
        'manage_deploy': True,
        'manage_env_protection': False,
        'manage_members': False,
        'manage_product': True,
        'review_app': False,
        'edit_app': True,
        'delete_app': False,
        'manage_cloud_api': True,
        'manage_access_control': True,
    },
)
application_resource.add_role(
    'operator',
    {
        'view_application': True,
        'checkout_source': False,
        'commit_source': False,
        'manage_services': False,
        'manage_processes': False,
        'view_logs': False,
        'manage_deploy': False,
        'manage_env_protection': False,
        'manage_members': False,
        'manage_product': True,
        'review_app': True,
        'edit_app': True,
        'delete_app': False,
        'manage_cloud_api': True,
        'manage_access_control': True,
    },
)

def get_user_app_role(user, obj: Union[Application, Module]):
    # Module 权限完全兼容 Application
    if isinstance(obj, Module):
        application = obj.application
    else:
        application = obj

    try:
        # Currently, user can only have one role
        membership = ApplicationMembership.objects.get(application=application, user=user.pk)
    except (ApplicationMembership.DoesNotExist, ApplicationMembership.MultipleObjectsReturned):
        return ApplicationRole.NOBODY.value
    return membership.role

def application_perm_class(perm_name):
    """A factory function which generates a Permission class for DRF permission check"""
    if perm_name not in application_resource._permissions:
        raise ValueError('"%s" is not a valid permission name for Application' % perm_name)

    class Permission(permissions.BasePermission):
        def has_object_permission(self, request, view, obj):
            return user_has_perm(request.user, perm_name, obj)

    return Permission

def check_application_perms(user, perm_names, application):
    for perm_name in perm_names:
        if perm_name not in application_resource._permissions:
            raise ValueError('"%s" is not a valid permission name for Application' % perm_name)
    if not user_has_perms(user, perm_names, application):
        raise PermissionDenied('You are not allowed to do this operation.')

def application_perm_required(perm_name):
    """This decorator can only be used when these conditions are meet:

    - decorated func if a method in ViewSet
    - there is a "code" kwargs in request path which represents application code
    """
    if perm_name not in application_resource._permissions:
        raise ValueError('"%s" is not a valid permission name for Application' % perm_name)

    def decorated(func):
        def view_func(self, request, *args, **kwargs):
            application = get_object_or_404(Application, code=kwargs['code'])
            if not user_has_perm(request.user, perm_name, application):
                raise PermissionDenied('You are not allowed to do this operation.')

            return func(self, request, *args, **kwargs)

        return view_func

    return decorated

def application_perm_required_with_uuid(perm_name):
    """This decorator can only be used when these conditions are meet:

    - decorated func if a method in ViewSet
    - there is a "application_id" kwargs in request.data
    """
    if perm_name not in application_resource._permissions:
        raise ValueError('"%s" is not a valid permission name for Application' % perm_name)

    def decorated(func):
        def view_func(self, request, *args, **kwargs):
            application = get_object_or_404(Application, pk=request.data.get('application_id'))
            if not user_has_perm(request.user, perm_name, application):
                raise PermissionDenied('You are not allowed to do this operation.')

            return func(self, request, *args, **kwargs)

        return view_func

    return decorated