# coding:utf8
import torch

import torch.nn as nn
from torch.autograd import Variable

import numpy as np

def print_model_parm_nums(model):
    total = sum([param.nelement() for param in model.parameters()])
    print('  Number of params: %.2fM' % (total / 1e6))
    para = sum([np.prod(list(p.size())) for p in model.parameters()])
    print('  Model {} : size: {:4f}M'.format(model._get_name(), para * 4 / 1000 / 1000))

def print_model_parm_flops(model, input):
    # prods = {}
    # def save_prods(self, input, output):
    # print 'flops:{}'.format(self.__class__.__name__)
    # print 'input:{}'.format(input)
    # print '_dim:{}'.format(input[0].dim())
    # print 'input_shape:{}'.format(np.prod(input[0].shape))
    # grads.append(np.prod(input[0].shape))

    prods = {}

    def save_hook(name):
        def hook_per(self, input, output):
            # print 'flops:{}'.format(self.__class__.__name__)
            # print 'input:{}'.format(input)
            # print '_dim:{}'.format(input[0].dim())
            # print 'input_shape:{}'.format(np.prod(input[0].shape))
            # prods.append(np.prod(input[0].shape))
            prods[name] = np.prod(input[0].shape)
            # prods.append(np.prod(input[0].shape))

        return hook_per

    list_1 = []

    def simple_hook(self, input, output):
        list_1.append(np.prod(input[0].shape))

    list_2 = {}

    def simple_hook2(self, input, output):
        list_2['names'] = np.prod(input[0].shape)

    multiply_adds = False
    list_conv = []

    def conv_hook(self, input, output):
        batch_size, input_channels, input_height, input_width = input[0].size()
        output_channels, output_height, output_width = output[0].size()

        kernel_ops = self.kernel_size[0] * self.kernel_size[1] * (self.in_channels / self.groups) * (
            2 if multiply_adds else 1)
        bias_ops = 1 if self.bias is not None else 0

        params = output_channels * (kernel_ops + bias_ops)
        flops = batch_size * params * output_height * output_width

        list_conv.append(flops)

    list_linear = []

    def linear_hook(self, input, output):
        batch_size = input[0].size(0) if input[0].dim() == 2 else 1

        weight_ops = self.weight.nelement() * (2 if multiply_adds else 1)
        bias_ops = self.bias.nelement()

        flops = batch_size * (weight_ops + bias_ops)
        list_linear.append(flops)

    list_bn = []

    def bn_hook(self, input, output):
        list_bn.append(input[0].nelement())

    list_relu = []

    def relu_hook(self, input, output):
        list_relu.append(input[0].nelement())

    list_pooling = []

    def pooling_hook(self, input, output):
        batch_size, input_channels, input_height, input_width = input[0].size()
        output_channels, output_height, output_width = output[0].size()

        kernel_ops = self.kernel_size * self.kernel_size
        bias_ops = 0
        params = output_channels * (kernel_ops + bias_ops)
        flops = batch_size * params * output_height * output_width

        list_pooling.append(flops)

    def foo(net):
        childrens = list(net.children())
        if not childrens:
            if isinstance(net, torch.nn.Conv2d):
                # net.register_forward_hook(save_hook(net.__class__.__name__))
                # net.register_forward_hook(simple_hook)
                # net.register_forward_hook(simple_hook2)
                net.register_forward_hook(conv_hook)
            if isinstance(net, torch.nn.Linear):
                net.register_forward_hook(linear_hook)
            if isinstance(net, torch.nn.BatchNorm2d):
                net.register_forward_hook(bn_hook)
            if isinstance(net, torch.nn.ReLU):
                net.register_forward_hook(relu_hook)
            if isinstance(net, torch.nn.MaxPool2d) or isinstance(net, torch.nn.AvgPool2d):
                net.register_forward_hook(pooling_hook)
            return
        for c in childrens:
            foo(c)

    foo(model)
    out = model(input)

    total_flops = (sum(list_conv) + sum(list_linear) + sum(list_bn) + sum(list_relu) + sum(list_pooling))

    print('  Number of FLOPs: %.2fG' % (total_flops / 1e9))

    # print list_bn

    # print 'prods:{}'.format(prods)
    # print 'list_1:{}'.format(list_1)
    # print 'list_2:{}'.format(list_2)
    # print 'list_final:{}'.format(list_final)

def print_forward(model, input):
    select_layer = model.layer1[0].conv1

    grads = {}

    def save_grad(name):
        def hook(self, input, output):
            grads[name] = input

        return hook

    select_layer.register_forward_hook(save_grad('select_layer'))

    out = model(input)
    # print grads['select_layer']
    print(grads)

def print_value():
    grads = {}

    def save_grad(name):
        def hook(grad):
            grads[name] = grad

        return hook

    x = Variable(torch.randn(1, 1), requires_grad=True)
    y = 3 * x
    z = y ** 2

    # In here, save_grad('y') returns a hook (a function) that keeps 'y' as name
    y.register_hook(save_grad('y'))
    z.register_hook(save_grad('z'))
    z.backward()
    print('HW')
    print("grads['y']: {}".format(grads['y']))
    print(grads['z'])

def print_layers_num(model):
    # resnet = models.resnet18()
    def foo(net):
        childrens = list(net.children())
        if not childrens:
            # if isinstance(net, torch.nn.Conv2d):
            # 可以用来统计不同层的个数
            # net.register_backward_hook(print)
            return 1
        count = 0
        for c in childrens:
            count += foo(c)
        return count

    print(foo(model))

def check_summary(model):
    def torch_summarize(model, show_weights=True, show_parameters=True):
        """Summarizes torch model by showing trainable parameters and weights."""
        from torch.nn.modules.module import _addindent

        tmpstr = model.__class__.__name__ + ' (\n'
        for key, module in model._modules.items():
            # if it contains layers let call it recursively to get params and weights
            if type(module) in [
                torch.nn.modules.container.Container,
                torch.nn.modules.container.Sequential
            ]:
                modstr = torch_summarize(module)
            else:
                modstr = module.__repr__()
            modstr = _addindent(modstr, 2)

            params = sum([np.prod(p.size()) for p in module.parameters()])
            weights = tuple([tuple(p.size()) for p in module.parameters()])

            tmpstr += '  (' + key + '): ' + modstr
            if show_weights:
                tmpstr += ', weights={}'.format(weights)
            if show_parameters:
                tmpstr += ', parameters={}'.format(params)
            tmpstr += '\n'

        tmpstr = tmpstr + ')'
        return tmpstr

    # Test
    print(torch_summarize(model))

# https://gist.github.com/wassname/0fb8f95e4272e6bdd27bd7df386716b7
# summarize a torch model like in keras, showing parameters and output shape
def show_summary(model, input):
    from collections import OrderedDict
    import pandas as pd

    import torch
    from torch import nn

    def get_names_dict(model):
        """
        Recursive walk to get names including path
        """
        names = {}

        def _get_names(module, parent_name=''):
            for key, module in module.named_children():
                name = parent_name + '.' + key if parent_name else key
                names[name] = module
                if isinstance(module, torch.nn.Module):
                    _get_names(module, parent_name=name)

        _get_names(model)
        return names

    def torch_summarize_df(x, model, weights=False, input_shape=True, nb_trainable=False):
        """
        Summarizes torch model by showing trainable parameters and weights.

        author: wassname
        url: https://gist.github.com/wassname/0fb8f95e4272e6bdd27bd7df386716b7
        license: MIT

        Modified from:
        - https://github.com/pytorch/pytorch/issues/2001#issuecomment-313735757
        - https://gist.github.com/wassname/0fb8f95e4272e6bdd27bd7df386716b7/

        Usage:
            import torchvision.models as models
            model = models.alexnet()
            df = torch_summarize_df(input_size=(3, 224,224), model=model)
            print(df)

            #              name class_name        input_shape       output_shape  nb_params
            # 1     features=>0     Conv2d  (-1, 3, 224, 224)   (-1, 64, 55, 55)      23296#(3*11*11+1)*64
            # 2     features=>1       ReLU   (-1, 64, 55, 55)   (-1, 64, 55, 55)          0
            # ...
        """

        def register_hook(module):
            def hook(module, input, output):
                name = ''
                for key, item in names.items():
                    if item == module:
                        name = key
                # <class 'torch.nn.modules.conv.Conv2d'>
                class_name = str(module.__class__).split('.')[-1].split("'")[0]
                module_idx = len(summary)

                m_key = module_idx + 1

                summary[m_key] = OrderedDict()
                summary[m_key]['name'] = name
                summary[m_key]['class_name'] = class_name
                if input_shape:
                    summary[m_key][
                        'input_shape'] = (-1,) + tuple(input[0].size())[1:]
                summary[m_key]['output_shape'] = (-1,) + tuple(output.size())[1:]
                if weights:
                    summary[m_key]['weights'] = list(
                        [tuple(p.size()) for p in module.parameters()])

                #             summary[m_key]['trainable'] = any([p.requires_grad for p in module.parameters()])
                if nb_trainable:
                    params_trainable = sum(
                        [torch.LongTensor(list(p.size())).prod() for p in module.parameters() if p.requires_grad])
                    summary[m_key]['nb_trainable'] = params_trainable
                params = sum([torch.LongTensor(list(p.size())).prod() for p in module.parameters()])
                summary[m_key]['nb_params'] = params

            if not isinstance(module, nn.Sequential) and \
                    not isinstance(module, nn.ModuleList) and \
                    not (module == model):
                hooks.append(module.register_forward_hook(hook))

        # Names are stored in parent and path+name is unique not the name
        names = get_names_dict(model)

        if next(model.parameters()).is_cuda:
            x = x.cuda()

        # create properties
        summary = OrderedDict()
        hooks = []

        # register hook
        model.apply(register_hook)

        # make a forward pass
        model(x)

        # remove these hooks
        for h in hooks:
            h.remove()

        # make dataframe
        df_summary = pd.DataFrame.from_dict(summary, orient='index')

        return df_summary

    # Test on alexnet
    df = torch_summarize_df(input, model=model)
    print(df)

    # # Output
    #              name class_name        input_shape       output_shape  nb_params
    # 1     features=>0     Conv2d  (-1, 3, 224, 224)   (-1, 64, 55, 55)      23296#nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),
    # 2     features=>1       ReLU   (-1, 64, 55, 55)   (-1, 64, 55, 55)          0
    # 3     features=>2  MaxPool2d   (-1, 64, 55, 55)   (-1, 64, 27, 27)          0
    # 4     features=>3     Conv2d   (-1, 64, 27, 27)  (-1, 192, 27, 27)     307392
    # 5     features=>4       ReLU  (-1, 192, 27, 27)  (-1, 192, 27, 27)          0
    # 6     features=>5  MaxPool2d  (-1, 192, 27, 27)  (-1, 192, 13, 13)          0
    # 7     features=>6     Conv2d  (-1, 192, 13, 13)  (-1, 384, 13, 13)     663936
    # 8     features=>7       ReLU  (-1, 384, 13, 13)  (-1, 384, 13, 13)          0
    # 9     features=>8     Conv2d  (-1, 384, 13, 13)  (-1, 256, 13, 13)     884992
    # 10    features=>9       ReLU  (-1, 256, 13, 13)  (-1, 256, 13, 13)          0
    # 11   features=>10     Conv2d  (-1, 256, 13, 13)  (-1, 256, 13, 13)     590080
    # 12   features=>11       ReLU  (-1, 256, 13, 13)  (-1, 256, 13, 13)          0
    # 13   features=>12  MaxPool2d  (-1, 256, 13, 13)    (-1, 256, 6, 6)          0
    # 14  classifier=>0    Dropout         (-1, 9216)         (-1, 9216)          0
    # 15  classifier=>1     Linear         (-1, 9216)         (-1, 4096)   37752832
    # 16  classifier=>2       ReLU         (-1, 4096)         (-1, 4096)          0
    # 17  classifier=>3    Dropout         (-1, 4096)         (-1, 4096)          0
    # 18  classifier=>4     Linear         (-1, 4096)         (-1, 4096)   16781312
    # 19  classifier=>5       ReLU         (-1, 4096)         (-1, 4096)          0
    # 20  classifier=>6     Linear         (-1, 4096)         (-1, 1000)    4097000

def show_save_tensor():
    from torchvision import utils
    import torchvision.models as models
    from matplotlib import pyplot as plt

    def vis_tensor(tensor, ch=0, all_kernels=False, nrow=8, padding=2):
        '''
        ch: channel for visualization
        allkernels: all kernels for visualization
        '''
        n, c, h, w = tensor.shape
        if all_kernels:
            tensor = tensor.view(n * c, -1, w, h)
        elif c != 3:
            tensor = tensor[:, ch, :, :].unsqueeze(dim=1)

        rows = np.min((tensor.shape[0] // nrow + 1, 64))
        grid = utils.make_grid(tensor, nrow=nrow, normalize=True, padding=padding)
        # plt.figure(figsize=(nrow,rows))
        plt.imshow(grid.numpy().transpose((1, 2, 0)))  # CHW HWC

    def save_tensor(tensor, filename, ch=0, all_kernels=False, nrow=8, padding=2):
        n, c, h, w = tensor.shape
        if all_kernels:
            tensor = tensor.view(n * c, -1, w, h)
        elif c != 3:
            tensor = tensor[:, ch, :, :].unsqueeze(dim=1)
        utils.save_image(tensor, filename, nrow=nrow, normalize=True, padding=padding)

    vgg = models.resnet18(pretrained=True)
    mm = vgg.double()
    filters = mm.modules
    body_model = [i for i in mm.children()][0]
    # layer1 = body_model[0]
    layer1 = body_model
    tensor = layer1.weight.data.clone()
    vis_tensor(tensor)
    save_tensor(tensor, 'test.png')

    plt.axis('off')
    plt.ioff()
    plt.show()

def print_autograd_graph(model, input):
    from graphviz import Digraph
    import torch
    from torch.autograd import Variable

    def make_dot(var, params=None):
        """ Produces Graphviz representation of PyTorch autograd graph

        Blue nodes are the Variables that require grad, orange are Tensors
        saved for backward in torch.autograd.Function

        Args:
            var: output Variable
            params: dict of (name, Variable) to add names to node that
                require grad (TODO: make optional)
        """
        if params is not None:
            # assert all(isinstance(p, Variable) for p in params.values())
            param_map = {id(v): k for k, v in params.items()}

        node_attr = dict(style='filled',
                         shape='box',
                         align='left',
                         fontsize='12',
                         ranksep='0.1',
                         height='0.2')
        dot = Digraph(node_attr=node_attr, graph_attr=dict(size="12,12"))
        seen = set()

        def size_to_str(size):
            return '(' + (', ').join(['%d' % v for v in size]) + ')'

        def add_nodes(var):
            if var not in seen:
                if torch.is_tensor(var):
                    dot.node(str(id(var)), size_to_str(var.size()), fillcolor='orange')
                elif hasattr(var, 'variable'):
                    u = var.variable
                    # name = param_map[id(u)] if params is not None else ''
                    # node_name = '%s\n %s' % (name, size_to_str(u.size()))
                    node_name = '%s\n %s' % (param_map.get(id(u.data)), size_to_str(u.size()))
                    dot.node(str(id(var)), node_name, fillcolor='lightblue')

                else:
                    dot.node(str(id(var)), str(type(var).__name__))
                seen.add(var)
                if hasattr(var, 'next_functions'):
                    for u in var.next_functions:
                        if u[0] is not None:
                            dot.edge(str(id(u[0])), str(id(var)))
                            add_nodes(u[0])
                if hasattr(var, 'saved_tensors'):
                    for t in var.saved_tensors:
                        dot.edge(str(id(t)), str(id(var)))
                        add_nodes(t)

        add_nodes(var.grad_fn)
        return dot

    y = model(Variable(input))
    # print(y)

    g = make_dot(y, params=model.state_dict())
    g.view()
    # g

def modelsize(model, input, type_size=4):
    para = sum([np.prod(list(p.size())) for p in model.parameters()])
    print('Model {} : params: {:4f}M'.format(model._get_name(), para * type_size / 1000 / 1000))

    input_ = input.clone()
    input_.requires_grad_(requires_grad=False)

    mods = list(model.modules())
    out_sizes = []

    for i in range(1, len(mods)):
        m = mods[i]
        if isinstance(m, nn.ReLU):
            if m.inplace:
                continue
        out = m(input_)
        out_sizes.append(np.array(out.size()))
        input_ = out

    total_nums = 0
    for i in range(len(out_sizes)):
        s = out_sizes[i]
        nums = np.prod(np.array(s))
        total_nums += nums

    print('Model {} : intermedite variables: {:3f} M (without backward)'
          .format(model._get_name(), total_nums * type_size / 1000 / 1000))
    print('Model {} : intermedite variables: {:3f} M (with backward)'
          .format(model._get_name(), total_nums * type_size * 2 / 1000 / 1000))

if __name__ == '__main__':
    from models.doc_unet import DeepLabv3_plus_zj

    # from model import Doc_UNet
    torch.manual_seed(1)
    input = torch.randn(1, 3, 592, 800)
    # model = Xception(input_channels=3)
    print('DeepLabv3_plus_zj')
    model = DeepLabv3_plus_zj(input_channels=3, n_classes=2, features='xception')
    print_model_parm_nums(model)
    print_model_parm_flops(model, input)


import xarray as xr
import config
import numpy as np
import gsw
import dask

#dask.config.set(scheduler='single-threaded')

class orca(object):
    ''' get model object and process '''

    def __init__(self):
        self.data_path = config.data_path() + 'ORCA/'
        self.ds = {}
        #self.grid_keys = ['_T', '_U', '_V', '_I']
        self.grid_keys = ['_I']
        for pos in self.grid_keys:
            self.ds[pos] = xr.open_mfdataset(self.data_path +
                        '/ORCA_PATCH_*' + pos + '.nc',
                        compat='override',coords='minimal',
                        chunks={'time_counter':10}, decode_cf=True)

    #def calc_mld(self, ref_depth=10,threshold=0.03):
    #    """
    #    Calculated Mixed Layer Depth
    #    Default threshold is 0.03 referenced to 10m
    #    Data should be in format depth,distance
    #    """
    #
    #    mld[i]=(depth[(np.abs((density[:,i]-density[ref_depth,i ]))>=
    #                   threshold)].min())

    #def save_mld(self):
    #    ''' calc mixed layer depth and save '''
    #    mld = xr.DataArray(self.calc_mld(),
    #                       coords={'distance': self.giddy.distance},
    #                       dims='distance')
    #    mld_ds = self.giddy.assign(mld=mld).drop(['salt','temp','dens'])
    #    mld_ds.to_netcdf(self.root + 'Giddy_2020/giddy_mld.nc')

    def save_area_mean_all(self):
        ''' save lateral mean of all data '''

        for grid in self.grid_keys:
            print ('mean :', grid)
            ds = self.ds[grid].mean(['X','Y']).load()
            for key in ds.keys():
                ds = ds.rename({key: key + '_mean'})
            ds.to_netcdf(self.data_path + 'ORCA_PATCH_mean'
                        + grid + '.nc')

    def save_area_std_all(self):
        ''' save lateral standard deviation of all data '''

        for grid in self.grid_keys:
            print ('std :', grid)
            ds = self.ds[grid].std(['X','Y']).load()
            for key in ds.keys():
                ds = ds.rename({key: key + '_std'})
            ds.to_netcdf(self.data_path + 'ORCA_PATCH_std'
                         + grid + '.nc')

m = orca()
m.save_area_mean_all()
m.save_area_std_all()


'''
This file is using to handle all training data,following the online data structure
'''

import pandas as pd
import numpy as np
from collections import Counter
path = "/Users/songxinyi/Desktop/21chengdu80/Online_Data/trainning_data/corporate_attributes"

def handle_company_base_info(filename):
    data = pd.read_csv(filename,index_col=0)
    data['honor_num'] = data['honor_num'].fillna(data['honor_num'].median())
    data['illegad_num'] = data['illegad_num'].fillna(data['illegad_num'].median())
    data['yea_sub'] = data['yea_sub'].fillna(data['yea_sub'].median())
    data['web_dum'] = data['web_dum'].fillna(data['web_dum'].median())
    data.drop('REGCAP',axis=1, inplace=True)
    data['REGCAPCUR'] = data['REGCAPCUR'].apply(lambda x:1 if x==156 else 0)
    data['RECCAP'] = data['RECCAP'].fillna(data['RECCAP'].median())
    data['PARNUM'] = data['PARNUM'].fillna(data['PARNUM'].median())
    data['LIMPARNUM'] = data['LIMPARNUM'].fillna(data['LIMPARNUM'].median())
    data['PARFORM'] = data['PARFORM'].fillna(data['PARFORM'].median())
    data['EXENUM'] = data['EXENUM'].fillna(data['EXENUM'].median())
    data['brand_num'] = data['brand_num'].fillna(data['brand_num'].median())
    data['cancel_dum'] = data['cancel_dum'].fillna(data['cancel_dum'].median())
    data['certificate_num'] = data['certificate_num'].fillna(data['certificate_num'].median())
    return data

def all_equal(data):
    a = data.to_numpy()
    if (a[0] == a[1:]).all():
        return 0
    else:
        return 1

def handle_company_ar(filename):
    data = pd.read_csv(filename,index_col=0)
    df = data.groupby(by='entid').agg({'EMPNUM':np.mean,'WOMEMPNUM':np.mean})
    df['BUSST'] = data.groupby(by='entid').apply(lambda df: all_equal(df) )
    return df
def handle_company_ar_assetsinfo(filename):
    data = pd.read_csv(filename,index_col=0)
    df = data.groupby(by='entid').agg({'ASSGRO':np.mean,'LIAGRO':np.mean,
                                       'VENDINC':np.mean,'MAIBUSINC':np.mean,
                                       'PROGRO':np.mean,'NETINC':np.mean,
                                       'RATGRO':np.mean,'TOTEQU':np.mean})
    df['ispublic'] = data.groupby(by='entid').apply(lambda df: all_equal(df))
    return df
def handle_company_ar_nz(filename):
    data = pd.read_csv(filename,index_col=0)
    df = data.groupby(by='entid').agg({'MEMNNUM': np.mean, 'FARNUM': np.mean,
                                       'ANNNEWMEM': np.mean, 'ANNREDMEM': np.mean})
    df['MEMNNUM'] = df.fillna(df['MEMNNUM'].median())
    df['FARNUM'] = df.fillna(df['FARNUM'].median())
    df['ANNNEWMEM'] = df.fillna(df['ANNNEWMEM'].median)
    return df
def handle_tax_abnormal(filename):
    data = pd.read_csv(filename)
    print(data.entid)
    data.drop('cdate',axis=1, inplace=True)
    print(data.entid)
    data.drop_duplicates(subset=['entid'], keep='first', inplace=True)
    return data

def handle_company_ar_alterstockinfo(filename):
    df = pd.read_csv(filename)
    df = df.sort_values("entid")
    df1 = df.groupby(by='entid').agg({'INV': np.count_nonzero, 'TRANSAMPR': np.mean, 'TRANSAMAFT': np.mean})
    df1['SUB'] = df1['TRANSAMAFT'] - df1['TRANSAMPR']
    df1['SUB'] = df1['SUB'].fillna(df1['SUB'].median())
    df1['INV'] = df1['INV'].fillna(df1['INV'].median())
    return df1[['INV', 'SUB']]

def handle_company_modify(filename):
    data = pd.read_csv(filename)
    data = data.fillna(0)
    data.drop('ALTDATE',axis=1, inplace=True)
    data.drop_duplicates(subset=['entid'],keep='first', inplace=True)
    return data

def handle_el_company_history_inv(filename):
    df = pd.read_csv(filename)
    def type_not_4(x):
        return len(x) - Counter(x)[4]
    def type_4(x):
        return Counter(x)[4]
    df3 = df.groupby("entid").agg({"ttype": [type_not_4, type_4]})
    df3.columns = df3.columns.droplevel()
    print(df3)
    return df3

def handle_el_company_history_manager(filename):
    df = pd.read_csv(filename)
    table = pd.pivot_table(df, index=["entid"], values=["NAME"], aggfunc=len, fill_value=0)
    return table

def tool1(name,l,temp):
    flag = 0
    for index, row in temp.iterrows():
        if not pd.isnull(row[name]):
            flag = 1
            l.append(row[name])
            break
    if flag==0:
        l.append(np.nan)
    return l
def handle_tax_company(filename):
    tax_company = pd.read_csv(filename)
    new_tax_company = pd.DataFrame(columns=(
    'entid', 'region_id', 'entertype_tax', 'industry_tax', 'registertype', 'economic_type', 'incometax_rate',
    'collection_type', 'avg_EMPNUM', 'std_EMPNUM'))
    entid = tax_company['entid'].unique().tolist()
    print(len(entid))
    print(type(entid))
    for i in entid:
        l = []
        temp = tax_company.loc[tax_company['entid'] == i]

        l.append(i)
        if (all_equal(temp.region_id)):
            l.append(temp.region_id.iloc[0])
        else:
            l.append(0)
        l = tool1('entertype_tax', l, temp)
        l = tool1('industry_tax', l, temp)
        l = tool1('registertype', l, temp)
        l = tool1('economic_type', l, temp)
        l = tool1('incometax_rate', l, temp)
        l = tool1('collection_type', l, temp)

        avg_EMPNUM = temp.EMPNUM.mean()
        std_EMPNUM = temp.EMPNUM.std()
        l.append(avg_EMPNUM)
        l.append(std_EMPNUM)
        new_tax_company.loc[len(new_tax_company)] = l
    return new_tax_company

def handle_tax_year(filename):
    df = pd.read_csv(filename,index_col=0)
    df1 = df.groupby(by='entid').agg({'islatest': np.max,
                                            'profit_loss': np.mean, 'revenue': np.mean, 'sales_cost': np.mean,
                                            'sales_tax': np.mean, 'sales_expense': np.mean, 'G_expense': np.mean,
                                            'finance_expense': np.mean, 'asset_devaluation_losses': np.mean,
                                            'fair_value_change': np.mean, 'investment_income': np.mean,
                                            'operation_profit': np.mean, 'non_operating_income': np.mean,
                                            'non_operating_expense': np.mean, 'total_profit': np.mean,
                                            'assets': np.mean, 'assets_re': np.mean, 'payrol_expense': np.mean,
                                            'welfare_expenses': np.mean, 'education_expenses': np.mean,
                                            'union_funds': np.mean, 'ad_expenses': np.mean,
                                            'entertainment_expenses': np.mean,
                                            'assets_loss': np.mean, 'asset_devaluation_allowance': np.mean,
                                            'main_business_income': np.mean, 'main_business_cost': np.mean,
                                            'other_business_income': np.mean, 'other_operating_cost': np.mean,
                                            'retained_profits': np.mean, 'gross_profit': np.mean,
                                            'main_business_gross': np.mean,
                                            'income_tax_adjuste': np.mean, 'income_taxable': np.mean,
                                            'incometax_taxable': np.mean,
                                            'incometax_amount': np.mean, 'incometax_actual': np.mean})
    df1 = df1.fillna(0)
    df1["welfare_expenses"] = df1["welfare_expenses"].map(lambda x: 1 if x != 0 else 0)
    df1["education_expenses"] = df1["education_expenses"].map(lambda x: 1 if x != 0 else 0)
    df1["union_funds"] = df1["union_funds"].map(lambda x: 1 if x != 0 else 0)
    df1["ad_expenses"] = df1["ad_expenses"].map(lambda x: 1 if x != 0 else 0)
    df1["entertainment_expenses"] = df1["entertainment_expenses"].map(lambda x: 1 if x != 0 else 0)
    df1["assets_loss"] = df1["assets_loss"].map(lambda x: 1 if x != 0 else 0)
    df1["asset_devaluation_allowance"] = df1["asset_devaluation_allowance"].map(lambda x: 1 if x != 0 else 0)
    return df1

out_path = "./newdata"
handle_company_base_info(path+"/"+"company_base_info.csv").to_csv(out_path+"/company_base_info.csv")
handle_company_ar(path+"/"+"company_ar.csv").to_csv(out_path+"/company_ar.csv")
handle_company_ar_assetsinfo(path+"/"+"company_ar_assetsinfo.csv").to_csv(out_path+"/company_ar_assetsinfo")
handle_company_ar_nz(path+"/"+"company_ar_nz.csv").to_csv(out_path+"/company_ar_nz.csv")
handle_tax_abnormal(path+"/"+"tax_abnormal.csv").to_csv(out_path+"/tax_abnormal.csv")
handle_company_ar_alterstockinfo(path+"/"+"company_ar_alterstockinfo.csv").to_csv(out_path+"/company_ar_alterstockinfo.csv")
handle_company_modify(path+"/"+"company_modify.csv").to_csv(out_path+"/company_modify.csv")
handle_el_company_history_inv(path+"/"+"el_company_history_inv.csv").to_csv(out_path+"/el_company_history_inv.csv")
handle_el_company_history_manager(path+"/"+"el_company_history_manager.csv").to_csv(out_path+"/el_company_history_manager.csv")
handle_tax_company(path+"/"+"tax_company.csv").to_csv(out_path+"/tax_company.csv",index=False)
handle_tax_year(path+"/"+"tax_year.csv").to_csv(out_path+"/tax_year.csv")


from imports import *

accu_score = []
f1_score = []
precision =[]
mse = []
mae =[]
recall = []

from sklearn.model_selection import KFold
k = 10
cv = KFold(n_splits=k)

def build(df, x_np, y_np, x_names, y_names):

    for train_index, test_index in cv.split(df):
        x_np_train, y_np_train = x_np[train_index], y_np[train_index]
        x_np_test, y_np_test = x_np[test_index], y_np[test_index]

        scaler = StandardScaler()
        x_np_train= scaler.fit_transform(x_np_train)
        x_np_test = scaler.transform(x_np_test)

        model = RandomForestClassifier(
                                            n_estimators=100,
                                           max_features="sqrt",max_depth=4,max_samples= None,
                                           criterion="gini",
                                           min_impurity_decrease=0,
                                           # # min_samples_leaf= 1, ##only used for H2o comparison
                                           min_samples_split=1,  ##only used for Pyspark comparison
                                           random_state=42)

        ##TRAINING THE MODEL
        model.fit(x_np_train, y_np_train)
        preds = model.predict(x_np_test)

        ##FINDING METRICS
        accur = accuracy_score(y_np_test, preds)
        precise = precision_score(y_np_test, preds)
        MAE = mean_absolute_error(y_np_test, preds)
        MSE = mean_squared_error(y_np_test, preds)
        rcall = recall_score(y_np_test, preds)
        F1 = 2 * (precise * rcall) / (precise + rcall)

        accu_score.append(accur)
        precision.append(precise)
        mse.append(MSE)
        mae.append(MAE)
        recall.append(rcall)
        f1_score.append(F1)

    print(" ")
    print(f"Accuracy : {np.mean(accu_score)}")
    print(f"F1_Score : {np.mean(f1_score)}")
    print(f"Precision: {np.mean(precision)}")
    print(f"M A E    : {np.mean(mae)}")
    print(f"M S E    :{np.mean(mse)}")
    print(f"Recall   : {np.mean(recall)}")
    print(" ")


import sys
import re
from bs4 import BeautifulSoup, Tag, NavigableString

with open(sys.argv[1]) as fp:
    soup = BeautifulSoup(fp, 'lxml-xml')

root = soup.find_all(value='ROOT')[0]

def find_leaf(tag, phrase):
    tmp_wd = []

    if(tag['value'] == 'CC'):
        cc_wd = tag.find_all('leaf')[0]['value']

        prev_sib = list(tag.previous_siblings)[1]
        prev_sib_depth = str(len(list(prev_sib.parents)))
        prev_sib1 = prev_sib['value']+prev_sib_depth

        next_sib = list(tag.next_siblings)[1]
        next_sib_depth = str(len(list(next_sib.parents)))
        next_sib1 = next_sib['value'] + next_sib_depth

        phrase.append([tag['value']+str(len(list(tag.parents))), prev_sib1, next_sib1, cc_wd])

    #elif (tag.name != 'leaf'):
    elif (re.match( r'^NP|WHNP|PP|WHPP|ADJP|WHADJP|ADVP|WHAVP|X|SBAR|NAC|NML|CONJP|FRAG|INTJ|LST|NAC|NX|QP|PRC|PRN|PRT|QP|RRC|UCP|ROOT|S|,$', tag['value']) and (tag.name != 'leaf')):
        for wd in tag.find_all('leaf'):
                tmp_wd.append(wd['value'])
        phrase.append([tag['value']+str(len(list(tag.parents))), tmp_wd])

def find_lwg(tag,tmp_lwg,flag):
    for tag1 in tag.contents:
        if (isinstance(tag1, Tag)):
            if (re.match(r'^VB|RB|TO|MD', tag1['value'])):
                tmp_lwg.append(tag1.find_all('leaf')[0]['value'])

            elif (re.match(r'^VP', tag1['value'])):
                find_lwg(tag1,tmp_lwg,1)
    if(flag == 0):
        phrase.append([tag['value']+'_LWG'+str(len(list(tag.parents))), tmp_lwg])

def process_tag(tag, phrase):
        if (isinstance(tag, Tag)):
            if (tag['value'] == 'VP') and (tag.parent['value'] != 'VP'):
                tmp_lwg = []
                #find_leaf(tag)
                find_lwg(tag,tmp_lwg,0)
                for tag1 in tag.contents:
                    if (isinstance(tag1, Tag)):
                        process_tag(tag1, phrase)

            elif(re.match(r'CC',tag['value'])):
                find_leaf(tag, phrase)

            else:
                find_leaf(tag, phrase)
                for tag1 in tag.contents:
                    if (isinstance(tag1, Tag)):
                        process_tag(tag1, phrase)

phrase = []
for ch in root.contents:
    if(isinstance(ch, Tag)):
        process_tag(ch,phrase)

for p in phrase:
    if(re.match(r'CC',p[0])):
        print(p[0], ':', p[1],'-', p[3], '-', p[2])
    else:
        print(p[0],':',' '.join(p[1]))


# get_ipython().magic('matplotlib inline')
import matplotlib
import numpy as np
import matplotlib.pyplot as plt
from sklearn import linear_model
import functools
from scipy.stats import poisson

# Generate training data for sales probability regression
def generate_train_data(B=100):

    def rank(a, p):
        return np.argsort(np.hstack((a, p)))[:,0]

    our_price = 10 + np.random.uniform(0, 10, (B, 1))
    competitor_prices = 10 + np.random.uniform(0, 10, (B, 5))
    our_rank = np.reshape(rank(our_price, competitor_prices), (B, 1))
    X = np.hstack((our_price, competitor_prices, our_rank))

    # Y = (our_rank == 0).astype(int).ravel()
    # Y = np.maximum(0, (3 - our_rank)).astype(int).ravel()
    Y = np.round(np.random.uniform(0, 1, our_rank.shape) * (1 - our_rank / 11)).ravel()

    return (X, Y)

X, Y = generate_train_data()

# Train sales probability model
def make_model(X, Y):
    regr = linear_model.LinearRegression()
    regr.fit(X, Y)
    def predict(x):
        return np.maximum(0, regr.predict(x))
        # return regr.predict_proba(x)[:, 1]
    return predict

sales_model = make_model(X, Y)

T=20
N=5
price_range=np.arange(10, 20, 0.1)
L=0.01
delta=0.99
Z=1

# competitor_prices = 10 + np.random.uniform(0, 10, 5)
competitor_prices = np.array([14, 15, 16, 17, 18])
computed_values = {}

def rank(a, p):
    _rank = p.shape[0]
    for i in range(p.shape[0]):
        if a < p[i]:
            _rank = _rank - 1
    return _rank

def _V(price, t, n):
    x = np.hstack((price, competitor_prices, rank(price, competitor_prices))).reshape(1, -1)
    # sales_prob = round(sales_model(x)[0])
    sales_prob = sales_model(x)[0]

    _sum = 0
    # TODO: Check here
    # for i in range(2):

    # print(sales_prob)

    pi_sum = 0
    for i in range(int(poisson.ppf(0.9999, sales_prob)) + 1):
        pi = poisson.pmf(i, sales_prob)
        pi_sum += pi
        today_profit = min(n, i) * price
        holding_costs = n * L
        _, V_future = V(t + 1, max(0, n - i))
        exp_future_profits = delta * V_future
        _sum += pi * (today_profit - holding_costs + exp_future_profits)
    # print(pi_sum)
    return _sum

def V(t, n):
    if (t,n) in computed_values:
        return computed_values[t,n]
    if t >= T:
        computed_values[t,n] = (0, n * Z)
        return (0, n * Z)
    if n <= 0:
        computed_values[t,n] = (0, 0)
        return (0, 0)

    V_opt = -100000
    price_opt = -100000
    for _, price in enumerate(price_range):
        v = _V(price, t, n)
        if v > V_opt:
            V_opt = v
            price_opt = price

    computed_values[t,n] = (price_opt, V_opt)
    return (price_opt, V_opt)

V(0, N)

print(computed_values)

for i_n in range(N + 1):
    datapoints = [ computed_values[i_t, i_n][0] for i_t in range(T + 1) if(i_t, i_n) in computed_values ]
    print(datapoints)
    label_text = 'N=' + str(i_n)
    plt.plot(datapoints, label=label_text)
    plt.ylabel('n')
    plt.xlabel('T')
    plt.legend()
    plt.show()


#!/usr/bin/env python
from __future__ import print_function
import csv
import json
import requests
import os
import os.path
from argparse import ArgumentParser
from utils import login, get, post

def add_device(dnac, name, serial, pid):
    payload = [{
	"deviceInfo": {
		"name": name,
		"serialNumber": serial,
		"pid": pid,
		"sudiRequired": False,
		"userSudiSerialNos": [],
		"stack": False,
		"aaaCredentials": {
			"username": "",
			"password": ""
		}
	}
}]
    device = post(dnac, "onboarding/pnp-device/import", payload)
    try:
        deviceId = device.json()['successList'][0]['id']
    except IndexError as e:
        print ('##SKIPPING device:{},{}:{}'.format(name, serial, device.json()['failureList'][0]['msg']))
        deviceId = None

    return deviceId

def claim_device(dnac,deviceId, configId, workflowId, params):
    payload = {
	"deviceClaimList": [{
		"deviceId": deviceId,
		"configList": [{
			"configId": configId,
			"configParameters": params
		}]
	}],
	"projectId": None,
	"workflowId": workflowId,
	"configId": None,
	"imageId": None,
	"populateInventory": True
}
    #print json.dumps(payload, indent=2)

    claim = post(dnac,"onboarding/pnp-device/claim", payload)
    return claim.json()['message']

def get_workflow(dnac,workflowName):
    response = get (dnac, "onboarding/pnp-workflow")
    for workflow in response.json():
        if workflow['name'] == workflowName:
            workflowId = workflow['id']
            #print json.dumps(workflow, indent=4)
            for tasks in workflow['tasks']:
                configId = tasks['configInfo']['configId']
            return workflowId, configId

    raise ValueError("Cannot find template:{}".format(workflowName))

def get_template(dnac, configId, supplied_params):
    params=[]
    response = get(dnac, "template-programmer/template/{}".format(configId))
    for vars in response.json()['templateParams']:
        name = vars['parameterName']
        params.append({"key": name, "value": supplied_params[name]})
    #print params
    return params

def create_and_upload(dnac, devices):

    f = open(devices, 'rt')
    try:
        reader = csv.DictReader(f)
        for device_row in reader:
            #print ("Variables:",device_row)

            try:
                workflowId, configId = get_workflow(dnac, device_row['workflow'])
            except ValueError as e:
                print("##ERROR {},{}: {}".format(device_row['name'],device_row['serial'], e))
                continue

            params = get_template(dnac, configId, device_row)

            deviceId = add_device(dnac, device_row['name'], device_row['serial'], device_row['pid'])
            if deviceId is not None:
                #claim
                claim_status = claim_device(dnac, deviceId, configId, workflowId, params)
                if "Claimed" in claim_status:
                    status = "PLANNED"
                else:
                    status = "FAILED"
                print ('Device:{} name:{} workflow:{} Status:{}'.format(device_row['serial'],
                                                                    device_row['name'],
                                                                    device_row['workflow'],
                                                                    status))
    finally:
        f.close()

if __name__ == "__main__":
    parser = ArgumentParser(description='Select options.')
    parser.add_argument( 'devices', type=str,
            help='device inventory csv file')
    args = parser.parse_args()

    dnac = login()
    print ("Using device file:", args.devices)

    print ("##########################")
    create_and_upload(dnac, devices=args.devices)


import taichi as ti

ti.init(debug=True)

dt = 1e-5
planets_limit = 128

num_planets = ti.field(ti.i32, shape=())

x = ti.Vector.field(2, ti.f32, shape=planets_limit, needs_grad=True)
v = ti.Vector.field(2, ti.f32, shape=planets_limit)
potential = ti.field(ti.f32, shape=(), needs_grad=True)

@ti.kernel
def compute_potential(n: ti.i32):
    for i, j in ti.ndrange(n, n):
        dist = (x[i] - x[j]).norm(1e-3)
        potential[None] += -1 / dist

@ti.kernel
def update():
    for i in range(num_planets[None]):
        v[i] += dt * -x.grad[i]

    for i in range(num_planets[None]):
        x[i] += dt * v[i]

def step():
    with ti.Tape(potential):
        compute_potential(num_planets[None])

    update()

def make_planet(pos_x: ti.i32, pos_y: ti.i32):
    new_id = num_planets[None]
    num_planets[None] += 1

    x[new_id] = [pos_x, pos_y]

gui = ti.GUI('Gravity', res=(720, 720), background_color=0x000000)

while gui.running:
    for e in gui.get_events(ti.GUI.PRESS):
        if e.key in [ti.GUI.ESCAPE, ti.GUI.EXIT]:
            exit()
        elif e.key == ti.GUI.LMB and num_planets[None] < planets_limit:
            make_planet(e.pos[0], e.pos[1])

    for i in range(50):
        step()
    gui.circles(x.to_numpy(), radius=3)
    gui.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from xlsxwriter import Workbook
import ibm_db_dbi as db2

cur = db2.connect().cursor()
cur.execute("select cusnum, lstnam, cdtlmt, baldue, cdtdue from qiws.qcustcdt")

headers = [desc[0] for desc in cur.description]

with Workbook('qcustcdt_format.xlsx') as workbook:
    fmt = workbook.add_format({'font_size': 20})

    hdr_fmt = workbook.add_format({'font_size': 20, 'align':'center', 'border':1})
    red_fmt = workbook.add_format({'font_size': 20, 'bg_color': '#FF0000'})

    ws = workbook.add_worksheet()

    ws.conditional_format("D2:D13", {'type': 'cell',
                                     'criteria': '>',
                                     'value': 'C2*0.5',
                                     'format': red_fmt})

    ws.set_column(0, len(headers)-1, 20)

    ws.write_row('A1', headers, hdr_fmt)
    ws.set_row(0, 22)

    for rownum, row in enumerate(cur, start=1):
        ws.write_row(rownum, 0, row)
        ws.set_row(rownum, 22, fmt)


import cv2
import numpy as np
import struct

class CompressedImageWriter:
	def __init__(self,name,w,h):
		self.f=open(name,"wb")
		self.f.write(b'XKCD')
		self.f.write(struct.pack(">HH",w,h))
	def write(self,n,c):
		c=int(c)
		while n>0:
			if n<128:
				b=n*2+c
				b=struct.pack(">B",b)
				self.f.write(b)
				n-=n
			else:
				b=127*2+c
				b=struct.pack(">B",b)
				self.f.write(b)
				n-=127
	def close(self):
		self.f.close()

img=cv2.imread("xkcd.bmp",0)/255
h,w=img.shape[:2]
out=CompressedImageWriter("test.xkcd",w,h)
for row in img:
	color=row[0]
	count=0
	for item in row:
		if item==color:
			count+=1
		else:
			out.write(count,color)
			color=item
			count=1
	out.write(count,color)
out.close()


from app.main import main
from app.main import multiThreadRequest
import json
import time
# from .utils import *
from flask import current_app, make_response, request, make_response, \
    jsonify, redirect, current_app, abort

from config import *
from ..models import Idataset
from .utils import inference
# @main.before_app_request
# def request_images_from_heycloud():
#     json = jsonify(request.get_json('test'))
#     return None

@main.route('/detection/aircraft', methods=['POST'])
def info():
    if request.content_type == 'application/json':
        args = dict(request.get_json(force=True))
    else:
        args = {
            key: value[0] if len(value) == 1 else value
            for key, value in request.form.items()
        }
        args['bbox'] = [float(el) for el in args['bbox'][1:-1].split(',')]

    SLICE_HEIGHT = args['height'] if args.get('height') is not None else 608
    SLICE_WIDTH = args['width'] if args.get('width') is not None else 608
    SP_SLICE_HEIGHT = SLICE_HEIGHT * SP_RES
    SP_SLICE_WIDTH = SLICE_WIDTH * SP_RES

    cfg = current_app.config
    width_min = args['bbox'][0]
    height_min = args['bbox'][1]
    width_max = args['bbox'][2]
    height_max = args['bbox'][3]
    post_params = {
        "bbox": [],
        "bands": [0,1,2],
        "height": SLICE_HEIGHT,
        "width": SLICE_WIDTH

    }

    headers = cfg['IMAGE_REQUEST_HEADERS']
    request_url = cfg['IMAGE_REQUEST_HOST'] +  \
                '/heycloud/api/data/idataset' + \
                '/37605aef-913e-40b6-8859-822e72a51a19' + \
                '/extract'

    # threads = multiThreadRequest(INFER_IMAGE_PATH,
    #                     height_min,
    #                     width_min,
    #                     height_max,
    #                     width_max,
    #                     SLICE_HEIGHT,
    #                     SLICE_WIDTH,
    #                     SP_SLICE_HEIGHT,
    #                     SP_SLICE_WIDTH,
    #                     IMAGE_HEIGHT,
    #                     IMAGE_WIDTH,
    #                     SLICE_OVERLAP,
    #                     post_params,
    #                     request_url,
    #                     headers
    #                     )
    start = time.time()

    # for th in threads:
    #     th.setDaemon(True)
    #     th.start()
    # for t in threads:
    #     t.join()
    print("\n[INFO] total request time: %ss\n" % round(time.time()-start, 2))

    json = inference(NET_CONFIG_FILE,
              INFER_DATA_FILE,
              WEIGHTS, BATCH_SIZE,
              IMAGE_PATH_TXT,
              INFER_RES_TXT,
              CONF_THRESH,
              SP_RES,
              args['bbox'],
              NMS_THRESH,
              INFER_RES_PATH)

    return json

@main.route('/test', methods=['POST'])
def test():
    if request.content_type == 'application/json':
        args = dict(request.get_json(force=True))
    else:
        args = {
            key: value[0] if len(value) == 1 else value
            for key, value in request.form.items()
        }
        args['bbox'] = [float(el) for el in args['bbox'][1:-1].split(',')]

    # peter = Idataset.query.filter_by(id='test').first_or_404(description='There is no data with {}'.format('test'))
    # print(Idataset.file)
    abort(404)
    return str(request.accept_mimetypes.accept_json)


#!/usr/bin/python
# -*- coding: UTF-8 -*-
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('-i', type=str, required=True)
parser.add_argument('-o', type=str, required=True)

def majority(input, output):
	out = open(output,"w")
	with open(input) as f:
		lines = f.readlines()
		for _ in range(len(lines)):
			out.write("0\n")

def main(args):
	majority(args.i, args.o)

if __name__ == '__main__':
	args = parser.parse_args()
	main(args)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Dec 21 13:52:42 2020

@author: jonasg
"""

# NOTE: these were extracted from one of the NetCDF files, quite a few site
# locations do not contain obsdata (only NaNs), I added a print below in
# case you choose one of such sites accidentally =)
ALL_SITES = ['ATHENS-NOA' 'Agoufou' 'Alta_Floresta' 'Appledore_Island' 'Arcachon'
             'Arica' 'Aubiere_LAMP' 'Autilla' 'Avignon' 'BONDVILLE' 'Bac_Lieu'
             'Bach_Long_Vy' 'Baengnyeong' 'Bambey-ISRA' 'Bandung' 'Banizoumbou'
             'Barcelona' 'Beijing' 'Beijing_RADI' 'Belsk' 'Billerica' 'Blida'
             'Bonanza_Creek' 'Bratts_Lake' 'Brussels' 'Bucharest_Inoe' 'Burjassot'
             'CARTEL' 'CCNY' 'CEILAP-BA' 'CLUJ_UBB' 'COVE_SEAPRISM' 'CRPSM_Malindi'
             'CUIABA-MIRANDA' 'Cabauw' 'Cabo_da_Roca' 'Caceres' 'Cairo_EMA_2'
             'CalTech' 'Camaguey' 'Campo_Grande_SONDA' 'Cape_San_Juan' 'Capo_Verde'
             'Carpentras' 'Cart_Site' 'Chapais' 'Chen-Kung_Univ' 'Chiang_Mai_Met_Sta'
             'Chilbolton' 'Cordoba-CETT' 'DMN_Maine_Soroa' 'Dakar' 'Dayton' 'Dhadnah'
             'Dongsha_Island' 'Dunkerque' 'Dushanbe' 'EPA-NCU' 'Easton_Airport'
             'Eforie' 'Egbert' 'Ersa' 'Evora' 'Fresno' 'GSFC' 'Gandhi_College'
             'Granada' 'Guadeloup' 'Gual_Pahari' 'Gustav_Dalen_Tower' 'Gwangju_GIST'
             'Halifax' 'Hamburg' 'Harvard_Forest' 'Helgoland' 'Helsinki'
             'Hong_Kong_Hok_Tsui' 'Hong_Kong_PolyU' 'Huelva' 'Hyytiala' 'IER_Cinzana'
             'IMS-METU-ERDEMLI' 'Irkutsk' 'Ispra' 'Jaipur' 'Ji_Parana_SE' 'KONZA_EDC'
             'Kanpur' 'Karachi' 'Karlsruhe' 'Kellogg_LTER' 'Kelowna_UAS' 'Kuopio'
             'Kuwait_University' 'Kyiv' 'LISCO' 'LSU' 'La_Laguna' 'La_Parguera'
             'Laegeren' 'Lahore' 'Lake_Argyle' 'Lampedusa' 'Lecce_University'
             'Leipzig' 'Lille' 'London-UCL-UAO' 'MCO-Hanimaadhoo' 'MD_Science_Center'
             'MVCO' 'Mainz' 'Malaga' 'Manila_Observatory' 'Messina' 'Mezaira' 'Minsk'
             'Modena' 'Moldova' 'Moscow_MSU_MO' 'Munich_University' 'Mussafa'
             'NCU_Taiwan' 'NGHIA_DO' 'Nes_Ziona' 'Noto' 'Oostende' 'Osaka' 'Palaiseau'
             'Paris' 'Pickle_Lake' 'Pokhara' 'Pune' 'Ragged_Point' 'Red_River_Delta'
             'Resolute_Bay' 'Rio_Branco' 'Rome_Tor_Vergata' 'SAGRES'
             'SANTA_CRUZ_UTEPSA' 'SEDE_BOKER' 'SERC' 'Saada' 'Santa_Cruz_Tenerife'
             'Sao_Martinho_SONDA' 'Sao_Paulo' 'Saturn_Island' 'Sevastopol' 'Seysses'
             'Shirahama' 'Silpakorn_Univ' 'Singapore' 'Skukuza' 'Solar_Village'
             'Songkhla_Met_Sta' 'TUBITAK_UZAY_Ankara' 'Taihu' 'Taipei_CWB'
             'Thessaloniki' 'Thompson_Farm' 'Tomsk' 'Toravere' 'Toulon' 'UMBC'
             'Ubon_Ratchathani' 'Univ_of_Houston' 'Univ_of_Lethbridge' 'Ussuriysk'
             'Venise' 'Vientiane' 'Villefranche' 'Wallops' 'WaveCIS_Site_CSI_6'
             'Xanthi' 'XiangHe' 'Xinglong' 'Yekaterinburg' 'Zinder_Airport']
result = []

import glob, os
import numpy as np
import pandas as pd
import xarray as xr

data_dir = '/lustre/storeA/project/aerocom/aerocom-users-database/PYAEROCOM_COLDATA/WEB_EVAL/aerocom/AP3-abs'
var_mask = 'abs550' # there is also some called abs550csaer or abs550aercs
exp = '25COV'

# INSERT HERE THE SITES THAT YOU ARE INTERESTED IN
sitenames = ['Alta_Floresta', 'Dakar']

result = {}
for mod in os.listdir(data_dir):
    print(mod)
    files = glob.glob(f'{data_dir}/{mod}/{var_mask}*{exp}*.nc')
    if not len(files) > 0: # ignore folders without data
        print(f'Skip {mod}')
        continue

    # make sure only one filematch
    assert len(files) == 1
    file = files[0]
    var_name = os.path.basename(files[0]).split('_')[0]
    data = xr.open_dataset(file)[var_name]

    print(data.station_name.data)

    #make sure this is in monthly resolution
    assert data.ts_type == 'monthly'

    result[mod] = {}

    timestamps = data.time.data

    for i, site in enumerate(sitenames):
        result[mod][site] = {}
        # get plot axes for this site

        sitedata = data.sel(station_name=site)
        obs_numpy = sitedata.data[0]
        if all(np.isnan(obs_numpy)):
            print(f'No obsdata available at {site}')

        mod_numpy = sitedata.data[1]

        obs_ts = pd.Series(obs_numpy, timestamps)
        mod_ts = pd.Series(obs_numpy, timestamps)

        result[mod][site]['obs'] = obs_ts
        result[mod][site]['mod'] = mod_ts


import ply.lex as lex

# Tokens so small that don't deserve a unique rule (at least they have a good personality)
literals = [',','{','}','=','\'','[',']','.']

# List of token names
tokens = [
    'DATETIME',
    'TIME',
    'HEX',
    'OCT',
    'BIN',
    'FLOAT',
    'INT',
    'INFINITY',
    'NAN',
    'BOOL',
    'WORD',
    'LITERALMULT',
    'QUOTEMULT',
    'LITERALS',
    'STRING',
]

t_WORD = r'[a-zA-Z_\-0-9]+'

def t_LITERALMULT(t):
    r"'''.*'''"
    t.value = str(t.value)[3:-3]
    return t

def t_QUOTEMULT(t):
    r'""".*"""'
    t.value = str(t.value)[3:-3]
    return t

def t_STRING(t):
    r'"[^"]*"'
    t.value = str(t.value)[1:-1]
    return t

def t_LITERALS(t):
    r"'[^']*'"
    t.value = str(t.value)[1:-1]
    return t

def t_DATETIME(t):
    r'\d{4}-\d{2}-\d{2}([Tt]\d{2}:\d{2}:\d{2}([Zz]|(\.\d{6})?((\+|-)\d{2}:\d{2})?))?' # as said in RFC3339 "NOTE: Per [ABNF] and ISO8601, the "T" and "Z" characters in this syntax may alternatively be lower case "t" or "z" respectively."
    t.value = str(t.value).replace("T"," ").replace("Z","+00:00")
    return t

def t_TIME(t):
    r'\d{2}:\d{2}:\d{2}([Zz]|(\.\d{6})?((\+|-)\d{2}:\d{2})?)' #isto depois tem de ser convertido em string para o json
    t.value = str(t.value).replace("T"," ").replace("Z","+00:00")
    return t

def t_BOOL(t):
    r'(true|false)'
    if t.value == "true":
        t.value = bool(True)
    elif t.value == "false":
        t.value = bool(False)
    return t

def t_OCT(t):
    r'0o[0-7_]+'
    filtered = ''.join((filter(lambda x: x != '_', t.value)))
    t.value = int(filtered,8)
    return t

def t_BIN(t):
    r'0b[01_]+'
    filtered = ''.join((filter(lambda x: x != '_', t.value)))
    t.value = int(filtered,2)
    return t

def t_HEX(t):
    r'0x[A-Fa-f0-9_]+'
    filtered = ''.join((filter(lambda x: x != '_', t.value)))
    t.value = int(filtered,16)
    return t

def t_FLOAT(t):
    r'(\+|-)?((\d+_?)+([eE](\+|-)?(\d+_?)+)|(\d+_?)+\.(\d+_?)+([eE](\+|-)?(\d+_?)+)?)'
    t.value = float(str(t.value).replace("_",""))
    return t

def t_INT(t):
    r'(\+|-)?\d+'
    t.value = int(t.value)
    return t

def t_INFINITY(t):
    r'(\+|-)?inf'
    t.value = float(t.value)
    return t

def t_NAN(t):
    r'(\+|-)?nan'
    t.value = float(t.value)
    return t

t_ignore  = ' \t'

def t_ignore_COMMENT(t):
    r'\#.*\n'
    pass

def t_error(t):
    t.lexer.skip(1)

lexer = lex.lex()


import yfinance as yf
import sys

import tensorflow as tf

from sklearn.metrics import accuracy_score, mean_squared_error, mean_absolute_percentage_error

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt

def prediction(model, original, X_test, scaler, loss = 'mse'):

  prediction = model.predict(X_test)
  prediction = prediction.reshape(prediction.shape[0],1)
  prediction_copies_array = np.repeat(prediction, X_test.shape[2], axis=-1) #change this one

  pred = scaler.inverse_transform(np.reshape(prediction_copies_array,(len(prediction), X_test.shape[2])))[:,3]
  #pred = np.reshape(pred, (len(prediction), X_test.shape[2]))[:, 3]
  if loss == 'mse':
    testScore = mean_squared_error(original, pred)
  if loss == 'mape':
    testScore = mean_absolute_percentage_error(original, pred)

  return pred, testScore

def classification(data, data_main, change):
    if change == 'no change':

        data['Actual_change'] = np.where(data['Close_actual_change'] < data['Close_actual_change'].shift(1), 0, 1)
        data['Pred_change'] = np.where(data['Close_actual_change'] > data['Close_prediction_change'].shift(-1), 0, 1)
        data['Pred_change'] = data.Pred_change.shift(1)
        data = data[1:]
        data['Pred_change'] = data['Pred_change'].astype(int)
        classification_accuracy = len(data[(data.Actual_change == data.Pred_change)]) / len(data)

    elif change == 'absolute':

        data['Actual_change'] = np.where(data['Close_actual_change'] < 0, 0, 1)
        data['Pred_change'] = np.where(data['Close_prediction_change'] < 0, 0, 1)
        data = data[1:]
        data['Pred_change'] = data['Pred_change'].astype(int)
        data['Close_actual'] = data_main.loc['2021-01-01':, 'Close'][30:]
        # data['Close_prediction'] = dataset_test['Close'].shift(1) + data.Close_prediction_change
        classification_accuracy = len(data[(data.Actual_change == data.Pred_change)]) / len(data)

    return data, classification_accuracy

# def plot_results(ticker, df, data, data_tr, pred, change):
#     plt.figure(figsize=(12, 6))
#     y_test_change = data_tr.loc['2021-01-01':]
#     y_test_change = np.array(y_test_change.iloc[30:, 3]) #takes only Close value
#     plt.plot(y_test_change, color='green', label='Real Price')
#     plt.plot(pred, color='purple', label='Predicted Price')
#     plt.title(f'{ticker}')
#     plt.xlabel('Time')
#     plt.ylabel('Price')
#     plt.legend()
#     plt.show()
#
#     if change == 'absolute':
#
#         pd.concat([data.loc['2021-01-01':]['Close'], pd.DataFrame(data.loc['2021-01-01':]['Close'].shift(1) + df.Close_prediction_change)], axis=1)[31:].plot(figsize=(12, 8))
#         plt.title('Close Absolute Change Prediction')
#
#         pd.concat([data.loc['2021-01-01':]['Close'][31:], pd.DataFrame(data.loc['2021-01-01':]['Close'][31] + df.Close_prediction_change)], axis=1).plot(figsize=(12, 8))
#         plt.title('Close Absolute Change Prediction (only adding changes)')
#
#     else:
#         pass


from sklearn.metrics import average_precision_score
import pandas as pd
import numpy as np

def mAP_score(preds: pd.DataFrame, trues: pd.DataFrame):
    """ Funkcja obliczająca wynik modelu sieci neuronowej

        :param preds: Wartości przewidywane przez model
        :type preds: pd.DataFrame
        :param trues: Wartości prawdziwe
        :type trues: pd.DataFrame
        :return: Ogólny wynik modelu oraz wyniki dla każdej z cech w postaci listy
        :rtype: float, list
        """
    labels_score = []
    trues = trues.replace(-1,0)
    score = 0.0
    for p, t in zip(preds, trues):
        ap = 0.0
        if np.sum(trues[t]) != 0:
            ap = average_precision_score(trues[t], preds[p])
        labels_score.append(ap)
        score += ap

    score /= preds.shape[1]

    return score, labels_score


# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

from scrapy.pipelines.files import  FilesPipeline
class FiledownloadPipeline(object):
    def process_item(self, item, spider):
        tmp = item['file_urls']
        item['file_urls'] = []

        for i in tmp:
            if "?" in i:
                item['file_urls'].append(i.split('?')[0])
            else:
                item['file_urls'].append(i)
        return item

class  MyFilesPipeline(FilesPipeline):
    def file_path(self, request, response=None, info=None):
        file_path = request.url
        file_path = file_path.split('/')[-1]
        print("下载图片"+ file_path)
        return 'full/%s' % (file_path)


import scraper
import xlsxwriter

workbook = xlsxwriter.Workbook('data.xlsx')
main = workbook.add_worksheet()
schedule = workbook.add_worksheet()

data = scraper.data
headers1 = scraper.headers1
headers2 = ["Class (CRN)"] + scraper.headers2

r, c = 0,0
for header in headers1:
    main.write(r, c, header)
    c += 1
c = 0
for header in headers2:
    schedule.write(r, c, header)
    c+=1
r, rs = 1, 1
for row in data:
    main.write(r, 0, row["CRN"])
    main.write(r, 1, row["Subj"])
    main.write(r, 2, row["Crse"])
    main.write(r, 3, row["Sec"])
    main.write(r, 4, row["Units"])
    main.write(r, 5, row["Title"])
    main.write(r, 6, row["Campus"])
    r += 1
    for day in row["Schedule"]:
        schedule.write(rs, 0, row["CRN"])
        schedule.write(rs, 1, day["StartDate"])
        schedule.write(rs, 2, day["EndDate"])
        schedule.write(rs, 3, day["Days"])
        schedule.write(rs, 4, day["Times"])
        schedule.write(rs, 5, day["Bldg"])
        schedule.write(rs, 6, day["Room"])
        rs += 1
    schedule.write(rs, 0, "")
    rs += 1

workbook.close()
print("xls file created")


# -*- coding: utf-8 -*-
"""
{{cookiecutter.root_module_name}}.cli
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Command line entry point.
"""

#
# Imports
#

import argparse
import logging
import sys

#
# Module variables
#

_LOGGER = logging.getLogger(__name__)

#
# Functions
#

def _parse_cli_args():
    """Parse the command line arguments"""

    # get an instance of a parser
    parser = argparse.ArgumentParser()

    # setup logging flags
    logging_flags_group = parser.add_mutually_exclusive_group()
    logging_flags_group.add_argument(
        '--verbose', action='store_true', help="Show logging from INFO or above")
    logging_flags_group.add_argument(
        '--debug', action='store_true', help="Show logging from DEBUG or above")

    # here you'd setup any other arguments/flags you cared about

    # return result of parsing cli args
    return parser.parse_args()

def _configure_logging(log_level=None):
    """Configure logging"""
    logging_format = '%(asctime)s %(levelname)s %(module)s %(lineno)d %(name)s - %(message)s'
    if log_level is None:
        log_level = logging.WARNING
    logging.basicConfig(stream=sys.stderr, level=log_level, format=logging_format)

def _update_logging_config(parsed_args):
    """Update the logging config based on the parsed cli args"""
    if parsed_args.verbose:
        _configure_logging(log_level=logging.INFO)
    elif parsed_args.debug:
        _configure_logging(log_level=logging.DEBUG)

def main():
    """Command line entry point"""
    _configure_logging()
    parsed_args = _parse_cli_args()
    _update_logging_config(parsed_args)

    _LOGGER.info("Begin running")
    _LOGGER.debug("Here is where you'd put calls to functions etc that did the work of your app")
    _LOGGER.info("Finished running")


# -*- coding: utf-8 -*-
# @File    : check_mask.py
# @Time    : 02/02/2023
# @Author  : Fanyi Sun
# @Github  : https://github.com/sunfanyi
# @Software: PyCharm

import os
import numpy as np
import skimage.io
import matplotlib.pyplot as plt

dataset_dir = os.path.abspath("")

file_name = 'aeroplaneFGL1_BGL1\\n02690373_16'
annotation_path = os.path.join(dataset_dir, 'annotations', file_name) + '.npz'
image_path = os.path.join(dataset_dir, 'images', file_name) + '.jpeg'

annotation = np.load(annotation_path, allow_pickle=True)
image = skimage.io.imread(image_path)

plt.figure()
plt.imshow(image.astype(np.uint8))
plt.axis('off')
plt.show()

print(annotation.files)
print(annotation['source'])
print(image.shape)

mask = (annotation['mask'] > 200)
plt.figure()
plt.title('occluded original mask')
plt.axis('off')
plt.imshow(mask.astype(np.uint8))
plt.show()

color = [255, 100, 100]
masked_image = image.copy()
alpha = 0.5
for c in range(3):
    masked_image[:, :, c] = np.where(mask == 1,
                                     # image[:, :, c] *
                                     (1 - alpha) + alpha * color[c] * 255,
                                     image[:, :, c])

plt.figure()
plt.title('masked_image')
plt.axis('off')
plt.imshow(masked_image.astype(np.uint8))
plt.show()


from django.urls import path
from .views import (RetrieveExameView,
                    ListExameView,
                    RegisterExameView,
                    DeleteExameView,
                    UpdateExameView)

app_name = "api-exame"

urlpatterns = [
    path('register_exame', RegisterExameView.as_view(), name='register_exame'),
    path('list_exame', ListExameView.as_view(), name='list_exame'),
    path('retrieve_exame/<int:id>', RetrieveExameView.as_view(), name='retrieve_exame'),
    path('delete_exame/<int:id>', DeleteExameView.as_view(), name='delete_exame'),
    path('update_exame/<int:id>', UpdateExameView.as_view(), name='update_exame')
]


from bson import ObjectId
from pymongo import MongoClient, WriteConcern
from pymongo.read_concern import ReadConcern

from sdk.common.utils.common_functions_utils import escape

# CHANGEME
url = "mongodb+srv://ppmaintenanceuser:<password>@hu-europe-west2-prod.b7d74.gcp.mongodb.net"

# CHANGEME
db_name = "pp_eu_prod"
mongo_client = MongoClient(url)
db = mongo_client.get_database(db_name)

def ask_for_agreement(deployment_name: str):
    answer = input(
        f"Are you sure you want to delete devices for {deployment_name} deployment? [Y|N]: "
    )
    if answer.lower() in ["y", "yes", "ok", ""]:
        return True
    return False

def lookup_deployment(search: str = None, deployment_id: str = None) -> dict:
    d = db.get_collection("deployment")
    if search:
        result = d.find({"name": {"$regex": escape(search), "$options": "i"}})
    elif deployment_id:
        result = d.find_one({"_id": ObjectId(deployment_id)})
        return result
    else:
        raise NotImplementedError

    deployments = []
    for i, elem in enumerate(result):
        print(f"\n\tNumber: {i}\n\tName:\t{elem['name']}\n\tID:\t{elem['_id']}\n")
        deployments.append(elem)

    if deployments:
        choice = int(input("Enter which deployment number: "))
        return deployments[choice]

def delete_devices(deployment_id: str):
    d = db.get_collection("user")
    d2 = db.get_collection("device")
    resource = f"deployment/{deployment_id}"
    result = d.find({"roles": {"roleId": "User", "resource": resource}})
    users = list(result)
    print(f"{len(users)} users found. Deleting devices...")
    count = 0
    with mongo_client.start_session() as session:
        with session.start_transaction(
            read_concern=ReadConcern("snapshot"),
            write_concern=WriteConcern("majority", wtimeout=1000),
        ):
            for user in users:
                res = d2.delete_many({"userId": user["_id"]}, session=session)
                count += res.deleted_count

    print(f"[+] Done. {count} records were deleted.")

def main():
    deployment = None
    while not deployment:
        search = input("Enter deployment search term: ")

        if ObjectId.is_valid(search):
            deployment = lookup_deployment(deployment_id=search)
        else:
            deployment = lookup_deployment(search)

        if deployment and ask_for_agreement(deployment["name"]):
            delete_devices(str(deployment["_id"]))

if __name__ == "__main__":
    main()


import numpy as np
import cv2
from sample import sample

def make_image_matrix(dir_name, filenames, num_pixels):
    # number of exposures
    num_exposures = filenames.shape[0]

    # Create the vector of sample indices
    # We need np(P-1) > (Zmax - Zmin)
    # Assuming the maximum (Zmax - Zmin) = 255,
    # np = (255 * 2) / (P-1) clearly fulfills this requirement
    num_samples = int(np.ceil(255.0 * 2.0 / (num_exposures - 1.0)) * 2.0)

    # create a random sampling matrix, telling us which
    # pixels of the original image we want to sample
    # using ceil fits the indices into the range [1,numPixels+1],
    # i.e. exactly the range of indices of zInput
    step = np.float128(num_pixels) / np.float128(num_samples)

    sample_indices = np.floor(np.array(np.arange(0.0, np.float128(num_pixels), step)))
    sample_indices = sample_indices.astype(int);

    # allocate resulting matrices
    z_red = np.zeros((num_samples, num_exposures))
    z_green = np.zeros((num_samples, num_exposures))
    z_blue = np.zeros((num_samples, num_exposures))

    for i in range(0, num_exposures):
        # read the nth image
        image = cv2.imread(dir_name + filenames[i])

        # sample the image for each color channel
        [z_red_temp, z_green_temp, z_blue_temp] = sample(image, sample_indices)

        # build the resulting, small image consisting
        # of samples of the original image
        z_red[:, i] = z_red_temp
        z_green[:, i] = z_green_temp
        z_blue[:, i] = z_blue_temp

    return [z_red, z_green, z_blue]


from random import SystemRandom
from string import ascii_letters, digits
from pyrogram.filters import command, regex
from pyrogram.handlers import MessageHandler, CallbackQueryHandler
from asyncio.subprocess import PIPE, create_subprocess_exec as exec
from bot import bot, config_dict, status_dict_lock, status_dict
from bot.helper.ext_utils.menu_utils import Menus
from bot.helper.telegram_helper.bot_commands import BotCommands
from bot.helper.telegram_helper.filters import CustomFilters
from bot.helper.ext_utils.rclone_utils import get_rclone_path, is_rclone_config, list_remotes
from bot.helper.telegram_helper.message_utils import sendStatusMessage
from bot.modules.tasks_listener import MirrorLeechListener
from bot.helper.mirror_leech_utils.status_utils.sync_status import SyncStatus

SOURCE= None
listener_dict= {}

async def handle_sync(client, message):
    user_id= message.from_user.id
    tag = f"@{message.from_user.username}"
    if await is_rclone_config(user_id, message):
        await list_remotes(message, menu_type=Menus.SYNC, remote_type='source')
        listener_dict[message.id] = MirrorLeechListener(message, tag, user_id)

async def sync_callback(client, query):
    data= query.data.split("^")
    message= query.message
    user_id= query.from_user.id
    msg_id= query.message.reply_to_message.id

    listener= listener_dict[msg_id]
    path = await get_rclone_path(user_id, message)

    if data[1] == "source":
        await query.answer()
        globals()['SOURCE']= data[2]
        await list_remotes(message, menu_type=Menus.SYNC, remote_type='destination', edit=True)
    elif data[1] == "destination":
        await query.answer()
        destination = data[2]
        await start_sync(message, path, destination, listener)
    else:
        await query.answer()
        await message.delete()

async def start_sync(message, path, destination, listener):
    cmd = ["rclone", "sync", "--delete-during", "-P", f'--config={path}', f"{SOURCE}:", f"{destination}:"]
    if config_dict["SERVER_SIDE"]:
        cmd.append("--server-side-across-configs")

    process = await exec(*cmd, stdout=PIPE, stderr=PIPE)

    gid = ''.join(SystemRandom().choices(ascii_letters + digits, k=10))
    async with status_dict_lock:
        status = SyncStatus(process, gid, SOURCE, destination, listener)
        status_dict[listener.uid] = status
    await sendStatusMessage(listener.message)
    await status.start()

    return_code = await process.wait()

    if return_code == 0:
        msg= 'Sync completed successfully✅\n\n'
        msg+= '<b>Note:</b>'
        msg+= '\n1.Use dedupe command to remove duplicate file/directory'
        msg+= '\n2.Use rmdir command to remove empty directories'
        await listener.onRcloneSyncComplete(msg)
    else:
        err= await process.stderr.read()
        await listener.onDownloadError(str(err))

    await message.delete()

bot.add_handler(MessageHandler(handle_sync, filters=command(BotCommands.SyncCommand) & (CustomFilters.user_filter | CustomFilters.chat_filter)))

bot.add_handler(CallbackQueryHandler(sync_callback, filters= regex("syncmenu")))


import wx, wx.grid
import widget_properties
from widget_properties import GridProperty
from xml.sax.saxutils import escape, quoteattr

class CodeProperty(widget_properties.TextProperty):
    def __init__(self, owner, name='extracode'):
        setattr(owner, name, "")
        def get(): return getattr(owner, name)
        def set(val): return setattr(owner, name, val)
        owner.access_functions[name] = (get, set)
        widget_properties.TextProperty.__init__(self, owner, name, None,
                                                True, multiline=True)

    def _show(self, notebook, label=_('Code')):
        panel, new = _find_or_create_page(notebook, label)
        self.display(panel)
        szr = wx.BoxSizer(wx.VERTICAL)
        szr.Add(self.panel, 1, wx.EXPAND)
        if new:
            panel.SetSizerAndFit(szr)
            notebook.AddPage(panel, label)
        else:
            sizer = panel.GetSizer()
            sizer.Add(szr, 1, wx.EXPAND)
            panel.Layout()

    def display(self, parent):
        self.id = wx.NewId()
        val = self.get_value()
        val = val.replace('\\n', '\n')

        sb = wx.StaticBox(parent, -1, "")

        label = widget_properties.wxGenStaticText(parent, -1,
                                                  _('Extra code for this widget'))
        self._enabler = wx.CheckBox(parent, self.id+1, '')
        tooltip = """\
You can use this property to add some extra code to that generated by wxGlade.

Please note that you should use this ability only if you have the \
"Overwrite existing sources" option set.

NOTE: at the moment, this property is supported only by the following code \
generators:
    Python C++ XRC"""
        style = wx.TE_MULTILINE|wx.HSCROLL
        self.text = wx.TextCtrl(parent, self.id, val, style=style, size=(1, -1))
        font = wx.Font(12, wx.FONTFAMILY_MODERN, wx.FONTSTYLE_NORMAL,
                       wx.FONTWEIGHT_NORMAL)
        self.text.SetFont(font)

        label.SetToolTip(wx.ToolTip(tooltip))
        wx.EVT_CHECKBOX(self._enabler, self.id+1,
                        lambda event: self.toggle_active(event.IsChecked()))
        self.text.Enable(self.is_active())
        self._enabler.SetValue(self.is_active())
        self._target = self.text
        szr = wx.BoxSizer(wx.HORIZONTAL)
        szr.Add(label, 1, wx.ALL|wx.EXPAND, 3)
        szr.Add(self._enabler, 0, wx.ALL|wx.EXPAND, 3)
        sizer = wx.StaticBoxSizer(sb, wx.VERTICAL)
        sizer.Add(szr, 0, wx.EXPAND)
        sizer.Add(self.text, 1, wx.ALL|wx.EXPAND, 3)
        h = self.text.GetCharHeight()
        sizer.SetItemMinSize(self.text, -1, h*3)
        self.panel = sizer
        self.bind_event(self.on_change_val)
        wx.EVT_CHAR(self.text, self.on_char)

# end of class CodeProperty

class ExtraPropertiesProperty(GridProperty):
    def __init__(self, owner):
        setattr(owner, 'extraproperties', [])
        def get(): return getattr(owner, 'extraproperties')
        def set(val): return setattr(owner, 'extraproperties', val)
        owner.access_functions['extraproperties'] = (get, set)

        cols = [(_('Property'), GridProperty.STRING),
                (_('Value'), GridProperty.STRING)]
        self.label = _('Extra properties for this widget')
        GridProperty.__init__(self, owner, 'extraproperties', None, cols,
                              can_insert=False)

    def write(self, outfile, tabs):
        if self.getter:
            props = self.getter()
        else:
            props = self.owner[self.name][0]()
        if props:
            written = False
            write = outfile.write
            stab = '    ' * (tabs+1)
            for name, value in props:
                if value:
                    if not written:
                        written = True
                        write('    ' * tabs + '<extraproperties>\n')
                    write('%s<property name=%s>%s</property>\n' %
                          (stab, quoteattr(name), escape(value.strip())))
            if written:
                write('    ' * tabs + '</extraproperties>\n')

    def display(self, panel):
        GridProperty.display(self, panel)
        self.btn.Hide()
        wx.grid.EVT_GRID_CELL_CHANGE(self.grid, self.on_change_val)
        tooltip = """\
You can use this property to add some extra custom properties to this widget.

For each property "prop" with value "val", wxGlade will generate a \
"widget.SetProp(val)" line (or a "<prop>val</prop>" line for XRC).

NOTE: at the moment, this property is supported only by the following code \
generators:
    Python C++ XRC Perl"""
        self.grid.SetToolTip(wx.ToolTip(tooltip))

    def add_row(self, event):
        GridProperty.add_row(self, event)
        self.on_change_val(event)

    def remove_row(self, event):
        GridProperty.remove_row(self, event)
        self.on_change_val(event)

    def set_value(self, val):
        if isinstance(val, dict):
            val = [[k, val[k]] for k in sorted(val.keys())]
        GridProperty.set_value(self, val)

    def _show(self, notebook, label=_('Code')):
        panel, new = _find_or_create_page(notebook, label)
        self.display(panel)
        szr = wx.BoxSizer(wx.VERTICAL)
        szr.Add(self.panel, 1, wx.EXPAND)
        if new:
            panel.SetSizerAndFit(szr)
            notebook.AddPage(panel, _('Code'))
        else:
            sizer = panel.GetSizer()
            sizer.Add(szr, 1, wx.EXPAND)
            sizer.Layout()

# end of class ExtraPropertiesProperty

class ExtraPropertiesPropertyHandler(object):
    def __init__(self, owner):
        self.owner = owner
        self.props = {}
        self.prop_name = None
        self.curr_prop = []

    def start_elem(self, name, attrs):
        if name == 'property':
            self.prop_name = attrs['name']

    def end_elem(self, name):
        if name == 'property':
            if self.prop_name and self.curr_prop:
                self.props[self.prop_name] = ''.join(self.curr_prop)
            self.prop_name = None
            self.curr_prop = []
        elif name == 'extraproperties':
            self.owner.properties['extraproperties'].set_value(self.props)
            val = [[k, self.props[k]] for k in sorted(self.props.keys())]
            self.owner.extraproperties = val
            return True # to remove this handler

    def char_data(self, data):
        data = data.strip()
        if data:
            self.curr_prop.append(data)

# end of class ExtraPropertiesPropertyHandler

def _find_or_create_page(notebook, label):
    """\
    Searches the given notebook for a page whose label is "label".
    """
    for i in xrange(notebook.GetPageCount()):
        if notebook.GetPageText(i) == label:
            return notebook.GetPage(i), False
    return wx.Panel(notebook, -1, style=wx.TAB_TRAVERSAL), True


from pathlib import Path
import copy

raw = Path('Day 10\input.txt').read_text()
#raw = Path('Day 10\\test.txt').read_text()
raw = raw.strip()
raw = raw.split("\n")

length = len(raw)

list = []
for i in range(length): list.append(int(raw[i]))
list.sort()

num = 0
jolt1 = 0
jolt3 = 0

new = [0]
for i in range(length):
    if list[i] == num + 3:
        jolt3 += 1
        num = list[i]
    elif list[i] == num + 1:
        jolt1 += 1
        num = list[i]
    if num not in new: new.append(num)
jolt3 += 1
laptop = num + 3
print(jolt1 * jolt3)

ara = [0]
percent = 0
print(str(percent) + "%")
for x in range(length):
    if percent < round(x/length * 100):
        percent = round(x/length * 100)
        print(str(percent) + "%")
    for y in range(len(ara)):
        if ara[y] == num: continue
        j1 = ara[y] + 1 in list
        j2 = ara[y] + 2 in list
        j3 = ara[y] + 3 in list
        if j1 and j2 and j3:
            ara.append(ara[y] + 1)
            ara.append(ara[y] + 2)
            ara[y] += 3
        elif j1 and j3:
            ara.append(ara[y] + 1)
            ara[y] += 3
        elif j2 and j3:
            ara.append(ara[y] + 2)
            ara[y] += 3
        elif j1 and j2:
            ara.append(ara[y] + 1)
            ara[y] += 2
        else:
            if j1: ara[y] += 1
            if j2: ara[y] += 2
            if j3: ara[y] += 3
    print("current length: " + str(len(ara)))
print(len(ara))


import os
from typing import Any, Dict, List

import boto3
import requests
from aws_requests_auth.aws_auth import AWSRequestsAuth
from aws_requests_auth.boto_utils import BotoAWSRequestsAuth
from chaoslib.discovery.discover import (
    discover_actions,
    discover_activities,
    discover_probes,
    initialize_discovery_result,
)
from chaoslib.exceptions import InterruptExecution
from chaoslib.types import Configuration, DiscoveredActivities, Discovery, Secrets
from logzero import logger

__version__ = "0.25.0"
__all__ = ["__version__", "discover", "aws_client", "signed_api_call"]

def get_credentials(secrets: Secrets = None) -> Dict[str, str]:
    """
    Credentials may be provided via the secrets object. When they aren't,
    they will be loaded from the process environment (for instance, read from
    `~/.aws/credentials`).

    See: https://boto3.readthedocs.io/en/latest/guide/configuration.html#guide-configuration
    """  # noqa: E501
    creds = dict(
        aws_access_key_id=None, aws_secret_access_key=None, aws_session_token=None
    )

    if secrets:
        creds["aws_access_key_id"] = secrets.get("aws_access_key_id")
        creds["aws_secret_access_key"] = secrets.get("aws_secret_access_key")
        creds["aws_session_token"] = secrets.get("aws_session_token")

    return creds

def aws_client(
    resource_name: str, configuration: Configuration = None, secrets: Secrets = None
):
    """
    Create a boto3 client for the given resource.

    You may pass the `aws_region` key in the `configuration` object to
    be explicit about which region you want to use.

    You may pass `aws_profile_name` value to the `configuration` object so that
    we load the appropriate profile to converse with the AWS services. In that
    case, make sure your local `~/aws/credentials` config is properly setup, as
    per https://boto3.readthedocs.io/en/latest/guide/configuration.html#aws-config-file

    Also, if you want to assume a role, you should setup that file as per
    https://boto3.readthedocs.io/en/latest/guide/configuration.html#assume-role-provider
    as we do not read those settings from the `secrets` object.
    """  # noqa: E501
    configuration = configuration or {}
    aws_profile_name = configuration.get("aws_profile_name")
    aws_assume_role_arn = configuration.get("aws_assume_role_arn")
    params = get_credentials(secrets)

    region = configuration.get("aws_region")
    if not region:
        logger.debug(
            "The configuration key `aws_region` is not set, looking in the "
            "environment instead for `AWS_REGION` or `AWS_DEFAULT_REGION`"
        )
        region = os.getenv("AWS_REGION", os.getenv("AWS_DEFAULT_REGION"))
        if not region:
            raise InterruptExecution("AWS requires a region to be set!")

    if region:
        logger.debug(f"Using AWS region '{region}'")
        params["region_name"] = region

    if boto3.DEFAULT_SESSION is None:
        # we must create our own session so that we can populate the profile
        # name when it is provided. Only create the default session once.
        boto3.setup_default_session(profile_name=aws_profile_name, **params)

    if not aws_assume_role_arn:
        logger.debug(
            "Client will be using profile '{}' from boto3 session".format(
                aws_profile_name or "default"
            )
        )
        return boto3.client(resource_name, **params)
    else:
        logger.debug(
            "Fetching credentials dynamically assuming role '{}'".format(
                aws_assume_role_arn
            )
        )

        aws_assume_role_session_name = configuration.get("aws_assume_role_session_name")
        if not aws_assume_role_session_name:
            aws_assume_role_session_name = "ChaosToolkit"
            logger.debug(
                "You are missing the `aws_assume_role_session_name` "
                "configuration key. A unique one was generated: '{}'".format(
                    aws_assume_role_session_name
                )
            )

        client = boto3.client("sts", **params)
        params = {
            "RoleArn": aws_assume_role_arn,
            "RoleSessionName": aws_assume_role_session_name,
        }
        response = client.assume_role(**params)
        creds = response["Credentials"]
        logger.debug(
            "Temporary credentials will expire on {}".format(
                creds["Expiration"].isoformat()
            )
        )

        params = {
            "aws_access_key_id": creds["AccessKeyId"],
            "aws_secret_access_key": creds["SecretAccessKey"],
            "aws_session_token": creds["SessionToken"],
        }
        if region:
            params["region_name"] = region

        return boto3.client(resource_name, **params)

def signed_api_call(
    service: str,
    path: str = "/",
    method: str = "GET",
    configuration: Configuration = None,
    secrets: Secrets = None,
    params: Dict[str, Any] = None,
) -> requests.Response:
    """
    Perform an API call against an AWS service.

    This should only be used when boto does not already implement the service
    itself. See https://boto3.readthedocs.io/en/latest/reference/services/index.html

    for a list of supported services by boto. This function does not claim
    being generic enough to support the whole range of AWS API.

    The `configuration` object should look like this:

    ```json
    {
        "aws_region": "us-east-1",
        "aws_host": "amazonaws.com"
    }
    ```

    While both are optional, and default to the values shown in this snippet,
    you should make sure to be explicit about them to avoid confusion.

    The endpoint being called is built from the given `service` name, the
    given region and host as well as the `path` of the action being called on
    the service. By default, the call is made over `HTTPS` but this can be
    changed by setting `aws_endpoint_scheme` in the configuration dictionary.

    Pass any parameters of the API itself as part of the remaining `params`
    paramater is a dictionary. It should match the signature of the service
    you are trying to call and will be sent as a query-string when `method` is
    `"GET"` or `"DELETE"`, or as a JSON payload otherwise. Refer to the AWS
    documentation for each service type.

    This function does not support profile names so you must provide the
    credentials in secrets.
    """  # noqa: E501
    configuration = configuration or {}
    region = configuration.get("aws_region", "us-east-1") or ""
    host = configuration.get("aws_host", "amazonaws.com")
    scheme = configuration.get("aws_endpoint_scheme", "https")
    host = f"{service}.{region}.{host}"
    endpoint = configuration.get("aws_endpoint", f"{scheme}://{host}").replace(
        "..", "."
    )
    endpoint = f"{endpoint}{path}"
    creds = get_credentials(secrets)

    # when creds weren't provided via secrets, we let boto search for them
    # from the process environment

    if creds["aws_access_key_id"] and creds["aws_secret_access_key"]:
        auth = AWSRequestsAuth(
            aws_access_key=creds["aws_access_key_id"],
            aws_secret_access_key=creds["aws_secret_access_key"],
            aws_host=host,
            aws_region=region,
            aws_service=service,
        )
    else:
        auth = BotoAWSRequestsAuth(
            aws_host=host, aws_region=region, aws_service=service
        )

    headers = {"Accept": "application/json"}

    if method in ("DELETE", "GET"):
        return requests.request(
            method, endpoint, headers=headers, auth=auth, params=params
        )

    return requests.request(method, endpoint, headers=headers, auth=auth, json=params)

def discover(discover_system: bool = True) -> Discovery:
    """
    Discover AWS capabilities from this extension as well, if a aws
    configuration is available, some information about the AWS environment.
    """
    logger.info("Discovering capabilities from chaostoolkit-aws")

    discovery = initialize_discovery_result("chaostoolkit-aws", __version__, "aws")
    discovery["activities"].extend(load_exported_activities())

    return discovery

###############################################################################
# Private functions
###############################################################################
def load_exported_activities() -> List[DiscoveredActivities]:
    """
    Extract metadata from actions and probes exposed by this extension.
    """
    activities = []
    activities.extend(discover_actions("chaosaws.ec2.actions"))
    activities.extend(discover_probes("chaosaws.ec2.probes"))
    activities.extend(discover_actions("chaosaws.ecs.actions"))
    activities.extend(discover_probes("chaosaws.ecs.probes"))
    activities.extend(discover_actions("chaosaws.iam.actions"))
    activities.extend(discover_probes("chaosaws.iam.probes"))
    activities.extend(discover_actions("chaosaws.eks.actions"))
    activities.extend(discover_probes("chaosaws.eks.probes"))
    activities.extend(discover_actions("chaosaws.elbv2.actions"))
    activities.extend(discover_probes("chaosaws.elbv2.probes"))
    activities.extend(discover_actions("chaosaws.asg.actions"))
    activities.extend(discover_probes("chaosaws.asg.probes"))
    activities.extend(discover_actions("chaosaws.awslambda.actions"))
    activities.extend(discover_probes("chaosaws.awslambda.probes"))
    activities.extend(discover_actions("chaosaws.cloudwatch.actions"))
    activities.extend(discover_probes("chaosaws.cloudwatch.probes"))
    activities.extend(discover_actions("chaosaws.rds.actions"))
    activities.extend(discover_probes("chaosaws.rds.probes"))
    activities.extend(discover_actions("chaosaws.elasticache.actions"))
    activities.extend(discover_probes("chaosaws.elasticache.probes"))
    activities.extend(discover_actions("chaosaws.emr.actions"))
    activities.extend(discover_probes("chaosaws.emr.probes"))
    activities.extend(discover_actions("chaosaws.route53.actions"))
    activities.extend(discover_probes("chaosaws.route53.probes"))
    activities.extend(discover_actions("chaosaws.ssm.actions"))
    activities.extend(discover_actions("chaosaws.fis.actions"))
    activities.extend(discover_probes("chaosaws.s3.probes"))
    activities.extend(discover_actions("chaosaws.s3.actions"))
    activities.extend(discover_activities("chaosaws.s3.controls.upload", "control"))
    activities.extend(discover_probes("chaosaws.xray.probes"))
    return activities