import random
import os
from words import word_list
import time

def get_word():
    word = random.choice(word_list)
    return word.upper()

def clear():
    os.system('clear')

def build_gallows():
    gallows = []
    gallows.append('                   --------')
    gallows.append('                   |      |')# |
    gallows.append('                   |     ') #v'O'v
    gallows.append('                   |     ') # \|/
    gallows.append('                   |     ')#   |
    gallows.append('                   |     ') # / \
    gallows.append('                   |    ')#  ^   ^
    gallows.append('                   |')
    gallows.append('                   |')
    gallows.append('                ---------------')
    gallows.append('                |               |')
    return gallows

def display_gallows(gallows, guessed_letters, missing_letters):
    gallows[-2] = f"               --------------"
    gallows[-1] = f"               | {guessed_letters:<10} |"
    for row in gallows:
        print(row)
    print(f"{len(missing_letters)} {missing_letters}")
    #[print(row) for row in gallows]

def update_gallows(mistake_count, gallows):
    if mistake_count == 1:
        gallows[2]+=' O' # head
    if mistake_count == 2:#2:
        gallows[3]+=' |' # body
        gallows[4]+=' |'
    if mistake_count == 3: #3:
        gallows[3] +='/' #arm
    if mistake_count == 4:
        gallows[3] = '                   |     \\|/'
    if mistake_count == 5:
        gallows[5] += '/'
    if mistake_count == 6:
        gallows[5] += ' \\'
    if mistake_count == 7:
        gallows[2] += ' v'
    if mistake_count == 8:
        gallows[2] = '                   |    v O v'
    if mistake_count == 9:
        gallows[6] += '^'
    if mistake_count == 10:
        gallows[6] += '   ^'
    if mistake_count == 11:
        gallows[2] = '                   |    v x v'
        gallows.insert(1, gallows[1])
        gallows.insert(1, gallows[1])
        gallows.pop(9)
        gallows.pop(9)
    #make the guy fall if mistake_count == -1
    return gallows

def update_board(missing_letters, word, guess):
    missing_letters_list = list(missing_letters)
    indices = [i for i,l in enumerate(word) if l == guess]
    for i in indices:
        missing_letters_list[i] = guess
    missing_letters = ''.join(missing_letters_list)

    return missing_letters

def main():
    clear()
    gallows = build_gallows()
    guessed_letters = ''
    word = get_word()
    missing_letters = "*" * len(word)
    display_gallows(gallows, guessed_letters, missing_letters)
    i = 0
    # for i in range(11):
    while '*' in missing_letters:
        guess = input("Enter the next guess: ").upper()
        if guess in missing_letters or guess in guessed_letters:
            print(f"You already guessed {guess}.  Try again.")
            continue
        clear()
        if guess in word:
            missing_letters = update_board(missing_letters, word, guess)
        else:
            guessed_letters+=guess
            i += 1
            gallows = update_gallows(i, gallows)
        display_gallows(gallows, guessed_letters, missing_letters)

        if i == 11:
            print(f'You lost!  The word was {word}')
            return
    print(f"Good Job!  You got it!")

if __name__ == "__main__":
    main()


from __future__ import print_function

from future import standard_library

from CONSTANT import TUMBLR_CLIENT_SECRET, TUMBLR_CLIENT_ID, TUMBLR_REDIRECT_URL
from SocialMedia.SocialMedia import SocialMedia
from SocialMedia.Tumblr.TumblrAuth import get_authorization_url, get_access_token_from_url

standard_library.install_aliases()
import pytumblr

# HTML and MD are supported by Tumblr

class Tumblr(SocialMedia):
    request_token_url = 'http://www.tumblr.com/oauth/request_token'
    authorize_url = 'http://www.tumblr.com/oauth/authorize'
    access_token_url = 'http://www.tumblr.com/oauth/access_token'

    # TODO Make all extra function parameters same
    def __init__(self, client_id, client_secret, oauth_token, oauth_token_secret):
        self.client_id = client_id
        self.client_secret = client_secret
        self.oauth_token = oauth_token
        self.oauth_token_secret = oauth_token_secret

        self.post_id = None
        self.blog_name = None

        self.tumblr_api = pytumblr.TumblrRestClient(
            self.client_id,
            self.client_secret,
            self.oauth_token,
            self.oauth_token_secret
        )

    def publish_update(self, message, blog_name="", title=""):
        self.post_id = self.tumblr_api.create_text(blog_name,
                                                   state="published",
                                                   slug="testing-text-posts",
                                                   title=title,
                                                   body=message)
        self.blog_name = blog_name

        print("Tumblr Post ID:", self.post_id)
        post_url = self.get_link_latest_post()
        print('Tumblr Post URL:', post_url)

    def publish_update_with_attachment(self, body="", title="", url="", blog_name=""):
        self.post_id = self.tumblr_api.create_link(blog_name, title=title, url=url,
                                                   description=body)
        self.blog_name = blog_name

        print("Tumblr Post ID:", self.post_id)

        post_url = self.get_link_latest_post()
        print('Tumblr Post URL:', post_url)

    def publish_update_with_image_attachment(self, caption="", image_links=None,
                                             tags=None, blog_name="",
                                             format="markdown",
                                             state="published"):
        # https://github.com/tumblr/pytumblr#creating-a-photo-post
        # image_links here is a list of all the images to be posted
        self.blog_name = blog_name

        if tags is None:
            tags = [""]
        self.post_id = self.tumblr_api.create_photo(blog_name,
                                                    state=state, tags=tags, format=format,
                                                    data=image_links,
                                                    caption=caption)
        print("Tumblr Post ID:", self.post_id)

        post_url = self.get_link_latest_post()
        print('Tumblr Post URL:', post_url)

    def get_link_latest_post(self, post_id=None):
        try:
            if post_id is None:
                post_id = str(self.post_id['id'])
                post_url = 'https://' + self.blog_name + '.tumblr.com/post/' + post_id
                return post_url
        except (TypeError, KeyError) as e:
            print(e)
            return ""

    def get_profile_name(self):
        user = self.tumblr_api.info()['user']
        return user['name']

if __name__ == '__main__':
    url, res_key, res_sec = get_authorization_url(TUMBLR_CLIENT_ID,
                                                  TUMBLR_CLIENT_SECRET,
                                                  TUMBLR_REDIRECT_URL)
    print(url)
    tokens = get_access_token_from_url(TUMBLR_CLIENT_ID, TUMBLR_CLIENT_SECRET,
                                       res_key, res_sec,
                                       input('Allow then paste the full redirect URL here:\n'),
                                       callback_url=TUMBLR_REDIRECT_URL)
    tumblr = Tumblr(
        tokens['consumer_key'],
        tokens['consumer_secret'],
        tokens['oauth_token'],
        tokens['oauth_token_secret'],
    )

    # Post Status working
    tumblr.publish_update(
        message="""###Body Title""",
        title="Tit1le",
        blog_name='pymultiposter1')

    # Post Image working
    tumblr.publish_update_with_image_attachment(
        caption="""###Body Title""",
        image_links=["temp1.png", "temp3.png"],
        blog_name='pymultiposter1')

    # Post Link
    tumblr.publish_update_with_attachment(body="""###Body Title""",
                                          title="#Tile",
                                          url="www.googlel.com",
                                          blog_name="pymultiposter1")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Sep 30 11:53:43 2020

@author: zzhajun
"""

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.spatial.distance import cdist
from sklearn.metrics import make_scorer, roc_curve
from scipy.optimize import brentq
from scipy.interpolate import interp1d

#Problem 1.a
train=pd.read_csv("train.csv")
x_test_image=pd.read_csv("test.csv")

#Problem 1.b
x_train_image=train.drop('label',axis=1)#dataframe with index
y_train_label=train.label

X_train=x_train_image.values.reshape(42000,28,28).astype('float32')
X_test=x_test_image.values.reshape(28000,28,28).astype('float32')

x_train_data = np.array(x_train_image)#array 42000*784

images=[]
labels=[]
num=[1,0,16,7,3,8,21,6,10,11]
for i in range(10):
    images.append(X_train[num[i]])
    labels.append(y_train_label[num[i]])

def plot_images_labels(images,labels,idx,num=10):
    fig=plt.gcf()
    fig.set_size_inches(12,14)
    for i in range(0,10):
        ax=plt.subplot(5,5,i+1)
        ax.imshow(images[idx],cmap='binary')
        title='label='+str(labels[idx])

        ax.set_title(title,fontsize=10)
        ax.set_xticks([]);ax.set_yticks([])
        idx+=1
    plt.show()

plot_images_labels(images,labels,idx=0)

#Problem 1.c
count=np.zeros((10,), dtype=np.int)
prob=np.zeros((10,))
for i in range(len(y_train_label)):
    for j in range(10):
        if y_train_label[i]==j:
            count[j]+=1
print(count)

for i in range(10):
    prob[i]=count[i]/42000
    print(prob[i])
plt.hist(x = y_train_label, density=True, bins = 10, color = 'steelblue', edgecolor = 'black' )

plt.xlabel('number')
plt.ylabel('probability')
plt.title('prior probability')
plt.show()

#Problem 1d
a=0
dist=[10000,10000,10000,10000,10000,10000,10000,10000,10000,10000]
match=np.zeros(10)
for i in range(10):
    for j in range(42000):
        if j!=num[i]:
            a=np.linalg.norm(images[i]-X_train[j])
            if dist[i]>a:
                dist[i]=a
                match[i]=y_train_label[j]

print(dist)
print(match)

#Problem 1.e
num0=[]
image0=[]
total0=0
num1=[]
image1=[]
total1=0
for i in range(42000):
    if y_train_label[i]==0:
        num0.append(y_train_label[i])
        image0.append(x_train_data[i])
        total0+=1
    elif y_train_label[i]==1:
        num1.append(y_train_label[i])
        image1.append(x_train_data[i])
        total1+=1
#print(total0,total1)
dist00=cdist(image0,image0,'euclidean')
dist11=cdist(image1,image1,'euclidean')
dist01=cdist(image0,image1,'euclidean')

dist00=dist00.reshape(1,4132*4132)
dist11=dist11.reshape(1,4684*4684)
dist01=dist01.reshape(1,4132*4684)

dist00=np.array(dist00)
dist00=dist00.tolist()

dist11=np.array(dist11)
dist11=dist11.tolist()

dist01=np.array(dist01)
dist01=dist01.tolist()

dist_genu=dist00[0]+dist11[0]
dist_impo=dist01[0]+dist01[0]

sns.distplot(dist_genu, bins = 100, kde = False, hist_kws = {'color':'steelblue'}, label = 'genuine')
sns.distplot(dist_impo, bins = 100, kde = False, hist_kws = {'color':'purple'}, label = 'impostor')
plt.title('distance')
plt.legend()
plt.show()

#Problem 1.f

genu_min=min(dist_genu)
genu_max=max(dist_genu)
impo_min=min(dist_impo)
impo_max=max(dist_impo)
dist_max=max(genu_max,impo_max)
distmax=[]
distmax.append(dist_max)
print(dist_max)#max distance

inter=60
count_tp=np.zeros(60)
count_fp=np.zeros(60)

#TPR
for j in range(60):
    for i in range(39013280):
        if dist_genu[i]<distmax[0]-inter*j:
            count_tp[j]+=1
count_tp=count_tp/ 39013280

#FPr
for j in range(60):
    for i in range(38708576):
        if dist_impo[i]<distmax[0]-inter*j:
            count_fp[j]+=1
count_fp=count_fp/ 38708576
e=0.001
for i in range(60):
    if (abs(count_tp[i]+count_fp[i]-1)<e):
        print('ERR=',count_tp[i])

plt.plot(count_fp,count_tp,label='ROC')
plt.xlabel('FPR')
plt.ylabel('TPR')
plt.legend()
plt.show()


import torch

class OperationNumber():
    """
    统计网络处理一个数据所需的加乘操作数量
    """
    def __init__(self, net):
        """
        :param net: 已经加载预训练参数的网络模型
        这个待处理的网络输入的每一个 batch 包含不同的两部分图像，前一半是含噪图像数据 x + e，后一半是原始干净图像数据 x
        """
        self.net = net
        self.feature_name_list = list()
        self.layer_name_list = list()

        self.op_number_i_list = list()   # 保存当前层所需的操作数量

    def _is_leaf(self, module):
        """
        判断网络的一个模块是否是叶子节点，即层 layer
        :param module:
        :return:
        """
        if sum(1 for x in module.children()) == 0:
            return True
        elif module._get_name().startswith('BlurPool'):
            return True
        else:
            sum_padding = 0
            for name, layer in module.named_children():
                if 'padding' in name:
                    sum_padding += 1
            if sum_padding == sum(1 for x in module.children()):
                return True
        return False

    def hook_all_layers(self):
        """
        :return:
        """
        hook_OrderedDict = list()

        def hook(module, inputdata, output):
            """
            钩子函数，计算统计量
            :param module: 模块，层
            :param inputdata: 输入数据，元组，长度可能不是 1，保存张量
            :param output: 输出数据，元组
            :return:
            """
            m_name = module._get_name()
            if m_name.startswith('Conv'):   # 层类型，若是卷积保存其步长数
                self.layer_name_list.append(m_name+'_i{}o{}k{}s{}'.format(module.in_channels, module.out_channels, module.kernel_size[0], module.stride[0]))
            elif m_name.startswith('MaxPool') or m_name.startswith('AvgPool'):
                self.layer_name_list.append(m_name + '_k{}s{}'.format(module.kernel_size, module.stride))
            else:
                self.layer_name_list.append(module._get_name())
            #如果当前模块是叶子节点，则计算其所需的操作数量
            op_no = get_operation_NO(module, inputdata, output)
            self.op_number_i_list.append(op_no)

        def the_leaf(model, pre_name=''):
            """
            通过递归形式保存网络中所有层 layer 的名称，并给每一层后添加钩子函数
            :param model:
            :param pre_name:
            :return:
            """
            for index, param in enumerate(model.named_children()):                          # 逐个处理网络中的每一个模块
                name, layer = param
                name = name if pre_name == '' else pre_name + '.' + name
                if self._is_leaf(layer):                                                    # 若是层 layer
                    if layer._get_name().startswith('Dropout') or layer._get_name().startswith('Lambda'):
                        continue
                    hook_OrderedDict.append(layer.register_forward_hook(hook))              # 添加钩子函数
                    self.feature_name_list.append(name)                                     # 保存层名称
                else:
                    the_leaf(layer, pre_name = name)
        the_leaf(self.net, pre_name = '')

    def run(self, shape = (1,3,224,224)):
        input = torch.randn(shape)
        self.hook_all_layers()
        self.net(input)
        assert len(self.layer_name_list) == len(self.op_number_i_list), \
            'len(self.layer_name_list) = {}, len(self.op_number_i_list) = {}'\
                .format(len(self.layer_name_list),len(self.op_number_i_list))
        sum_op = 0
        for index, name in enumerate(self.layer_name_list):
            #feature_name = self.feature_name_list[index]
            text = '{} ==> {}'.format(name.ljust(48), self.op_number_i_list[index])
            sum_op += self.op_number_i_list[index]
            print(text)
        print('-'*100)
        print('-'*100)
        print('网络所需加乘操作数量总数为：{} ==> {} K ==> {} M ==> {} G'.
              format(sum_op, sum_op / (10**3), sum_op / (10**6), sum_op / (10**9)))

    def counter(self):
        params = list(self.net.parameters())
        k = 0
        for i in params:
            l = 1
            #print("该层的结构：" + str(list(i.size())))
            for j in i.size():
                l *= j
            #print("该层参数和：" + str(l))
            k = k + l
        print("总参数数量和：{} ==> {} K ==> {} M".format(k, k / 1000, k / 1000 / 1000))

def get_operation_NO(layer, inputdata, output):
    """
    计算 module 处理 inputdata 所需的加乘操作数量
    :param layer: module
    :param inputdata: input data
    :return:
    """
    type_name = layer._get_name()
    #print(type_name)
    if type_name in ['Conv2d']:
        x = inputdata[0]
        out_h = int((x.size()[2] + 2 * layer.padding[0] - layer.kernel_size[0]) // layer.stride[0] + 1)
        out_w = int((x.size()[3] + 2 * layer.padding[1] - layer.kernel_size[1]) // layer.stride[1] + 1)
        delta_ops = layer.in_channels * layer.out_channels * layer.kernel_size[0] * \
                    layer.kernel_size[1] * out_h * out_w // layer.groups

    elif type_name in ['DWT_3D_tiny']:
        x = inputdata[0]
        channels = x.size()[1]
        out_d = int((x.size()[2] + layer.kernel_size // 2) // layer.stride + 1)
        out_h = int((x.size()[3] + layer.kernel_size // 2) // layer.stride + 1)
        out_w = int((x.size()[4] + layer.kernel_size // 2) // layer.stride + 1)
        delta_ops = layer.kernel_size ** 3 * channels * out_d * out_h * out_w

    elif type_name in ['DWT_3D']:
        x = inputdata[0]
        channels = x.size()[1]
        out_d = int((x.size()[2] + layer.kernel_size // 2) // layer.stride + 1)
        out_h = int((x.size()[3] + layer.kernel_size // 2) // layer.stride + 1)
        out_w = int((x.size()[4] + layer.kernel_size // 2) // layer.stride + 1)
        delta_ops = 8 * layer.kernel_size ** 3 * channels * out_d * out_h * out_w

    elif type_name in ['IDWT_3D']:
        x = inputdata[0]
        channels = x.size()[1]
        out_d = int((x.size()[2] + layer.kernel_size // 2) + 1)
        out_h = int((x.size()[3] + layer.kernel_size // 2) + 1)
        out_w = int((x.size()[4] + layer.kernel_size // 2) + 1)
        delta_ops = 8 * layer.kernel_size ** 3 * channels * out_d * out_h * out_w

    elif type_name in ['Hardshrink']:
        x = inputdata[0]
        delta_ops = x.numel()

    elif type_name in ['Conv3d',]:
        x = inputdata[0]
        out_d = int((x.size()[2] + 2 * layer.padding[0] - layer.kernel_size[0]) // layer.stride[0] + 1)
        out_h = int((x.size()[3] + 2 * layer.padding[1] - layer.kernel_size[1]) // layer.stride[1] + 1)
        out_w = int((x.size()[4] + 2 * layer.padding[2] - layer.kernel_size[2]) // layer.stride[2] + 1)
        delta_ops = layer.in_channels * layer.out_channels * layer.kernel_size[0] * \
                    layer.kernel_size[1] * layer.kernel_size[2] * out_d * out_h * out_w // layer.groups

    elif type_name in ['Conv3d']:
        #print(type(inputdata), len(inputdata))
        #print(type(output), len(output))
        x = inputdata[0]
        #print(x.size())
        out_d = int((x.size()[2] + 2 * layer.padding[0] - layer.kernel_size[0]) // layer.stride[0] + 1)
        out_h = int((x.size()[3] + 2 * layer.padding[1] - layer.kernel_size[1]) // layer.stride[1] + 1)
        out_w = int((x.size()[4] + 2 * layer.padding[2] - layer.kernel_size[2]) // layer.stride[2] + 1)
        delta_ops = layer.in_channels * layer.out_channels * layer.kernel_size[0] * \
                    layer.kernel_size[1] * layer.kernel_size[2] * out_d * out_h * out_w // layer.groups

    ### ops_nonlinearity
    elif type_name in ['ReLU']:
        x = inputdata[0]
        delta_ops = x.numel()

    ### ops_pooling
    elif type_name in ['AvgPool2d']:
        x = inputdata[0]
        in_w = x.size()[2]
        in_h = x.size()[3]
        kernel_ops = layer.kernel_size * layer.kernel_size
        out_w = int((in_w + 2 * layer.padding - layer.kernel_size) // layer.stride + 1)
        out_h = int((in_h + 2 * layer.padding - layer.kernel_size) // layer.stride + 1)
        delta_ops = x.size()[1] * out_w * out_h * kernel_ops
    ### ops_pooling
    elif type_name in ['AvgPool3d', 'MaxPool3d', ]:
        x = inputdata[0]
        in_d = x.size()[2]
        in_w = x.size()[3]
        in_h = x.size()[4]
        kernel_ops = layer.kernel_size * layer.kernel_size * layer.kernel_size
        out_d = int((in_d + 2 * layer.padding - layer.kernel_size) // layer.stride + 1)
        out_w = int((in_w + 2 * layer.padding - layer.kernel_size) // layer.stride + 1)
        out_h = int((in_h + 2 * layer.padding - layer.kernel_size) // layer.stride + 1)
        delta_ops = x.size()[1] * out_d * out_w * out_h * kernel_ops

    elif type_name in ['AdaptiveAvgPool2d', 'AdaptiveAvgPool3d']:
        x = inputdata[0]
        delta_ops = x.numel()

    elif type_name in ['Linear']:
        weight_ops = layer.weight.numel()
        bias_ops = layer.bias.numel()
        delta_ops = weight_ops + bias_ops

    elif type_name in ['BatchNorm2d', 'BatchNorm3d']:
        x = inputdata[0]
        normalize_ops = x.numel()
        scale_shift = normalize_ops
        delta_ops = normalize_ops + scale_shift

    ### ops_nothing
    elif type_name in ['Dropout2d', 'DropChannel', 'Dropout']:
        delta_ops = 0

    elif type_name in ['SpatialGate_CT_C']:
        x = inputdata[0]
        delta_ops = 2 * x.numel()

    elif type_name in ['DWT2_2D']:
        x = inputdata[0]
        channels = x.size()[1]
        out_h = int((x.size()[2] + 2 * layer.pad_sizes[0] - layer.kernel_size) // layer.stride + 1)
        out_w = int((x.size()[3] + 2 * layer.pad_sizes[1] - layer.kernel_size) // layer.stride + 1)
        delta_ops = channels * layer.kernel_size * layer.kernel_size * out_h * out_w // layer.groups

    ### unknown layer type
    else:
        delta_ops = 0
        print('unknown layer type: %s' % type_name)

    return delta_ops

if __name__ == '__main__':
    from networks.neuron_net import Neuron_WaveSNet_V1, Neuron_WaveSNet_V2, Neuron_WaveSNet_V3, Neuron_WaveSNet_V4
    from networks.neuron_net import Neuron_SegNet, Neuron_UNet_V1, Neuron_UNet_V2
    net = Neuron_UNet_V1()
    net = Neuron_SegNet()
    #net = Neuron_WaveSNet_V4(wavename = 'db2')
    op_no = OperationNumber(net)
    op_no.run(shape = (1,1,32,128,128))
    op_no.counter()


import re

import numpy
import pandas
import seaborn
from matplotlib import pyplot

from scvae.analyses.figures import saving, style
from scvae.analyses.figures.utilities import _axis_label_for_symbol
from scvae.utilities import normalise_string, capitalise_string

def plot_learning_curves(curves, model_type, epoch_offset=0, name=None):

    figure_name = "learning_curves"
    figure_name = saving.build_figure_name("learning_curves", name)

    x_label = "Epoch"
    y_label = "Nat"

    if model_type == "AE":
        figure = pyplot.figure()
        axis_1 = figure.add_subplot(1, 1, 1)
    elif model_type == "VAE":
        figure, (axis_1, axis_2) = pyplot.subplots(
            nrows=2, sharex=True, figsize=(6.4, 9.6))
        figure.subplots_adjust(hspace=0.1)
    elif model_type == "GMVAE":
        figure, (axis_1, axis_2, axis_3) = pyplot.subplots(
            nrows=3, sharex=True, figsize=(6.4, 14.4))
        figure.subplots_adjust(hspace=0.1)

    for curve_set_name, curve_set in sorted(curves.items()):

        if curve_set_name == "training":
            line_style = "solid"
            colour_index_offset = 0
        elif curve_set_name == "validation":
            line_style = "dashed"
            colour_index_offset = 1

        def curve_colour(i):
            return style.STANDARD_PALETTE[
                len(curves) * i + colour_index_offset]

        for curve_name, curve in sorted(curve_set.items()):
            if curve is None:
                continue
            elif curve_name == "lower_bound":
                curve_name = "$\\mathcal{L}$"
                colour = curve_colour(0)
                axis = axis_1
            elif curve_name == "reconstruction_error":
                curve_name = "$\\log p(x|z)$"
                colour = curve_colour(1)
                axis = axis_1
            elif "kl_divergence" in curve_name:
                if curve_name == "kl_divergence":
                    index = ""
                    colour = curve_colour(0)
                    axis = axis_2
                else:
                    latent_variable = curve_name.replace("kl_divergence_", "")
                    latent_variable = re.sub(
                        pattern=r"(\w)(\d)",
                        repl=r"\1_\2",
                        string=latent_variable
                    )
                    index = "$_{" + latent_variable + "}$"
                    if latent_variable in ["z", "z_2"]:
                        colour = curve_colour(0)
                        axis = axis_2
                    elif latent_variable == "z_1":
                        colour = curve_colour(1)
                        axis = axis_2
                    elif latent_variable == "y":
                        colour = curve_colour(0)
                        axis = axis_3
                curve_name = "KL" + index + "$(q||p)$"
            elif curve_name == "log_likelihood":
                curve_name = "$L$"
                axis = axis_1
            epochs = numpy.arange(len(curve)) + epoch_offset + 1
            label = "{} ({} set)".format(curve_name, curve_set_name)
            axis.plot(
                epochs,
                curve,
                color=colour,
                linestyle=line_style,
                label=label
            )

    handles, labels = axis_1.get_legend_handles_labels()
    labels, handles = zip(*sorted(zip(labels, handles), key=lambda t: t[0]))

    axis_1.legend(handles, labels, loc="best")

    if model_type == "AE":
        axis_1.set_xlabel(x_label)
        axis_1.set_ylabel(y_label)
    elif model_type == "VAE":
        handles, labels = axis_2.get_legend_handles_labels()
        labels, handles = zip(*sorted(
            zip(labels, handles), key=lambda t: t[0]))
        axis_2.legend(handles, labels, loc="best")
        axis_1.set_ylabel("")
        axis_2.set_ylabel("")

        if model_type == "GMVAE":
            axis_3.legend(loc="best")
            handles, labels = axis_3.get_legend_handles_labels()
            labels, handles = zip(*sorted(
                zip(labels, handles), key=lambda t: t[0]))
            axis_3.legend(handles, labels, loc="best")
            axis_3.set_xlabel(x_label)
            axis_3.set_ylabel("")
        else:
            axis_2.set_xlabel(x_label)
        figure.text(-0.01, 0.5, y_label, va="center", rotation="vertical")

    seaborn.despine()

    return figure, figure_name

def plot_separate_learning_curves(curves, loss, name=None):

    if not isinstance(loss, list):
        losses = [loss]
    else:
        losses = loss

    if not isinstance(name, list):
        names = [name]
    else:
        names = name

    names.extend(losses)
    figure_name = saving.build_figure_name("learning_curves", names)

    x_label = "Epoch"
    y_label = "Nat"

    figure = pyplot.figure()
    axis = figure.add_subplot(1, 1, 1)
    seaborn.despine()

    for curve_set_name, curve_set in sorted(curves.items()):

        if curve_set_name == "training":
            line_style = "solid"
            colour_index_offset = 0
        elif curve_set_name == "validation":
            line_style = "dashed"
            colour_index_offset = 1

        def curve_colour(i):
            return style.STANDARD_PALETTE[
                len(curves) * i + colour_index_offset]

        for curve_name, curve in sorted(curve_set.items()):
            if curve is None or curve_name not in losses:
                continue
            elif curve_name == "lower_bound":
                curve_name = "$\\mathcal{L}$"
                colour = curve_colour(0)
            elif curve_name == "reconstruction_error":
                curve_name = "$\\log p(x|z)$"
                colour = curve_colour(1)
            elif "kl_divergence" in curve_name:
                if curve_name == "kl_divergence":
                    index = ""
                    colour = curve_colour(0)
                else:
                    latent_variable = curve_name.replace("kl_divergence_", "")
                    latent_variable = re.sub(
                        pattern=r"(\w)(\d)",
                        repl=r"\1_\2",
                        string=latent_variable
                    )
                    index = "$_{" + latent_variable + "}$"
                    if latent_variable in ["z", "z_2"]:
                        colour = curve_colour(0)
                    elif latent_variable == "z_1":
                        colour = curve_colour(1)
                    elif latent_variable == "y":
                        colour = curve_colour(0)
                curve_name = "KL" + index + "$(q||p)$"
            elif curve_name == "log_likelihood":
                curve_name = "$L$"
            epochs = numpy.arange(len(curve)) + 1
            label = curve_name + " ({} set)".format(curve_set_name)
            axis.plot(
                epochs,
                curve,
                color=colour,
                linestyle=line_style,
                label=label
            )

    handles, labels = axis.get_legend_handles_labels()
    labels, handles = zip(*sorted(zip(labels, handles), key=lambda t: t[0]))

    axis.legend(handles, labels, loc="best")

    axis.set_xlabel(x_label)
    axis.set_ylabel(y_label)

    return figure, figure_name

def plot_accuracy_evolution(accuracies, name=None):

    figure_name = saving.build_figure_name("accuracies", name)
    figure = pyplot.figure()
    axis = figure.add_subplot(1, 1, 1)
    seaborn.despine()

    for accuracies_kind, accuracies in sorted(accuracies.items()):
        if accuracies is None:
            continue
        elif accuracies_kind == "training":
            line_style = "solid"
            colour = style.STANDARD_PALETTE[0]
        elif accuracies_kind == "validation":
            line_style = "dashed"
            colour = style.STANDARD_PALETTE[1]

        label = "{} set".format(capitalise_string(accuracies_kind))
        epochs = numpy.arange(len(accuracies)) + 1
        axis.plot(
            epochs,
            100 * accuracies,
            color=colour,
            linestyle=line_style,
            label=label
        )

    handles, labels = axis.get_legend_handles_labels()
    labels, handles = zip(*sorted(zip(labels, handles), key=lambda t: t[0]))

    axis.legend(handles, labels, loc="best")

    axis.set_xlabel("Epoch")
    axis.set_ylabel("Accuracies")

    return figure, figure_name

def plot_kl_divergence_evolution(kl_neurons, scale="log", name=None):

    figure_name = saving.build_figure_name("kl_divergence_evolution", name)
    n_epochs, __ = kl_neurons.shape

    kl_neurons = numpy.sort(kl_neurons, axis=1)

    if scale == "log":
        kl_neurons = numpy.log(kl_neurons)
        scale_label = "$\\log$ "
    else:
        scale_label = ""

    figure = pyplot.figure()
    axis = figure.add_subplot(1, 1, 1)

    cbar_dict = {"label": scale_label + "KL$(p_i|q_i)$"}

    number_of_epoch_labels = 10
    if n_epochs > 2 * number_of_epoch_labels:
        epoch_label_frequency = int(numpy.floor(
            n_epochs / number_of_epoch_labels))
    else:
        epoch_label_frequency = True

    epochs = numpy.arange(n_epochs) + 1

    seaborn.heatmap(
        pandas.DataFrame(kl_neurons.T, columns=epochs),
        xticklabels=epoch_label_frequency,
        yticklabels=False,
        cbar=True, cbar_kws=cbar_dict, cmap=style.STANDARD_COLOUR_MAP,
        ax=axis
    )

    axis.set_xlabel("Epochs")
    axis.set_ylabel("$i$")

    seaborn.despine(ax=axis)

    return figure, figure_name

def plot_centroid_probabilities_evolution(probabilities, distribution,
                                          linestyle="solid", name=None):

    distribution = normalise_string(distribution)

    y_label = _axis_label_for_symbol(
        symbol="\\pi",
        distribution=distribution,
        suffix="^k"
    )

    figure_name = "centroids_evolution-{}-probabilities".format(distribution)
    figure_name = saving.build_figure_name(figure_name, name)

    n_epochs, n_centroids = probabilities.shape

    centroids_palette = style.darker_palette(n_centroids)
    epochs = numpy.arange(n_epochs) + 1

    figure = pyplot.figure()
    axis = figure.add_subplot(1, 1, 1)
    seaborn.despine()

    for k in range(n_centroids):
        axis.plot(
            epochs,
            probabilities[:, k],
            color=centroids_palette[k],
            linestyle=linestyle,
            label="$k = {}$".format(k)
        )

    axis.set_xlabel("Epochs")
    axis.set_ylabel(y_label)

    axis.legend(loc="best")

    return figure, figure_name

def plot_centroid_means_evolution(means, distribution, decomposed=False,
                                  name=None):

    symbol = "\\mu"
    if decomposed:
        decomposition_method = "PCA"
    else:
        decomposition_method = ""
    distribution = normalise_string(distribution)
    suffix = "(y = k)"

    x_label = _axis_label_for_symbol(
        symbol=symbol,
        coordinate=1,
        decomposition_method=decomposition_method,
        distribution=distribution,
        suffix=suffix
    )
    y_label = _axis_label_for_symbol(
        symbol=symbol,
        coordinate=2,
        decomposition_method=decomposition_method,
        distribution=distribution,
        suffix=suffix
    )

    figure_name = "centroids_evolution-{}-means".format(distribution)
    figure_name = saving.build_figure_name(figure_name, name)

    n_epochs, n_centroids, latent_size = means.shape

    if latent_size > 2:
        raise ValueError("Dimensions of means should be 2.")

    centroids_palette = style.darker_palette(n_centroids)
    epochs = numpy.arange(n_epochs) + 1

    figure = pyplot.figure()
    axis = figure.add_subplot(1, 1, 1)
    seaborn.despine()

    colour_bar_scatter_plot = axis.scatter(
        means[:, 0, 0], means[:, 0, 1], c=epochs,
        cmap=seaborn.dark_palette(style.NEUTRAL_COLOUR, as_cmap=True),
        zorder=0
    )

    for k in range(n_centroids):
        colour = centroids_palette[k]
        colour_map = seaborn.dark_palette(colour, as_cmap=True)
        axis.plot(
            means[:, k, 0],
            means[:, k, 1],
            color=colour,
            label="$k = {}$".format(k),
            zorder=k + 1
        )
        axis.scatter(
            means[:, k, 0],
            means[:, k, 1],
            c=epochs,
            cmap=colour_map,
            zorder=n_centroids + k + 1
        )

    axis.legend(loc="best")

    colour_bar = figure.colorbar(colour_bar_scatter_plot)
    colour_bar.outline.set_linewidth(0)
    colour_bar.set_label("Epochs")

    axis.set_xlabel(x_label)
    axis.set_ylabel(y_label)

    return figure, figure_name

def plot_centroid_covariance_matrices_evolution(covariance_matrices,
                                                distribution, name=None):

    distribution = normalise_string(distribution)
    figure_name = "centroids_evolution-{}-covariance_matrices".format(
        distribution)
    figure_name = saving.build_figure_name(figure_name, name)

    y_label = _axis_label_for_symbol(
        symbol="\\Sigma",
        distribution=distribution,
        prefix="|",
        suffix="(y = k)|"
    )

    n_epochs, n_centroids, __, __ = covariance_matrices.shape
    determinants = numpy.empty([n_epochs, n_centroids])

    for e in range(n_epochs):
        for k in range(n_centroids):
            determinants[e, k] = numpy.prod(numpy.diag(
                covariance_matrices[e, k]))

    if determinants.all() > 0:
        line_range_ratio = numpy.empty(n_centroids)
        for k in range(n_centroids):
            determinants_min = determinants[:, k].min()
            determinants_max = determinants[:, k].max()
            line_range_ratio[k] = determinants_max / determinants_min
        range_ratio = line_range_ratio.max() / line_range_ratio.min()
        if range_ratio > 1e2:
            y_scale = "log"
        else:
            y_scale = "linear"

    centroids_palette = style.darker_palette(n_centroids)
    epochs = numpy.arange(n_epochs) + 1

    figure = pyplot.figure()
    axis = figure.add_subplot(1, 1, 1)
    seaborn.despine()

    for k in range(n_centroids):
        axis.plot(
            epochs,
            determinants[:, k],
            color=centroids_palette[k],
            label="$k = {}$".format(k)
        )

    axis.set_xlabel("Epochs")
    axis.set_ylabel(y_label)

    axis.set_yscale(y_scale)

    axis.legend(loc="best")

    return figure, figure_name


# you can write to stdout for debugging purposes, e.g.
# print("this is a debug message")

def solution(x):
    d=[]
    e=0
    for i in range(1,x):
        if i%3==0 or i%5==0:
            d.append(i)
    return sum(d)


import sys

import tensorflow as tf

from utils import residual_block, downsample_block

# Declaring exception names:
class ConfigurationException(Exception): pass
class IncompleteFeedDict(Exception): pass

# Main class
class resnet(object):
    '''Define a network model and run training

    U resnet implementation
    '''
    def __init__(self, params):
        '''initialization

        Requires a list of parameters as python dictionary

        Arguments:
            params {dict} -- Network parameters

        Raises:
            ConfigurationException -- Missing a required parameter
        '''
        required_params =[
            'MINIBATCH_SIZE',
            'SAVE_ITERATION',
            'NUM_LABELS',
            'N_INITIAL_FILTERS',
            'BATCH_NORMALIZATION',
            'NETWORK_DEPTH',
            'RESIDUAL_BLOCKS_PER_LAYER',
            'LOGDIR',
            'BASE_LEARNING_RATE',
            'TRAINING',
            'RESTORE',
            'TRAINING_ITERATIONS',
        ]

        for param in required_params:
            if param not in params:
                raise ConfigurationException("Missing parameter "+ str(param))

        self._params = params

    def construct_network(self, dims):
        '''Build the network model

        Initializes the tensorflow model according to the parameters
        '''

        tf.reset_default_graph()

        # Initialize the input layers:
        self._input_image  = tf.placeholder(tf.float32, dims, name="input_image")
        self._input_labels = tf.placeholder(tf.int64,
            [dims[0], self._params['NUM_LABELS']], name="input_image")

        logits = self._build_network(self._input_image)

        self._softmax = tf.nn.softmax(logits)
        self._predicted_labels = tf.argmax(logits, axis=-1)

        # Keep a list of trainable variables for minibatching:
        with tf.variable_scope('gradient_accumulation'):
            self._accum_vars = [tf.Variable(tv.initialized_value(),
                                trainable=False) for tv in tf.trainable_variables()]

        # Accuracy calculations:
        with tf.name_scope('accuracy'):
            self._total_accuracy   = tf.reduce_mean(tf.cast(
                tf.equal(self._predicted_labels,
                         tf.argmax(self._input_labels, axis=-1)), tf.float32))

            # Add the accuracies to the summary:
            tf.summary.scalar("Total_Accuracy", self._total_accuracy)

        # Loss calculations:
        with tf.name_scope('cross_entropy'):

            # Unreduced loss, shape [BATCH, L, W]
            losses = tf.nn.softmax_cross_entropy_with_logits(
                labels=self._input_labels, logits=logits)

            self._loss = tf.reduce_mean(losses)

            # Add the loss to the summary:
            tf.summary.scalar("Total_Loss", self._loss)

        # Optimizer:
        with tf.name_scope("training"):
            self._global_step = tf.Variable(0, dtype=tf.int32,
                trainable=False, name='global_step')
            if self._params['BASE_LEARNING_RATE'] <= 0:
                opt = tf.train.AdamOptimizer()
            else:
                opt = tf.train.AdamOptimizer(self._params['BASE_LEARNING_RATE'])

            # Variables for minibatching:
            self._zero_gradients =  [tv.assign(tf.zeros_like(tv)) for tv in self._accum_vars]
            self._accum_gradients = [self._accum_vars[i].assign_add(gv[0]) for
                                     i, gv in enumerate(opt.compute_gradients(self._loss))]
            self._apply_gradients = opt.apply_gradients(zip(self._accum_vars, tf.trainable_variables()),
                global_step = self._global_step)

        # Merge the summaries:
        self._merged_summary = tf.summary.merge_all()

    def apply_gradients(self,sess):

        return sess.run( [self._apply_gradients], feed_dict = {})

    def feed_dict(self, images, labels, weights=None):
        '''Build the feed dict

        Take input images, labels and (optionally) weights and match
        to the correct feed dict tensorrs

        Arguments:
            images {numpy.ndarray} -- Image array, [BATCH, L, W, F]
            labels {numpy.ndarray} -- Label array, [BATCH, L, W, F]

        Keyword Arguments:
            weights {numpy.ndarray} -- (Optional) input weights, same shape as labels (default: {None})

        Returns:
            [dict] -- Feed dictionary for a tf session run call

        Raises:
            IncompleteFeedDict -- If weights are requested in the configuration but not provided.
        '''
        fd = dict()
        fd.update({self._input_image : images})
        if labels is not None:
            fd.update({self._input_labels : labels})

        return fd

    def losses():
        pass

    def make_summary(self, sess, input_data, input_label, input_weight=None):
        fd = self.feed_dict(images  = input_data,
                            labels  = input_label,
                            weights = input_weight)
        return sess.run(self._merged_summary, feed_dict=fd)

    def zero_gradients(self, sess):
        sess.run(self._zero_gradients)

    def accum_gradients(self, sess, input_data, input_label, input_weight=None):

        feed_dict = self.feed_dict(images  = input_data,
                                   labels  = input_label,
                                   weights = input_weight)

        ops = [self._accum_gradients]
        doc = ['']
        # classification
        ops += [self._loss, self._total_accuracy]
        doc += ['loss', 'acc. all']

        return sess.run(ops, feed_dict = feed_dict ), doc

    def run_test(self,sess, input_data, input_label, input_weight=None):
        feed_dict = self.feed_dict(images   = input_data,
                                   labels  = input_label,
                                   weights = input_weight)

        ops = [self._loss, self._total_accuracy,]
        doc = ['loss', 'acc. all']

        return sess.run(ops, feed_dict = feed_dict ), doc

    def inference(self,sess,input_data,input_label=None):

        feed_dict = self.feed_dict(images=input_data, input_label=input_label)

        ops = [self._softmax]
        if input_label is not None:
          ops.append(self._accuracy_allpix)
          ops.append(self._accuracy_nonzero)

        return sess.run( ops, feed_dict = feed_dict )

    def global_step(self, sess):
        return sess.run(self._global_step)

    def _build_network(self, input_placeholder):

        x = input_placeholder
        # Initial convolution to get to the correct number of filters:
        x = tf.layers.conv3d(x, self._params['N_INITIAL_FILTERS'],
                             kernel_size=[5, 5, 5],
                             strides=[1, 1, 1],
                             padding='same',
                             use_bias=False,
                             trainable=self._params['TRAINING'],
                             name="Conv2DInitial",
                             reuse=None)

        # ReLU:
        x = tf.nn.relu(x)

        # Begin the process of residual blocks and downsampling:
        for i in xrange(self._params['NETWORK_DEPTH']):

            for j in xrange(self._params['RESIDUAL_BLOCKS_PER_LAYER']):
                x = residual_block(x, self._params['TRAINING'],
                                   batch_norm=self._params['BATCH_NORMALIZATION'],
                                   name="resblock_down_{0}_{1}".format(i, j))

            x = downsample_block(x, self._params['TRAINING'],
                                batch_norm=self._params['BATCH_NORMALIZATION'],
                                name="downsample_{0}".format(i))

        # At the bottom, do another residual block:
        for j in xrange(self._params['RESIDUAL_BLOCKS_PER_LAYER']):
            x = residual_block(x, self._params['TRAINING'],
                batch_norm=self._params['BATCH_NORMALIZATION'],
                name="deepest_block_{0}".format(j))

        # At this point, we ought to have a network that has the same shape as the initial input, but with more filters.
        # We can use a bottleneck to map it onto the right dimensions:
        x = tf.layers.conv3d(x,
                             self._params['NUM_LABELS'],
                             kernel_size=[1, 1, 1],
                             strides=[1, 1, 1],
                             padding='same',
                             activation=None,
                             use_bias=False,
                             trainable=self._params['TRAINING'],
                             name="BottleneckConv2D",)

        # Apply global average pooling to each filter to get the values for signal/bkg

        # For global average pooling, need to get the shape of the input:
        shape = (x.shape[1], x.shape[2], x.shape[3])

        x = tf.nn.pool(x,
                       window_shape=shape,
                       pooling_type="AVG",
                       padding="VALID",
                       dilation_rate=None,
                       strides=None,
                       name="GlobalAveragePool",
                       data_format=None)

        # Reshape to remove empty dimensions:
        x = tf.reshape(x, [tf.shape(x)[0], self._params['NUM_LABELS']],
                     name="global_pooling_reshape")

        # The final activation is softmax across the pixels.  It gets applied in the loss function
#         x = tf.nn.softmax(x)
        return x


# -*- coding: utf-8 -*-
"""
Created on Fri Apr 20 23:34:30 2018

@author: dell
"""

def transpose(matrix):
    col=len(matrix[0])
    row=len(matrix)
    newmatrix=[]
    for i in range(col):
        newlist=[]
        for j in range (row):
            newlist.append(matrix[j][i])
            j=j+1
        newmatrix.append(newlist)
    return newmatrix

#    b=[[c[i] for c in matrix ] for i in range(col)]
#
##    b=[ [col[row] for col in matrix ] for row in range(col)]
#    return b

matrix = [[1,2,3], [4,5,6]]
a=transpose(matrix)
print (a)


import socket
import sys
import time
new_socket=socket.socket()
host_name=socket.gethostname()
s_ip=socket.gethostbyname(host_name)
port=8080
new_socket.bind((host_name,port))
name=input("Enter name: ")
new_socket.listen(1)
conn,add=new_socket.accept()
print("receved connection from",add[0])
print("connection estabilished from",add[0])
client=(conn.recv(1024)).decode()
conn.send(name.encode())
while True:
    message=input("me")
    conn.send(message.encode())
    message=conn.recv(1024)
    message=message.decode()
    print(client,':',message)


import logging

from ..widgets import (
    ImageWidget,
)
from .managed_window import ManagedWindow

log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())

class ManagedImageWindow(ManagedWindow):
    """
    Display experiment output with an :class:`~pymeasure.display.widgets.image_widget.ImageWidget`
    class.

    :param procedure_class: procedure class describing the experiment (see
        :class:`~pymeasure.experiment.procedure.Procedure`)
    :param x_axis: the data-column for the x-axis of the plot, cannot be changed afterwards for
        the image-plot
    :param y_axis: the data-column for the y-axis of the plot, cannot be changed afterwards for
        the image-plot
    :param z_axis: the initial data-column for the z-axis of the plot, can be changed afterwards
    :param \\**kwargs: optional keyword arguments that will be passed to
        :class:`~pymeasure.display.windows.managed_image_window.ManagedWindow`

    """

    def __init__(self, procedure_class, x_axis, y_axis, z_axis=None, **kwargs):
        self.z_axis = z_axis
        self.image_widget = ImageWidget(
            "Image", procedure_class.DATA_COLUMNS, x_axis, y_axis, z_axis)

        if "widget_list" not in kwargs:
            kwargs["widget_list"] = ()
        kwargs["widget_list"] = kwargs["widget_list"] + (self.image_widget,)

        super().__init__(procedure_class, x_axis=x_axis, y_axis=y_axis, **kwargs)


import board
from digitalio import DigitalInOut, Direction, Pull
import time

# Configuration des entrées sorties
feu1V = DigitalInOut(board.GP18)
feu1V.direction = Direction.OUTPUT
feu1O = DigitalInOut(board.GP17)
feu1O.direction = Direction.OUTPUT
feu1R = DigitalInOut(board.GP16)
feu1R.direction = Direction.OUTPUT
bpiet1 = DigitalInOut(board.GP12)
bpiet1.direction = Direction.INPUT
bpiet1.pull = Pull.UP
bpiet2 = DigitalInOut(board.GP11)
bpiet2.direction = Direction.INPUT
bpiet2.pull = Pull.UP
bmag = DigitalInOut(board.GP13)
bmag.direction = Direction.INPUT
bmag.pull = Pull.UP

# ▓ Initialisation de l'automate
X0 = 1
X10 = 0
X20 = 0
Q1 = False
tempo = 0
while True:
    # Déterminer la valeur de la sortie de la temporisation
    if X10 == 1 :
        if tempo ==0 :
            t =time.monotonic()
            tempo = 1
            Q1 = 0
        if time.monotonic()-t > 3 :
            Q1 =True
            tempo = 0
            print(tempo)

    # Les conditions d'évolution
    CE0_10 = not bpiet1.value and X0
    CE10_20 = Q1 and X10
    CE20_0 = not bmag.value and X20

    # Les étapes
    if CE0_10:
        X10 = 1
        X0 = 0
    if CE10_20:
        X20=1
        X10=0
    if CE20_0:
        X0 = 1
        X20 = 0
    # Les actions
    feu1V.value = X0
    feu1O.value = X10
    feu1R.value = X20


import os
import tempfile
from kubernetes import client, config, watch
from kubernetes.utils import create_from_yaml

APPLY_YAML_DATA =  os.getenv('APPLY_YAML_DATA',
'''
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: istio-gateways-role
  namespace: {namespace}
rules:
- apiGroups:
  - networking.istio.io
  resources:
  - gateways
  verbs:
  - get
  - list
  - create
  - update
  - delete
  - patch
  - watch
''')
# Configs can be set in Configuration class directly or using helper utility
config.load_incluster_config()

v1 = client.CoreV1Api()

while True:
    try:
        w = watch.Watch()
        for event in w.stream(v1.list_namespace, _request_timeout=60):
            if event['type'] == "MODIFIED":
                print("Event: %s %s" % (event['type'], event['object'].metadata.name))
                tmp_data = APPLY_YAML_DATA.replace('{namespace}', event['object'].metadata.name)
                with tempfile.NamedTemporaryFile(dir='/tmp') as tmpfile:
                    tmp_file = tmpfile.name
                    tmpfile.close()
                print(tmp_file)
                f = open(tmp_file, "a")
                f.write(tmp_data)
                f.close()
                clientapi = client.ApiClient()
                create_from_yaml(clientapi, tmp_file, namespace=event['object'].metadata.name)
                os.remove(tmp_file)
    except Exception as e:
        print(e)


import os
from typing import Dict, Any
import json
from deepesn_ner.ner_esn import NerEsn
from deepesn_ner.dataset import EfficientEmbeddedNERDataset, EmbeddedNERDataset
from pytorch_optimize.optimizer import ESOptimizer
from pytorch_optimize.model import Model, SamplingStrategy
from pytorch_optimize.objective import Objective, Samples
import torch
from dataclasses import dataclass
from deepesn_ner.config import esn_embeddings_folder, ner_models_folder
from torch.utils.data import DataLoader
from tqdm import tqdm

@dataclass(init=True)
class BatchSamples(Samples):
    sentences: torch.Tensor
    inputs: torch.Tensor
    targets: torch.Tensor

class Accuracy(Objective):
    def __call__(self, model: Model, samples: EmbeddedNERDataset, device: str):
        esn_net: NerEsn = model.net
        samples = samples.to(device)
        report = esn_net.entity_eval(samples, False)
        return [report["micro f1"]]

def train_esn(esn_name, model_name, train_parameters: Dict[str, Any], bidirectional, device):
    """
    Trains with pre-generated embeddings in embeddings folder
    Returns:

    Args:
        esn_name:
        model_name:
        train_parameters:
        bidirectional:
            if dataset is bidirectional AND:
                * bidirectional = true -> trains model with a bidirectional embedding
                * bidirectional = False -> trains model with only FORWARD embeddings
            if dataset is unidirectional AND:
                * bidirectional = true -> ERROR
                * bidirectional = False -> trains model with the unidirectional embeddings provided

    Returns:

    """
    embd_folder = os.path.join(esn_embeddings_folder, esn_name)
    model_folder = os.path.join(ner_models_folder, esn_name + "--" + model_name)
    if os.path.exists(os.path.join(model_folder, "train_report.json")):
        print("model already trained")
        #return

    if not os.path.exists(model_folder):
        os.makedirs(model_folder)

    # save parameters
    with open(os.path.join(model_folder, "train.0.param"), "w") as f:
        json.dump(train_parameters, f)

    print("loaded res")
    train_ds: EmbeddedNERDataset = EfficientEmbeddedNERDataset.from_pickle(os.path.join(embd_folder, "test.0.pickle")).to_embedded_dataset()
    train_ds = train_ds.to(device)
    print("loaded train ds")
    dev_ds: EfficientEmbeddedNERDataset = EfficientEmbeddedNERDataset.from_pickle(os.path.join(embd_folder, "dev.0.pickle"))
    dev_ds = dev_ds.to(device)
    print("loaded dev ds")
    if train_ds.is_bidirectional != dev_ds.is_bidirectional:
        format_ds = lambda ds: f"{'bidirectional' if ds.is_bidirectional else 'unidirectional'}"
        raise ValueError(f"Datasets dont match: train dataset is {format_ds(train_ds)} while dev ds is"
                         f" {format_ds(dev_ds)}")

    # VERY important: This configures the behaviour of the dataset
    if bidirectional:
        if train_ds.is_bidirectional:
            train_ds.bidirectional_mode()
            dev_ds.bidirectional_mode()
        else:
            raise ValueError("Cannot use a unidirectional dataset for bidirectional training")
    else:
        train_ds.unidirectional_mode()
        dev_ds.unidirectional_mode()

    print("init ner class")
    print(f"train_ds tag mappint:type:{type(train_ds.tag_mapping)}")
    ner = NerEsn(train_ds.embedding_size, train_ds.tag_mapping, device)

    wrapped_ner = Model(ner, SamplingStrategy.ALL)

    #train_report = ner.train_model(train_ds, dev_ds, **train_parameters)
    # objective function (loss function)
    obj_measure = Accuracy()
    #devices=[f"cuda:{inx}" for inx in range(6)]
    devices =["cpu"]
    # optimizer
    es_optimizer = ESOptimizer(model=wrapped_ner, sgd_optimizer=torch.optim.Adadelta(ner.parameters(), lr=1e-3),
                               objective_fn=obj_measure, obj_weights=[1.0], sigma=1e-3, n_samples=100,
                               devices=devices, n_workers=6)

    dataloader = torch.utils.data.DataLoader(train_ds, 300, shuffle=True, collate_fn=lambda xs: (
        [x[0] for x in xs],
        [x[1] for x in xs],
        [x[2] for x in xs]
    ))

    reports = []

    for epoch in range(100):
        show_every = 1
        for i, (sentences, inputs, targets) in enumerate(tqdm(dataloader)):
            samples = EmbeddedNERDataset(sentences, inputs, targets, train_ds.tag_mapping, train_ds.split_name,
                                         train_ds.is_bidirectional)
            es_optimizer.gradient_step(samples)
        if (epoch + 1) % show_every == 0:
            report = ner.entity_eval(dev_ds, False)
            print(report)
            reports.append(report)

    with open("raj_opt_reports.json", "w") as f:
        json.dump(reports, f)

    #with open(os.path.join(model_folder, "train_report.json"), "w") as f:
    #json.dump(train_report, f)

    # saving ner
    torch.save(ner, os.path.join(model_folder, "model.pickle"))

if __name__ == "__main__":
    import multiprocess.context as ctx
    from multiprocessing import set_start_method
    ctx._force_start_method('spawn')
    set_start_method("spawn")

    train_esn("word_gs_2L_bi_:spec_radius0.99-0.99", "raj's_library", None, True, "cpu")


import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
from tensorflow.keras.layers import*
from tensorflow.keras.models import Model

def darknetConv2d_BN_Leakr(x, f_size, k_size, strides=1):
    x = Conv2D(filters=f_size,
               kernel_size=k_size,
               strides=strides,
               padding='same')(x)
    x = BatchNormalization()(x)
    x = LeakyReLU(alpha=0.1)(x)
    return x

def res_unit(x, f_size, k_size):
    x1 = x
    x = darknetConv2d_BN_Leakr(x, f_size // 2, k_size)
    x = darknetConv2d_BN_Leakr(x, f_size, k_size)
    return Add()([x, x1])

def resn(x, f_size, k_size=3, n=1, strides=2):
    x = darknetConv2d_BN_Leakr(x, f_size, k_size, strides=strides)
    for i in range(n):
        x = res_unit(x, f_size, k_size)
    return x

def daeknet_body(inputs=(416,416,3), include_fc=True, classes=1000):
    input_tensor = Input(inputs)
    x = darknetConv2d_BN_Leakr(input_tensor, f_size=32, k_size=3)
    x = resn(x, f_size=64, n=1)
    x = resn(x, f_size=128, n=2)
    x = resn(x, f_size=256, n=8)
    x = resn(x, f_size=512, n=8)
    x = resn(x, f_size=1024, n=4)

    if include_fc:
        x = GlobalAveragePooling2D()(x)
        x = Dense(classes, activation='softmax')(x)
    return Model(input_tensor, x)

if __name__ == '__main__':
    model = daeknet_body()
    model.summary()


import subprocess
import sys
import re

def ssh(command,machines):
    listproc = []
    #timer=5
    login="pmecchia-20"
    delete_list=[]
    for ip in machines:
        proc = subprocess.Popen(["ssh",login+"@"+ip,command],stdin=subprocess.PIPE, stdout = subprocess.PIPE,stderr = subprocess.PIPE)
        listproc.append(proc)

    for i in range(len(listproc)):
        try:
            out, err = listproc[i].communicate(timeout=10)
            code = listproc[i].returncode
            if code!=0:
                print(str(i)+" out: '{}'".format(out))
                print(str(i)+" err: '{}'".format(err))
                print(str(i)+" exit: {}".format(code))
                if code==255: ## nodename or servername not known
                    delete_list.append(i)
        except subprocess.TimeoutExpired:
            listproc[i].kill()
            print(machines[i]+" timeout")
            #print(f"{machines[i]} deleted from list")
            delete_list.append(i)
            #del machines[i]
    for idx,val in enumerate(delete_list):
        #print(val)
        #print("idx",idx)
        del machines[val-idx]
    return machines

def scp(localPath,distantPath,machines):
    listproc = []
    #timer=200
    login="pmecchia-20"
    for ip in machines:
        proc = subprocess.Popen(["scp",localPath,login+"@"+ip+":"+distantPath],stdin=subprocess.PIPE, stdout = subprocess.PIPE,stderr = subprocess.PIPE)
        listproc.append(proc)

    for i in range(len(listproc)):
        try:
            out, err = listproc[i].communicate()
            code = listproc[i].returncode
            if code !=0:
                print(str(i)+" out: '{}'".format(out))
                print(str(i)+" err: '{}'".format(err))
                print(str(i)+" exit: {}".format(code))
        except subprocess.TimeoutExpired:
            listproc[i].kill()
            print(str(i+1)+" timeout")

if __name__ == '__main__':
    nbMachines = int(re.findall(r'\d+',sys.argv[1])[0])
    ip_list=[]
    i=10
    while len(ip_list)<nbMachines:
        machine=f"tp-4b01-{i:02d}"
        ssh_result=ssh("hostname",[machine])
        if ssh_result: ##if not empty list
            ip_list.append(ssh_result[0])
        i+=1
    print("Number of machines:",len(ip_list))
    """
    for i in range(10,nbMachines + 1):
            machine=f"tp-4b01-{i:02d}"
            ip_list.append(machine)
    ip_list=ssh("hostname",ip_list) #keep only working machines
    """
    with open("list_ip.txt", "w") as f:
        for index,machine in enumerate(ip_list):
            #print(len(ip_list))
            if index < len(ip_list)-1:
                f.write(machine+"\n")
            else:
                f.write(machine)

    ip_list=ssh("mkdir -p /tmp/pmecchia-20",ip_list)
    scp("Slave.py", "/tmp/pmecchia-20",ip_list)
    scp("list_ip.txt", "/tmp/pmecchia-20/machines.txt", ip_list)


import torch
import numpy as np
def compute_cmvn_epoch(opt, train_loader, enhance_model, feat_model):
    enhance_model.eval()
    feat_model.eval()
    torch.set_grad_enabled(False)
    ##print(enhance_model.state_dict())
    enhance_cmvn_file = os.path.join(opt.exp_path, 'enhance_cmvn.npy')
    for i, (data) in enumerate(train_loader, start=0):
        utt_ids, spk_ids, clean_inputs, clean_log_inputs, mix_inputs, mix_log_inputs, cos_angles, targets, input_sizes, target_sizes = data
        enhance_out = enhance_model(mix_inputs, mix_log_inputs, input_sizes)
        enhance_cmvn = feat_model.compute_cmvn(enhance_out, input_sizes)
        if enhance_cmvn is not None:
            np.save(enhance_cmvn_file, enhance_cmvn)
            print('save enhance_cmvn to {}'.format(enhance_cmvn_file))
            break
    enhance_cmvn = torch.FloatTensor(enhance_cmvn)
    enhance_model.train()
    feat_model.train()
    torch.set_grad_enabled(True)
    return enhance_cmvn


import flask
from flask import request, jsonify, Response
import sqlite3
import pandas as pd

app = flask.Flask(__name__)

@app.route('/', methods=['GET'])
def home():
	return '''
Capstone API

aguscuk capstone project api
'''

@app.route('/docs', methods=['GET'])
def docs():
	return '''
Capstone API Documentation

Dependencies
\

- Pandas    (pip install pandas)
\

- flask    (pip install flask)
\

- gunicorn    (pip install gunicorn)
\

- requests    (pip install requests)
\

- jsonify    (pip install jsonify)
\

HOW TO

1. / , method = GET

Base Endpoint, returning welcoming string value.

	Here's : https://aguscuk-capstone-api.herokuapp.com

2. /api/v1/resources/<data_name>/all , method = GET

Return full data <data_name> in JSON format. Currently available data are:

- products : https://aguscuk-capstone-api.herokuapp.com/api/v1/resources/products/all

- suppliers : https://aguscuk-capstone-api.herokuapp.com/api/v1/resources/suppliers/all

- customers : https://aguscuk-capstone-api.herokuapp.com/api/v1/resources/customers/all

3. /api/v1/resources/top5_countries/dow , method = GET

Return top5 countries that have total omzet from orders = UnitPrice * Quantity * Discount

 For EDA you can read this : https://github.com/aguscuk/capstone-api/blob/master/capstone-api-markdown.ipynb

 TOP 5 Countries Omzet Total from Orders : https://aguscuk-capstone-api.herokuapp.com/api/v1/resources/top5_countries/dow

4. /api/v1/resources/filter/<values>

Returns full data of orders filter by :

- country=<values>, exp: country=France

- orderdate_start=<values>,  exp: orderdate_start=2012-01-01

- orderdate_end=<values>, exp: orderdate_end=2012-12-31

Filter implementation: https://aguscuk-capstone-api.herokuapp.com/api/v1/resources/filter?country=France&orderdate_start=2012-01-01&orderdate_end=2012-12-31

	'''

def dict_factory(cursor, row):
	d = {}
	for idx, col in enumerate(cursor.description):
		d[col[0]] = row[idx]
	return d

@app.route('/api/v1/resources/suppliers/all', methods=['GET'])
def list_suppliers():
	conn = sqlite3.connect("data_input/Northwind_small.sqlite")
	suppliers = pd.read_sql_query("SELECT Id SupplierId, \
			CompanyName, \
			ContactName, \
			ContactTitle, \
			Address, \
			City, \
			Region, \
			PostalCode, \
			Country, \
			Phone, \
			Fax \
			FROM Supplier;" \
			, conn)
	json_suppliers = suppliers.to_json()

	resp = Response(response=json_suppliers,status=200,mimetype="application/json")
	return(resp)

@app.route('/api/v1/resources/customers/all', methods=['GET'])
def list_customers():
	conn = sqlite3.connect("data_input/Northwind_small.sqlite")
	customers = pd.read_sql_query("SELECT * \
			FROM Customer;" \
			, conn)
	json_customers = customers.to_json()

	resp = Response(response=json_customers,status=200,mimetype="application/json")
	return(resp)

@app.route('/api/v1/resources/products/all', methods=['GET'])
def products_all():
	conn = sqlite3.connect("data_input/Northwind_small.sqlite")
	products = pd.read_sql_query("SELECT P.Id ProductId, \
			P.ProductName, \
			P.QuantityPerUnit, \
			P.UnitPrice, \
			P.UnitsInStock, \
			P.ReorderLevel, \
			P.Discontinued, \
			S.CompanyName SupplierName, \
			S.ContactName, \
			S.Address, \
			S.City, \
			S.Region \
			FROM Product P \
			LEFT JOIN \
			Supplier S ON P.SupplierId = S.Id \
			LEFT JOIN \
			Category C ON P.CategoryId = C.Id;", \
			conn)
	json_products = products.to_json()
	resp = Response(response=json_products,status=200,mimetype="application/json")
	return(resp)

@app.route('/api/v1/resources/top5_countries/dow', methods=['GET'])
def orders_all():
	conn = sqlite3.connect("data_input/Northwind_small.sqlite")

	# query untuk Orders join table OrderDetail, Product, Customer, Category
	orders = pd.read_sql_query("SELECT O.Id OrderId, \
   O.OrderDate, \
   O.RequiredDate, \
   O.ShippedDate, \
   O.ShipRegion, \
   O.ShipCountry, \
   Od.UnitPrice, \
   Od.Quantity, \
   Od.Discount, \
   P.ProductName, \
   ROUND(Od.UnitPrice * Od.Quantity, 2) AS SubTotal, \
   (Od.UnitPrice * Od.Quantity * Od.Discount) AS DiscPrice, \
   (Od.UnitPrice * Od.Quantity) - (Od.UnitPrice * Od.Quantity * Od.Discount) AS Total, \
   Cu.CompanyName AccountName, \
   Cu.ContactName, \
   Cu.ContactTitle, \
   Cu.Address, \
   Cu.City, \
   Cu.Region, \
   Cu.PostalCode, \
   Cu.Country, \
   Cu.Phone, \
   Cu.Fax, \
   Ca.CategoryName, \
   Ca.Description \
FROM [Order] O \
   LEFT JOIN \
   OrderDetail Od ON O.Id = Od.OrderId \
   LEFT JOIN \
   Product P ON Od.ProductId = P.Id \
   LEFT JOIN \
   Customer Cu ON O.CustomerId = Cu.Id \
   LEFT JOIN \
   Category Ca ON P.CategoryId = Ca.Id;" \
   , conn, parse_dates=["OrderDate", "RequiredDate", "ShippedDate"], index_col = "OrderId")

	# Ubah tipe data object beberapa field dibawah menjadi tipe Categorical
	orders[['ShipRegion', \
		'ShipCountry', \
		'ProductName', \
		'City', 'Region', \
		'Country', \
		'CategoryName']] = \
		orders[['ShipRegion', \
		'ShipCountry', \
		'ProductName' , \
		'City', \
		'Region', \
		'Country', \
		'CategoryName']].astype('category')

	# tambahkan field baru dari extract OrderDate untuk analisa datetime
	orders['OrderDate_dayname'] = orders['OrderDate'].dt.day_name()
	orders['OrderDate_week'] = orders['OrderDate'].dt.week
	orders['OrderDate_month'] = orders['OrderDate'].dt.month
	orders['OrderDate_year'] = orders['OrderDate'].dt.year
	orders['OrderDate_quarter'] = orders['OrderDate'].dt.to_period('Q')

	# urutkan kategori DOW nya
	dayorder = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
	orders['OrderDate_dayname'] = pd.Categorical(orders['OrderDate_dayname'],categories=dayorder,ordered=True)

	# query top5 countries groupby
	top5_orders = pd.read_sql_query(
		"SELECT \
		Od.UnitPrice, \
		Od.Quantity, \
		Od.Discount, \
		SUM((Od.UnitPrice * Od.Quantity) - (Od.UnitPrice * Od.Quantity * Od.Discount)) AS Total, \
		Cu.Country \
		FROM [Order] O \
		LEFT JOIN \
		OrderDetail Od ON O.Id = Od.OrderId \
		LEFT JOIN \
		Product P ON Od.ProductId = P.Id \
		LEFT JOIN \
		Customer Cu ON O.CustomerId = Cu.Id \
		LEFT JOIN \
		Category Ca ON P.CategoryId = Ca.Id \
		GROUP BY Cu.Country \
		ORDER BY Total DESC;" \
		, conn)

	# cari index list top5 countries
	top5 = top5_orders.groupby('Country').Total.sum().sort_values(ascending=False).head().index.to_list()

	# ambil data Orders hanya dari index list top5
	top5_data = orders[orders['Country'].isin(top5)].copy()

	# analisa data menggunakan pivot dengan agg SUM dari Total = UnitPrice x Quantity x Discount
	pivot_top5_data = pd.pivot_table(
    data=top5_data,
    index='OrderDate_dayname',
    columns=['Country'],
    values='Total',
    aggfunc='sum',
    margins = True
	)

	json_orders = pivot_top5_data.to_json()
	resp = Response(response=json_orders,status=200,mimetype="application/json")
	return(resp)

@app.errorhandler(404)
def page_not_found(e):
	return "
404

The resource could not be found.
", 404

@app.route('/api/v1/resources/filter', methods=['GET'])
def api_filter():
	query_parameters = request.args
	country = query_parameters.get('country')
	orderdate_start = query_parameters.get('orderdate_start')
	orderdate_end = query_parameters.get('orderdate_end')

	query = "SELECT O.Id OrderId, \
		O.OrderDate, \
		O.RequiredDate, \
		O.ShippedDate, \
		O.ShipRegion, \
		O.ShipCountry, \
		Od.UnitPrice, \
		Od.Quantity, \
		Od.Discount, \
		P.ProductName, \
		ROUND(Od.UnitPrice * Od.Quantity, 2) AS SubTotal, \
		(Od.UnitPrice * Od.Quantity * Od.Discount) AS DiscPrice, \
		(Od.UnitPrice * Od.Quantity) - (Od.UnitPrice * Od.Quantity * Od.Discount) AS Total, \
		Cu.CompanyName AccountName, \
		Cu.ContactName, \
		Cu.ContactTitle, \
		Cu.Address, \
		Cu.City, \
		Cu.Region, \
		Cu.PostalCode, \
		Cu.Country, \
		Cu.Phone, \
		Cu.Fax, \
		Ca.CategoryName, \
		Ca.Description \
		FROM [Order] O \
		LEFT JOIN \
		OrderDetail Od ON O.Id = Od.OrderId \
		LEFT JOIN \
		Product P ON Od.ProductId = P.Id \
		LEFT JOIN \
		Customer Cu ON O.CustomerId = Cu.Id \
		LEFT JOIN \
		Category Ca ON P.CategoryId = Ca.Id WHERE"
	to_filter = []

	if country:
		query += ' Country=? AND'
		to_filter.append(country)

	if orderdate_start:
		query += ' OrderDate BETWEEN ? AND'
		to_filter.append(orderdate_start)

	if orderdate_end:
		query += ' ? AND'
		to_filter.append(orderdate_end)

	if not (country or orderdate_start or orderdate_end):
		return page_not_found(404)

	query = query[:-4] + ';'
	conn = sqlite3.connect("data_input/Northwind_small.sqlite")
	conn.row_factory = dict_factory
	cur = conn.cursor()

	results = cur.execute(query, to_filter).fetchall()

	return jsonify(results)

if __name__ == '__main__':
	app.run(debug=True, port=5000) #run app in debug mode on port 5000


import os
import sys

dirr = '../tr_dir'

cmd = 'echo $(ls {})'.format(dirr)

tmp = os.popen(cmd)
tmp = tmp.read().strip().split(" ")

files = []
for f in tmp:
	loc = dirr+ '/' + f
	files.append(loc)

files_str = ' '.join(str(f) for f in files)

cmd = 'python finess_jsp.py {}'.format(files_str)

os.system(cmd)


import os
import pandas as pd
import numpy as np
from time import time
import matplotlib.pyplot as plt
import sys
sys.path.append('/shogunMMDutils')
import matplotlib as mpl
import fathon
from fathon import fathonUtils as fu

import pickle
import logging

from sklearn.preprocessing import StandardScaler, MinMaxScaler
def return_scaled(x):
    scaler = StandardScaler()
    x_scaled = scaler.fit_transform(np.array(x).reshape(-1,1))
    return np.asarray(x_scaled)

def logger(name, level=logging.INFO):
    tmp_logger = logging.getLogger(name)
    if not tmp_logger.handlers:
        formatter = logging.Formatter('%(asctime)s %(name)s  %(levelname)s %(message)s')
        handler = logging.StreamHandler()
        handler.setFormatter(formatter)

        tmp_logger.setLevel(level)
        tmp_logger.addHandler(handler)
    return tmp_logger

class volatilityEstimation(object):

    def __init__(self, df, tick_window=20, clean=True):
        self.df = df
        self.tick_window = tick_window
        self.mu_1 = np.sqrt((2 / np.pi))
        self.mu_43 = 2 ** (2 / 3) * scipy.special.gamma(7 / 6) ** scipy.special.gamma(1 / 2) ** (-1)
        self.clean = True  # set to true to remove nans at the beginning of the period
        self.trading_seconds_btc = 86400
        self.trading_seconds_futures = 23400

        self.column_open = 'micro_price_open'
        self.column_high = 'micro_price_high'
        self.column_low = 'micro_price_low'
        self.column_close = 'micro_price_close'

        self.log_hl = (df[str(self.column_high)] / df[str(self.column_low)]).apply(np.log)
        self.log_co = (df[str(self.column_close)] / df[str(self.column_open)]).apply(np.log)
        # median sampling frequency

        self.z = pd.DataFrame(
            [((x.hour * 60 + x.minute) * 60 + x.second) for x in self.df['TimeStamp_open']]).diff().fillna(0).astype(
            'float64')
        self.clean_arrival_rates = np.asarray(
            self.z[(self.z > self.z.quantile(0.003)) & (self.z < self.z.quantile(0.97))].dropna()).astype('float64')
        # clean arrival rates after i remove all the outliers
        self.mean_sampling_frequency = np.mean(np.asarray(self.clean_arrival_rates))

    def arrival_rates(self):

        z = pd.DataFrame([((x.hour * 60 + x.minute) * 60 + x.second) for x in self.df['TimeStamp_open']]).diff().fillna(
            0).astype('float64')
        clean_arrival_rates = np.asarray(z[(z > z.quantile(0.003)) & (z < z.quantile(0.97))].dropna()).astype('float64')
        # clean arrival rates after i remove all the outliers

        return clean_arrival_rates

    def M_parameter(self):
        """
        normalised sampling frequency for adjustments
        """

        M_btc = self.trading_seconds_btc / self.mean_sampling_frequency
        M_futures = self.trading_seconds_futures / self.mean_sampling_frequency

        return [M_btc, M_futures]

    def garmanKlass(self, trading_periods):

        rs = 0.5 * self.log_hl ** 2 - (2 * np.log(2) - 1) * self.log_co ** 2

        def f(v):
            return (trading_periods * v.mean()) ** 0.5

        result = rs.rolling(window=self.tick_window, center=False).apply(func=f)

        if self.clean:
            return result.dropna()
        else:
            return result

    def parkinson(self, trading_periods, clean=True):

        rs = (1.0 / (4.0 * np.log(2.0))) * (self.df[str(self.column_high)] / self.df[str(self.column_close)]).apply(
            np.log) ** 2.0

        def f(v):
            return trading_periods * v.mean() ** 0.5

        result = rs.rolling(window=self.tick_window, center=False).apply(func=f)

        if self.clean:
            return result.dropna()
        else:
            return result

    def bipower_variation(self, rollingWindow=5):
        '''
        Bipower Variation (BV) is the sum of the product of absolute time series returns
        :param column: price column
        :return: returns bivariate variation
        Barnhorf - Nielse & Shephard 2004 & 2006
        '''
        bv = self.mu_1 ** (-2) * (
            (self.log_hl.abs() * self.log_hl.shift(1).abs()).fillna(0).rolling(rollingWindow).sum())

        if self.clean:
            return bv.dropna()
        else:
            return bv

    def tripower_quarticity(self, rollingWindow=5, sampling_param=0):
        """
        using M for BTC here - change sampling param to 1 for everything else
        """
        tpq = self.M_parameter()[sampling_param] * self.mu_43 ** (-3) * ((self.log_hl.abs() ** (
                    4 / 3) * self.log_hl.shift(1).abs() ** (4 / 3) * self.log_hl.shift(2).abs() ** (4 / 3)).rolling(
            rollingWindow).sum().fillna(0))

        if self.clean:

            return tpq.dropna()
        else:
            return tpq

    def realised_variance(self, rollingWindow=5):
        """
        realised variance - andersen & bollerselv -1998
        """

        realvar = (self.log_hl ** 2).rolling(rollingWindow).sum().fillna(0)

        if self.clean:
            return realvar.dropna()
        else:
            return realvar

    def realised_absolute_variation(self, rollingWindow=5, sampling_param=0):
        """
        realised absolute variation - Forsberg & Ghysels 2007
        using M for BTC here - change sampling param to 1 for everything else
        """

        realabsovar = self.mu_1 ** (-2) * self.M_parameter[sampling_param] ** (-0.5) * self.log_hl.rolling(
            rollingWindow).sum().fillna(0)

        return realabsovar

    def realised_skewness_kurtosis(self, rollingWindow=15, sampling_param=0):

        """
        set the rolling window to 10-15
        sampling param for BTC ~ 0
        """

        rm3 = (self.log_hl ** 3).rolling(rollingWindow).sum().fillna(0)
        rm4 = (self.log_hl ** 4).rolling(rollingWindow).sum().fillna(0)

        rs = np.sqrt(self.M_parameter()[sampling_param]) * (
                    rm3 / self.realised_variance(rollingWindow=rollingWindow)) ** (3 / 2)
        rk = self.M_parameter()[sampling_param] * (rm4 / self.realised_variance(rollingWindow=rollingWindow)) ** 2

        return rs, rk

    def jumps_test(self, rollingWindow, sampling_param=0):
        """
        Jump test by Huang and Tauchen 2005
        adopted by repo: RealisedQuantities
        """

        j1 = (np.log(self.realised_variance(rollingWindow=rollingWindow)) - np.log(
            self.bipower_variation(rollingWindow=rollingWindow)))
        j2 = (((self.mu_1 ** -4) + 2 * (self.mu_1 ** -2) - 5) /
              (self.M_parameter()[0] * self.tripower_quarticity(rollingWindow=rollingWindow,
                                                                sampling_param=sampling_param)
               * (self.bipower_variation(rollingWindow=rollingWindow) ** -2)))

        j = j1 / (j2 ** 0.5)

        return (j.abs() >= stats.norm.ppf(0.995)) * 1  # can also look at 0.999)*

    def relative_jump_measure(self, rollingWindow, sampling_param=0):

        nominator = (self.realised_variance(rollingWindow=rollingWindow)
                     - self.bipower_variation(rollingWindow=rollingWindow))

        measure_to_return = nominator / self.bipower_variation(rollingWindow=rollingWindow)

        return measure_to_return

    def negative_semivariance(self, x):
        """
        input: returns
        negative realised semivariance
        Barndorff- Nielsen
        """
        return x.apply(lambda x: (x ** 2)) * (x < 0).sum()

    def positive_semivariance(self, x):
        """
        input: returns
        positive realised semivariance
        Barndorff- Nielsen
        """
        return x.apply(lambda x: (x ** 2)) * (x > 0).sum()

    def signed_jump_variation(self, x):

        sjv = self.positive_semivariance(x) - self.negative_semivariance(x)
        sjv_p = sjv * (sjv > 0)
        sjv_n = sjv * (sjv < 0)

        return sjv_p, sjv_n

    def continuous_discontinuous_quadratic_variance(self):

        """
        seperate continuous and discontinuous parts of quadratic variation

        """
        pass

class descriptiveStats(object):

    def __init__(self, df):
        self.df = df
        self.column_open = 'micro_price_open'
        self.column_high = 'micro_price_high'
        self.column_low = 'micro_price_low'
        self.column_close = 'micro_price_close'
        self.log_hl = (df[str(self.column_high)] / df[str(self.column_low)]).apply(np.log)
        self.log_co = (df[str(self.column_close)] / df[str(self.column_open)]).apply(np.log)

    def base_descriptive_stats(self):
        desc_stats = dict()
        desc_stats['median_hl'] = np.median(self.log_hl)
        desc_stats['mean_co'] = np.median(self.log_co)
        desc_stats['quantile_95_hl'] = self.log_hl.quantile(0.95)
        desc_stats['quantile_95_co'] = self.log_co.quantile(0.95)
        desc_stats['q3_75_hl'] = self.log_hl.quantile(0.75)
        desc_stats['q3_75_co'] = self.log_co.quantile(0.75)
        desc_stats['q1_25_hl'] = self.log_hl.quantile(0.25)
        desc_stats['q1_25_co'] = self.log_co.quantile(0.25)
        desc_stats['outlier_lower_limit_hl'] = desc_stats['q1_25_hl'] - 1.5 * (
                    desc_stats['q3_75_hl'] - desc_stats['q1_25_hl'])
        desc_stats['outlier_upper_limit_hl'] = desc_stats['q3_75_hl'] + 1.5 * (
                    desc_stats['q3_75_hl'] - desc_stats['q1_25_hl'])
        desc_stats['lower_limit_outliers'] = self.log_hl[self.log_hl > desc_stats['outlier_lower_limit_hl']].count()
        desc_stats['upper_limit_outliers'] = self.log_hl[self.log_hl > desc_stats['outlier_upper_limit_hl']].count()
        desc_stats['total_outliers'] = desc_stats['lower_limit_outliers'] + desc_stats['upper_limit_outliers']

        return desc_stats

class DataLoader:
    """
    loads data and constructs full LOB and calendar bars for each one
    """

    def __init__(self, data_location,
                 symbol,
                 dates,
                 use_columns,
                 calendar_resample,
                 trade_volume_width,
                 ticks_width,
                 usd_volume_width):

        self._data_location = data_location
        self._symbol = symbol
        self._input_dates = dates
        self._use_columns = use_columns
        self._calendar_resample = calendar_resample
        self._ticks_width = ticks_width
        self._trade_volume_width = trade_volume_width
        self._usd_volume_width = usd_volume_width
        self._dates = []
        self._bars_dict = {}
        self.calendar_bars = []
        self.volume_bars = []
        self.tick_bars = []
        self.usd_volume_bars = []
        self._logger = logger('DataLoader')
        #         self._symbols = sorted(['VIX',  'FB1',  'TU1',  'G_1',  'RX1',  'OE1',  'TY1',  'FV1',  'JB1',  'RX1',  'DU1',  'KE1',  'US1',  'YM1', 'XM1',  'VXX'] )
        self._data_folder = data_location

        self._quotes_string = "".join((self._symbol, '_Comdty_quotes'))
        self._trades_string = "".join((self._symbol, '_Comdty_trades'))

        #         self._quotes_files =sorted([ f for f in os.listdir(self._data_folder) if str('_quotes') in f])
        #         self._trades_files = sorted([f for f in os.listdir(self._data_folder) if str('_trades') in f])

        self._quotes_Files = os.path.join(self._data_folder, self._quotes_string)
        self._trades_Files = os.path.join(self._data_folder, self._trades_string)

    def load_and_format_data(self):
        mergedFile = dict()
        for date in self._input_dates:
            dateToLoad = os.path.join(date + '.csv')
            self._logger.info(f"Loading data for:{date}")

            quotesDateFile = os.path.join(self._quotes_Files, dateToLoad)
            print(quotesDateFile)
            tradesDateFile = os.path.join(self._trades_Files, dateToLoad)
            print(tradesDateFile)

            try:
                qt_tmp_df = pd.read_csv(quotesDateFile, usecols=self._use_columns)
                qt_tmp_df['TradeTime'] = pd.to_datetime(qt_tmp_df.time).values

                quotes_df = pd.merge_asof(
                    qt_tmp_df[qt_tmp_df['type'] == 'BID'].dropna().fillna("ffill").sort_values('TradeTime')
                    , qt_tmp_df[qt_tmp_df['type'] == 'ASK'].dropna().fillna("ffill").sort_values('TradeTime'),
                    on='TradeTime', allow_exact_matches=True)

                quotes_df = quotes_df.rename(columns={'value_x': 'BestBid', 'value_y': 'BestAsk',
                                                      'size_x': 'BidSize', 'size_y': 'AskSize',
                                                      'time_x': 'QuoteTime'})

                tr_tmp_df = pd.read_csv(tradesDateFile, usecols=self._use_columns)
                tr_tmp_df['TradeTime'] = pd.to_datetime(tr_tmp_df.time).values
                trades_df = tr_tmp_df.dropna().fillna("ffill").sort_values('TradeTime')
                trades_df = trades_df.rename(columns={'value': 'TradePrice',
                                                      'size': 'TradeSize'})
                trades_columns = ['TradeSize', 'type', 'TradePrice', 'TradeTime']

                LOB = pd.merge_asof(quotes_df, trades_df[trades_columns].sort_values('TradeTime'), on='TradeTime',
                                    allow_exact_matches=True)

                LOB.BidSize = LOB['BidSize'].replace(0, 1)
                LOB.AskSize = LOB['AskSize'].replace(0, 1)
                LOB['TimeStamp'] = pd.to_datetime(LOB.TradeTime).dt.time
                LOB['TradeVolume'] = LOB['TradeSize'].fillna(0)
                LOB['total_traded_volume'] = LOB.TradeVolume
                LOB['milliSeconds'] = [(((x.hour * 60 + x.minute) * 60 + x.second) * 1000) for x in
                                       LOB['TimeStamp']]
                LOB['dollar_traded_volume'] = pd.Series(LOB.TradePrice * LOB.TradeVolume).fillna(0)
                LOB['timeStampIdx'] = pd.DatetimeIndex(LOB.time_y)
                LOB['micro_price'] = (LOB.BestAsk * LOB.AskSize + LOB.BestBid * LOB.BidSize) / (
                            LOB.AskSize + LOB.BidSize)  #

                mergedFile[date] = LOB

            except FileNotFoundError:
                self._logger.info(f"Data for day {date} does not exist. Skipping this day")
        return mergedFile

    def apply_micro_structure_features(self, df):

        self._logger.info("Applying micro-structure features")

        df['weighted_average_bid_price'] = pd.DataFrame(df.filter(like='bid_price', axis=1).values).mul(
            pd.DataFrame(df.filter(like='bid_size', axis=1).values)).sum(axis=1) / pd.DataFrame(
            df.filter(like='bid_size', axis=1).values).sum(axis=1)

        df['weighted_average_ask_price'] = pd.DataFrame(df.filter(like='ask_price', axis=1).values).mul(
            pd.DataFrame(df.filter(like='ask_size', axis=1).values)).sum(
            axis=1) / pd.DataFrame(df.filter(like='ask_size', axis=1).values).sum(axis=1)
        df['weighted_activity_spread'] = df['weighted_average_ask_price'] - df['weighted_average_bid_price']
        df['total_size'] = (pd.DataFrame(df.filter(like='ask_size', axis=1).values).sum(axis=1) +
                            pd.DataFrame(df.filter(like='bid_size', axis=1).values).sum(axis=1))
        df['micro_price'] = ((pd.DataFrame(df.filter(like='bid_price', axis=1).values).mul(
            pd.DataFrame(df.filter(like='bid_size', axis=1).values)).sum(axis=1) +
                              pd.DataFrame(df.filter(like='ask_price', axis=1).values).mul(
                                  pd.DataFrame(df.filter(like='ask_size', axis=1).values)).sum(axis=1))) / df[
                                'total_size']
        df['price_imbalance'] = ((pd.DataFrame(df.filter(like='ask_price', axis=1).values).mul(
            pd.DataFrame(df.filter(like='ask_size', axis=1).values)).sum(axis=1) -
                                  pd.DataFrame(df.filter(like='bid_price', axis=1).values).mul(
                                      pd.DataFrame(df.filter(like='bid_size', axis=1).values)).sum(axis=1))) / df[
                                    'total_size']
        df['pct_change_micro_price'] = df.micro_price_close.pct_change()
        df['simple_mid_price'] = 0.5 * (pd.DataFrame(df.filter(like='ask_price', axis=1)).mean(axis=1) +
                                        pd.DataFrame(df.filter(like='bid_price', axis=1)).mean(axis=1))

        # TODO: do we want to be using pct_change here or log returns?
        return df

    @staticmethod
    def prep_bars(df, drop_col):
        df.columns = [f"{i[0]}_{j}" for i, j in zip(df.columns, ['open', 'high', 'low', 'close'] * len(df.columns))]
        if drop_col:
            df.drop([f"{drop_col}_{i}" for i in ['open', 'high', 'low', 'close']], axis=1, inplace=True)
        return df

    def drop_lob_columns(df):
        # not using this anywhere at the moment

        ask_price_columns_to_be_dropped = list(pd.DataFrame(df.filter(like='ask_price_', axis=1)).columns.values)
        ask_size_columns_to_be_dropped = list(pd.DataFrame(df.filter(like='ask_size_', axis=1)).columns.values)
        bid_size_columns_to_be_dropped = list(pd.DataFrame(df.filter(like='bid_size_', axis=1)).columns.values)
        bid_price_columns_to_be_dropped = list(pd.DataFrame(df.filter(like='bid_price_', axis=1)).columns.values)
        time_stamps = ['timestamp', 'timestamp.1', 'exchange_time', 'feed_time']
        columns_to_be_dropped = ask_price_columns_to_be_dropped + ask_size_columns_to_be_dropped + \
                                bid_size_columns_to_be_dropped + bid_price_columns_to_be_dropped + \
                                time_stamps
        df = df.drop(columns=columns_to_be_dropped)

        return df

    def get_bars(self, df):

        self._logger.info('Calculating calendar clock')
        calendar_bars = df.resample(rule=self._calendar_resample, on='timeStampIdx').agg(
            ['first', 'max', 'min', 'last'])
        calendar_bars = self.prep_bars(calendar_bars, 'timeStampIdx')

        self._logger.info('Calculating volume clock')
        # TODO: Doing this on day-by-day basis. We lose the last turn of the clock each day
        df['cum_total_trade_volume'] = df.total_traded_volume.cumsum()
        df['volume_bucket'] = np.floor(df.cum_total_trade_volume / self._trade_volume_width)
        volume_bars = df.groupby(by='volume_bucket').agg(['first', 'max', 'min', 'last']).iloc[1:-1]
        volume_bars = self.prep_bars(volume_bars, 'cum_total_trade_volume')
        df.drop(['cum_total_trade_volume', 'volume_bucket'], axis=1, inplace=True)

        self._logger.info('Calculating tick clock')
        # TODO: ticks here means any change to the order book. Not necessarily a trade. Can add that clock too
        df['tick_count'] = range(len(df))
        df['tick_bucket'] = np.floor(df.tick_count / self._ticks_width)
        tick_bars = df.groupby(by='tick_bucket').agg(['first', 'max', 'min', 'last']).iloc[1:-1]
        tick_bars = self.prep_bars(tick_bars, 'tick_count')
        df.drop(['tick_count', 'tick_bucket'], axis=1, inplace=True)

        self._logger.info('Calculating usd volume clock')
        df['cum_usd_volume'] = df.dollar_traded_volume.cumsum()
        df['usd_volume_bucket'] = np.floor(df.cum_usd_volume / self._usd_volume_width)
        usd_volume_bars = df.groupby(by='usd_volume_bucket').agg(['first', 'max', 'min', 'last']).iloc[1:-1]
        usd_volume_bars = self.prep_bars(usd_volume_bars, 'cum_usd_volume')
        df.drop(['cum_usd_volume', 'usd_volume_bucket'], axis=1, inplace=True)

        self.calendar_bars.append(calendar_bars)
        self.volume_bars.append(volume_bars)
        self.tick_bars.append(tick_bars)
        self.usd_volume_bars.append(usd_volume_bars)

        self._bars_dict['calendar_bars'] = self.calendar_bars
        self._bars_dict['volume_bars'] = self.volume_bars
        self._bars_dict['tick_bars'] = self.tick_bars
        self._bars_dict['usd_volume_bars'] = self.usd_volume_bars

    def get_all_dates_bars(self, input_dict):
        output_dict = {}
        for input_date in self._input_dates:
            output_dict[input_date] = self.get_bars(input_dict[input_date])

        return output_dict

    @staticmethod
    def get_concat_data(input_dict):
        concat_dict = dict()
        for bar in input_dict.keys():
            concat_dict[bar] = pd.concat([input_dict[bar][i]
                                          for i in range(len(input_dict[bar]))], ignore_index=False)
        return concat_dict

workDriveFolder = '/media/ak/WorkDrive/Data'
laptopDataFolder  = '/media/ak/LaCie'

figures_destination = '/home/ak/Documents/Research/Papers/figures/'
dataFolder = laptopDataFolder

quotes =[f for f in os.listdir(dataFolder) if str('_quotes') in f]
trades =[f for f in os.listdir(dataFolder) if str('_trades') in f]
symbols= [f.split("_")[0] for f in quotes]
if __name__ == "__main__":
    symbolIdx = 3
    symbol = sorted(symbols)[symbolIdx]
    print(symbol)
    quotesFileCh = os.path.join(dataFolder, sorted(quotes)[symbolIdx])
    tradesFileCh = os.path.join(dataFolder, sorted(trades)[symbolIdx])

    ## get common Dates
    quotesDates = sorted([f.split(".csv")[0] for f in os.listdir(quotesFileCh)])
    tradesDates = sorted([f.split(".csv")[0] for f in os.listdir(tradesFileCh)])
    intersectionDates = list(set(quotesDates).intersection(tradesDates))

    trades_cols = ['size', 'time', 'type', 'value']

    calendar_resample = "300S"
    trade_volume_width = 100
    ticks_width = 100
    usd_volume_width = 100
    dates_choice = quotesDates[0:20]

    testClass = DataLoader(data_location=dataFolder,
                           symbol=symbol,
                           dates=dates_choice,
                           use_columns=trades_cols,
                           calendar_resample="300S",
                           trade_volume_width=trade_volume_width,
                           ticks_width=ticks_width,
                           usd_volume_width=usd_volume_width)

    ## load data ##
    input_dict = testClass.load_and_format_data()


# Prototype 1
import numpy as np
import matplotlib.pyplot as plt
import statsmodels.formula.api as smf
import pandas as pd
from scipy.optimize import curve_fit

rng = np.random.default_rng(seed = 313)
x = sort(np.linspace(0, 1, num = 100))
u = rng.uniform(-1, 1, 100)
y1 = 2 + 3 * x + u
y2 = np.exp(3 * x ** 2) + u

func = lambda x, a, b: np.exp(a * x ** b)

param1 = curve_fit(func, x, y1)[0]
param2 = curve_fit(func, x, y2)[0]

def rsq(x, y):
    params = curve_fit(func, x, y)[0]
    lower = np.mean((np.mean(y) - y) ** 2)
    upper = np.mean((func(x, params[0], params[1]) - y) ** 2)
    return 1 - upper / lower

plt.scatter(x, y1)
plt.plot(x, func(x, param1[0], param1[1]), color = "black")
plt.show()

plt.scatter(x, y2)
plt.plot(x, func(x, param2[0], param2[1]), color = "black")
plt.show()

rsq(x, y1)
rsq(x, y2)

def nls_plotter(y, x, func):
    params = curve_fit(func, x, y)[0]
    lower = np.mean((np.mean(y) - y) ** 2)
    upper = np.mean((func(x, *params) - y) ** 2)
    plt.scatter(x, y)
    plt.plot(x, func(x, *params), color = "black")
    plt.show()
    return {"rsq": 1 - upper / lower, "params" : params}

nls_plotter(y2, x, lambda x, a, b, c: a * x ** 2 + b * x ** c)

extras/2021 Fall EBA3500 Data Analysis and Programming/notes/lecture2/notes5.py