from src.helpers.helper import \
    DATA_CARSTATUS_PATH, \
    DEFAULT_SUBSCRIBED_TOPIC, \
    CAR_INDEX, DATA_PATH, \
    TS_INDEX, \
    INDEX_LIMIT, \
    POS, \
    SPEED, \
    event_car_position_msg, \
    extract_gps_data, \
    extract_timestamp, \
    find_velocity_mph, \
    find_distance_meter, \
    create_carstatus_topic_data, \
    find_positions, \
    write_file

from src.publish_topics import publish_carstatus_topic, publish_events_topic
from ftfy import fix_text
from collections import deque

import os
import json
import glob

car_long_lat_data = []

def write_raw_data(msg):
    """Function to write the raw data to file for each car from subscribed topic
    carCoordinates

    :param msg: dictionary containing topic info and data
    """

    if msg.topic == DEFAULT_SUBSCRIBED_TOPIC:
        car = json.loads(
            msg.payload.decode()
        )[CAR_INDEX]

        raw_data = f'{car}' + '_' + DEFAULT_SUBSCRIBED_TOPIC + '.txt'

        # in a real application this would have been perhaps an S3 bucket writing
        # instead! (Minio locally) but let's keep it simple!
        path = os.path.join(DATA_PATH, raw_data)

        with open(path, 'a') as raw_data_file:
            raw_data_file.writelines(msg.payload.decode() + '\n')

        transform_topic_data(car)

    else:
        raise ValueError('wrong topic')

def load_subscribed_topics(client, path=DATA_CARSTATUS_PATH):
    """Function will read the data from the subscribed raw carStatus topic written to each car,
    sends them to be published.

    :param client: the mqtt client
    :param path: path for subscribed topic data
    """
    for filename in glob.iglob(path + '**/*', recursive=True):

        with open(filename, 'r') as file:
            rows = deque(file, 2)

            if len(rows) == 2:
                row0, row1 = rows[0], rows[1]

                if isinstance(row0, str):
                    row0 = fix_text(row0)
                    data0, data1 = json.loads(row0), json.loads(row1)

                else:
                    data0, data1 = json.loads(row0.decode('utf-8')), json.loads(row1.decode('utf-8'))

                if data1['type'] == 'SPEED':
                    publish_carstatus_topic(data1, 'SPEED', client)

                elif data1['type'] == 'POSITION':
                    ts = data1.get(TS_INDEX)
                    pos0, pos1 = data0.get('value'), data1.get('value')

                    publish_carstatus_topic(data1, 'POSITION', client)
                    msg = event_car_position_msg(data1, pos0, pos1)

                    if len(msg) > 0:
                        publish_events_topic(msg, ts, client)

                    file.close()

def transform_topic_data(car):
    """Function to retrieve information to create carstatus topic, will
    write data to file
    :param car: actual car
    """
    topic_filename = f'{car}' + '_' + DEFAULT_SUBSCRIBED_TOPIC + '.txt'

    with open(os.path.join(DATA_PATH, topic_filename), 'r') as raw_data_file:
        raw_data_carstatus = deque(raw_data_file, maxlen=2)

        st0, st1 = extract_gps_data(raw_data_carstatus)
        t0, t1 = extract_timestamp(raw_data_carstatus)

        if len(car_long_lat_data) == INDEX_LIMIT:
            car_long_lat_data[car] = [st1]
            car_positions = find_positions(car_long_lat_data)

            topic_carstatus_position = f'{car}' + '_carStatus_position.txt'

            # write position
            write_file(
                topic_carstatus_position,
                create_carstatus_topic_data(t1, car, POS, car_positions[car][1] + 1),
                DATA_CARSTATUS_PATH
            )

        else:
            car_long_lat_data.append([st1])

        distance = find_distance_meter(st0, st1)
        curr_speed = find_velocity_mph(distance, t0, t1)
        topic_carstatus_speed = f'{car}' + '_carStatus_speed.txt'

        # write speed
        write_file(
            topic_carstatus_speed,
            create_carstatus_topic_data(t1, car, SPEED, curr_speed),
            DATA_CARSTATUS_PATH
        )


# The QUBO formulation for clustering is specified in https://arxiv.org/pdf/1708.05753.pdf,
# with intensity / RGB difference as the distance metric.

from dwave_qbsolv import *
import cv2
import numpy as np
import matplotlib.pyplot as plt
from tqdm import tqdm
from os import path
from itertools import permutations

def ising_k(vectors, labels, clusters=3, hardware_precision=7, scale_range=np.arange(0, 1, 0.1), show_plot=True):
    if len(vectors.shape) != 2:
        print("Specify a valid dataset first.")
        exit()

    N, C = vectors.shape # Number of vectors and their dimensionality (C for "channels").
    K = clusters # cluster counts
    n = hardware_precision # hardware precision of annealer coupling / bias
    LAMBDA = (2 ** (n-1) - 1) / (2 * (K - 2 if K > 2 else K)) # calculate lambda cluster constraint

    peaks = [] # record all peak accuracies
    scales = [] # record associated scales
    best_assignments = [] # record best clusterings

    for SCALE in scale_range:
        print("Scaling all loss to %f" % SCALE)

        diffs = np.zeros((N, N)) # store pairwise differences
        sums = np.zeros(N) # store sum of all pairwise differences relative to i-th pixel

        print("Calculating pairwise differences...")

        for idx, vec in tqdm(enumerate(vectors)): # for all pixels in the image...
            diffs[idx] = np.linalg.norm((vectors - vec), axis=1) * SCALE # assign pairwise difference for all pixels, scaled accordingly
            sums[idx] = np.sum(diffs[idx]) # assign sum of all pairwise differences relative to i-th pixel

        h = {} # intrinsic weight matrix
        J = {} # coupling weight matrix

        print("Assigning pairwise clustering weights...")

        # assign general pairwise clustering weights - both biases and couplings
        for k in tqdm(range(K)): # for every cluster we have...
            for i in range(N): # for every pixel in the image...
                for j in range(N): # and every pixel it might be paired with...
                    # give each binary variable an unique number
                    var_i = 10 * i + k # s_ik = x_(10i+k)
                    var_j = 10 * j + k # s_jk = x_(10j+k)
                    if i == j: # intrinsic weights (bias) indexing - for every cluster assignment (s_ik for all k in K)
                        h[var_i] = sums[i] + (2 * LAMBDA * (K - 2 if K > 2 else K)) # assign weight
                    else:
                        J[(var_i, var_j)] = 0.5 * diffs[i][j] # the weighting for a cluster assignment for two vars is half their "distance"

        print("Assigning pairwise cluster possibility weights...")

        # assign pairwise clustering variables per pixel
        for i in tqdm(range(N)):
            for a in range(K):
                for b in range(K):
                    if a != b:
                        var_a = 10 * i + a
                        var_b = 10 * i + b
                        J[(var_a, var_b)] = LAMBDA

        # TODO: Guarantee that for all s_ik for each i, only one is "true"

        print("%d couplings defined between %d variables." % (len(J.keys()), N * K))

        response = QBSolv().sample_ising(h, J, verbosity=2) # sample / solve Ising problem

        print(response)

        clusters = list(response.samples()) # store clustering assignments
        clustered = np.zeros(N)

        accuracies = [] # record accuracies of every given solution
        assignments = [] # record corresponding assignments

        possible_labels = set(permutations(range(K))) # generate all possible permutations of clusters

        for i, clustering in enumerate(clusters): # for every low-energy solution...
            curr_max = float('-inf') # current max accuracy achieved with permutation
            clustered = np.zeros(N) # initialize empty array

            # mark cluster assignments for visualization
            for key in clustering.keys():
                # extract pixel number and cluster assignments (var. number format is {pixel}{cluster})
                key_str = str(key)
                # since pixel #0 cluster assignments will break this format (only {cluster}), fix manually
                if len(key_str) == 1:
                    key_str = "0%s" % key_str
                # extract all parameters from single number!
                key_pix, key_cluster, key_status = int(key_str[:-1]), int(key_str[-1:]), clustering[key]

                for label_set in possible_labels:
                    # if spin / state is +1, fill in cluster indicated by variable
                    if key_status == 1:
                        clustered[key_pix] = label_set[key_cluster] # replace with permuted cluster label possibility

                    # Count matches
                    matches = np.sum(np.where(clustered == labels, 1, 0))
                    percent = matches / N

                    # Update maximum accuracy if this permutation is better
                    if percent > curr_max: curr_max = percent

            # Record accuracy
            accuracies.append(curr_max)

            # Record assignment
            assignments.append(clustered)

        accuracies = np.array(accuracies)

        print(accuracies)

        SNR = np.linalg.norm(np.mean(np.mean(vectors, axis=0), axis=0)) / np.linalg.norm(np.std(np.std(vectors, axis=0), axis=0))
        print("Found input SNR to be %f." % SNR)

        peak = np.max(accuracies) * 100
        best = assignments[np.argmax(accuracies)]

        print("Highest accuracy: %f percent" % peak)
        print("Associated clustering: %s" % str(best))

        peaks.append(peak)
        scales.append(SCALE)
        best_assignments.append(best)

    print(peaks)
    print(scales)

    # Find best-fit polynomial for data
    # poly = np.poly1d(np.polyfit(scales, peaks, 10))
    # param_x = np.linspace(PARAM_MIN, PARAM_MAX, 100)
    # plt.plot(param_x, poly(param_x), 'r--')

    # Plot accuracy points and line of best fit
    plt.scatter(scales, peaks)
    plt.ylim(0,100)

    for x, y in zip(scales, peaks):
        plt.annotate(
            "%d" % y,
            (x, y),
            textcoords="offset points",
            xytext=(0,10),
            ha="center"
        )

    plt.xticks(scale_range)
    plt.yticks(np.arange(0, 100, 10))

    plt.xlabel("Scale")
    plt.ylabel("Accuracy")

    if show_plot: plt.show()

    return {
        'bests': best_assignments,
        'peaks': peaks,
        'scales': scales,
        # 'plot': plt.gcf()
    }


#! /usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import sys
import unittest

sys.path.append(os.path.abspath('../../../../../../'))
from examples.csj.s5.exp.dataset.load_dataset import Dataset
from utils.io.labels.character import Idx2char
from utils.io.labels.word import Idx2word
from utils.measure_time_func import measure_time

class TestLoadDataset(unittest.TestCase):

    def test(self):

        # self.check(label_type='kanji_wb_left', data_type='eva1')
        # self.check(label_type='kanji_wb_right', data_type='eva1')
        # self.check(label_type='kanji_wb_both', data_type='eva1')
        # self.check(label_type='kanji_wb_remove', data_type='eva1')
        #
        # raise ValueError

        # data_size
        self.check(label_type='word', data_type='eval1', data_size='aps_other')
        self.check(label_type='word', data_type='eval1', data_size='all')

        # data_type
        # self.check(label_type='word', data_type='train')
        self.check(label_type='word', data_type='dev')
        self.check(label_type='word', data_type='eval1')
        self.check(label_type='word', data_type='eval2')
        self.check(label_type='word', data_type='eval3')

        # label_type
        self.check(label_type='kanji')
        self.check(label_type='kanji_wb')
        # self.check(label_type='kana')
        # self.check(label_type='kana_wb')
        # self.check(label_type='phone')
        # self.check(label_type='phone_wb')
        self.check(label_type='pos')

        # sort
        self.check(label_type='word', sort_utt=True, sort_stop_epoch=True)

        # frame stacking
        self.check(label_type='word', frame_stacking=True)

        # splicing
        self.check(label_type='word', splice=11)

        # multi-GPU
        # self.check(label_type='word', num_gpus=8)

    @measure_time
    def check(self, label_type, data_type='dev',
              data_size='all', backend='pytorch',
              shuffle=False, sort_utt=True, sort_stop_epoch=None,
              frame_stacking=False, splice=1, num_gpus=1):

        print('========================================')
        print('  backend: %s' % backend)
        print('  label_type: %s' % label_type)
        print('  data_type: %s' % data_type)
        print('  data_size: %s' % data_size)
        print('  shuffle: %s' % str(shuffle))
        print('  sort_utt: %s' % str(sort_utt))
        print('  sort_stop_epoch: %s' % str(sort_stop_epoch))
        print('  frame_stacking: %s' % str(frame_stacking))
        print('  splice: %d' % splice)
        print('  num_gpus: %d' % num_gpus)
        print('========================================')

        num_stack = 3 if frame_stacking else 1
        num_skip = 3 if frame_stacking else 1
        dataset = Dataset(
            data_save_path='/n/sd8/inaguma/corpus/csj/kaldi',
            backend=backend,
            input_freq=80, use_delta=True, use_double_delta=True,
            data_type=data_type, data_size=data_size,
            label_type=label_type, batch_size=64,
            max_epoch=1, splice=splice, num_stack=num_stack, num_skip=num_skip,
            min_frame_num=40, shuffle=shuffle,
            sort_utt=sort_utt, reverse=False, sort_stop_epoch=sort_stop_epoch,
            num_gpus=num_gpus, tool='htk', num_enque=None)

        print('=> Loading mini-batch...')
        if 'word' in label_type:
            map_fn = Idx2word(dataset.vocab_file_path)
        else:
            map_fn = Idx2char(dataset.vocab_file_path)

        for batch, is_new_epoch in dataset:
            if data_type == 'train' and backend == 'pytorch':
                for i in range(len(batch['xs'])):
                    if batch['xs'].shape[1] < batch['ys'].shape[1]:
                        raise ValueError(
                            'input length must be longer than label length.')

            if dataset.is_test:
                str_true = batch['ys'][0][0]
            else:
                str_true = map_fn(batch['ys'][0][:batch['y_lens'][0]])

            print('----- %s (epoch: %.3f, batch: %d) -----' %
                  (batch['input_names'][0], dataset.epoch_detail, len(batch['xs'])))
            print(str_true)
            print('x_lens: %d' % (batch['x_lens'][0] * num_stack))
            if not dataset.is_test:
                print('y_lens: %d' % batch['y_lens'][0])

            if dataset.epoch_detail >= 0.1:
                break

if __name__ == '__main__':
    unittest.main()


# We want to compute
#
# I = int_0^1 x^{-1/2} / (exp(x) + 1) dx
#
# here we just use uniform random numbers -- we'll have problems near
# the origin, and our answer will be unreliable

import random
import matplotlib.pyplot as plt
import numpy as np

def f(x):
    return x**(-1.0/2.0)/(np.exp(x) + 1.0)

# plot our function
eps = 1.e-16
x = np.linspace(0.0+eps, 1.0, 1000)
plt.plot(x, f(x))
plt.ylim(0,10)
plt.xlabel("x")
plt.ylabel("f(x)")
plt.savefig("integrand-importance.png", dpi=150)

# do multiple runs
for t in range(5):

    I = 0.0
    N = 10000

    for n in range(N):
        x = np.random.uniform()
        I += f(x)

    I *= 1.0/N

    print("N: {}, I = {}".format(N, I))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Dec  6 14:04:16 2019

@author: sambhav
"""
from django.conf.urls import url
from .views import UserRegistrationView,UserLoginView,Products,Product_create,Product_update,Product_delete
from rest_framework_swagger.views import get_swagger_view

schema_view = get_swagger_view(title='Swagger API')

urlpatterns = [
	url(r'^$', schema_view),
    url(r'^signup', UserRegistrationView.as_view()),
    url(r'^signin', UserLoginView.as_view()),
    url(r'^products', Products),
    url(r'^product_create/$', Product_create),
    url(r'^product_update/(?P<product_id>[0-9]+)/$', Product_update),
    url(r'^product_delete/(?P<product_id>[0-9]+)/$', Product_delete),
    ############################################
    # url(r'^products', Products.as_view()),
    # url(r'^product_create', Product_create.as_view()),

    ]


import nnpu
import tvm
import topi
from nnpu.utils import ScheduleProcHelper
import numpy as np

def test():
    env = nnpu.get_env()
    shape = (16,)
    bigshape =(128,)
    dtype_n, dtype_w = env.cfg['dtype_n'], env.cfg['dtype_w']
    assert bigshape[0] % shape[0] == 0, 'the big vctr size is wrong'

    n_sheet=bigshape[0]//shape[0]
    sph = ScheduleProcHelper()

    a = tvm.placeholder(bigshape, dtype_n, 'a')
    b = tvm.placeholder(bigshape, dtype_n, 'b')
    a_buf, a_dram = nnpu.utils.CopyHtoBuf(a, 'a', sph)
    b_buf, b_dram = nnpu.utils.CopyHtoBuf(b, 'b', sph)

    strop = 'VAddV'

    c_buf = tvm.compute(bigshape, lambda *i: a_buf(*i) + b_buf(*i) , 'c_buf')
    sph.MarkScope(c_buf)
    c_host, c_dram = nnpu.utils.CopyBufToH(c_buf, 'sum',sph)

    s = tvm.create_schedule(c_host.op)
    sph.Transform(s)
    #tensorize
    xo,xi = s[c_buf].split(c_buf.op.axis[0], factor=shape[0])
    s[c_buf].reorder(xo,xi)
    s[c_buf].tensorize(xi, env.intrins.get(strop,  mode='n'))

    print(nnpu.lower(s, [a, b, c_host], simple_mode=True))

    func = nnpu.build(s, [a, b, c_host], 'nnpu', 'llvm', name='nnpu_func')

    ctx = tvm.nd.TVMContext(13, 0)
    a_np = np.random.randint(size=bigshape, dtype=a.dtype, low = -4, high = 4)
    a_nd = tvm.nd.array(a_np, ctx)
    b_np = np.random.randint(size=bigshape, dtype=b.dtype, low = -4, high = 4)
    b_nd = tvm.nd.array(b_np, ctx)

    c_nd = tvm.nd.array(np.zeros(bigshape, dtype=c_host.dtype), ctx)

    func(a_nd, b_nd, c_nd)
    print(strop)
    print(c_nd.asnumpy())
    gt = a_np + b_np
    np.testing.assert_allclose(c_nd.asnumpy(), gt)

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(description='test gemm with tiled/non-tiled data')
    parser.add_argument('--sim', type=str, help='the simulator to use',
                            default='S0', choices=['S0', 'S1', 'SC'])
    args = parser.parse_args()

    env = nnpu.get_env()
    nnpu.set_device(env, type=args.sim)
    test()


import json
import urllib.parse

from flask import request

from Database.Database import Database
from Shared.Events import EventManager, EventType
from Shared.Logger import Logger, LogVerbosity
from Shared.Network import RequestFactory
from Shared.Settings import Settings
from Shared.Util import to_JSON
from Webserver.APIController import app
from Webserver.Models import BaseMedia

class MovieController:

    movies_api_path = Settings.get_string("movie_api")
    server_uri = "http://localhost:50009/torrent"

    @staticmethod
    @app.route('/movies', methods=['GET'])
    def get_movies():
        page = int(request.args.get('page'))
        order_by = request.args.get('orderby')
        keywords = request.args.get('keywords')
        include_previous_pages = request.args.get('page') == "true"
        search_string = ""
        if keywords:
            search_string = "&keywords=" + urllib.parse.quote(keywords)

        if include_previous_pages:
            data = []
            current_page = 0
            while current_page != page:
                current_page += 1
                data += MovieController.request_movies(
                    MovieController.movies_api_path + "movies/" + str(current_page) + "?sort=" + urllib.parse.quote(
                        order_by) + search_string)

        else:
            data = MovieController.request_movies(MovieController.movies_api_path + "movies/" + str(page) + "?sort=" + urllib.parse.quote(order_by) + search_string)

        return to_JSON(data).encode()

    @staticmethod
    def request_movies(url):
        data = RequestFactory.make_request(url)

        if data is not None:
            return MovieController.parse_movie_data(data.decode('utf-8'))
        else:
            EventManager.throw_event(EventType.Error, ["get_error", "Could not get movie data"])
            Logger().write(LogVerbosity.Info, "Error fetching movies")
            return []

    @staticmethod
    @app.route('/movie', methods=['GET'])
    def get_movie_by_id():
        movie_id = request.args.get('id')
        Logger().write(LogVerbosity.Debug, "Get movie by id " + movie_id)
        response = RequestFactory.make_request(MovieController.movies_api_path + "movie/" + movie_id)
        data = json.loads(response.decode('utf-8'))

        seen = Database().get_history_for_id(movie_id)
        data['seen'] = len(seen) > 0
        if len(seen) > 0:
            seen = seen[-1]
            data['played_for'] = seen.played_for
            data['length'] = seen.length

        return json.dumps(data).encode('utf-8')

    @staticmethod
    def parse_movie_data(data):
        json_data = json.loads(data)
        if isinstance(json_data, list):
            return [Movie.parse_movie(x) for x in json_data]
        else:
            return Movie.parse_movie(json_data)

class Torrent:

    def __init__(self):
        self.url = None
        self.quality = None
        self.seeds = None
        self.peers = None
        self.size = None

class Movie(BaseMedia):

    def __init__(self, movie_id, poster, title):
        super().__init__(movie_id, poster, title)
        self.year = None
        self.rating = 0
        self.runtime = None
        self.genres = None
        self.synopsis = None
        self.youtube_trailer = None
        self.torrents = []
        self.released = None

    @staticmethod
    def parse_movie(movie_data):
        poster = ""
        if 'images' in movie_data:
            if 'poster' in movie_data['images']:
                poster = movie_data['images']['poster']
            elif len(movie_data['images']) > 0:
                poster = movie_data['images'][0]

        movie = Movie(
            movie_data['imdb_id'],
            poster,
            movie_data['title'])

        movie.rating = movie_data['rating']['percentage']
        movie.year = movie_data['year']
        movie.runtime = movie_data['runtime']
        movie.genres = movie_data['genres']
        movie.synopsis = movie_data['synopsis']
        movie.youtube_trailer = movie_data['trailer']
        movie.released = movie_data['released']
        movie.torrents = []
        for torrent_data in movie_data['torrents']['en']:
            torrent = Torrent()
            torrent.quality = torrent_data
            torrent_data = movie_data['torrents']['en'][torrent_data]
            torrent.url = torrent_data['url']
            torrent.seeds = torrent_data['seed']
            torrent.peers = torrent_data['peer']
            if 'size' in torrent_data:
                torrent.size = torrent_data['size']
            elif 'filesize' in torrent_data:
                torrent.size = torrent_data['filesize']
            movie.torrents.append(torrent)

        return movie


import csv
import requests
import re
import time
from bs4 import BeautifulSoup

def csv_dict_writer(path, fieldnames, data):
    """
    Writes a CSV file using DictWriter
    """
    with open(path, "w", encoding='utf8', newline='') as out_file:
        writer = csv.DictWriter(out_file, delimiter='#', fieldnames=fieldnames)
        writer.writeheader()
        for row in data:
            writer.writerow(row)

# var 1
def requester_with_soup_first(base_url, amount, delay_time):
    r = requests.get(BASE_URL)
    soap = BeautifulSoup(r.text, features="html.parser")
    ases = soap.find_all("a")
    data = ["id#url#date#header#content"]
    regexp = re.compile('[\/][n][e][w][s][\/].*')
    id = 0
    for i in ases:
        href = i.get("href")
        if regexp.match(str(href)) != None:
            id = id + 1
            url = "https://lenta.ru" + href
            if delay_time != 0:
                time.sleep(delay_time)
            r_page = requests.get(url)
            page = BeautifulSoup(r_page.text, features="html.parser")
            page_content = BeautifulSoup(str(page.find("div", {"class": "b-topic__content"})), features="html.parser")
            ppp = str(page_content.find_all("p"))
            content = BeautifulSoup(ppp, features="html.parser").text[1:-1].replace(".,", ".")
            date = page_content.time.contents[0]
            header = str(page_content.h1.contents[0]).replace("\\xa0", " ")
            data.append("{}#{}#{}#{}#{}".format(id, url, date, header, content))
            print(id)
            if id == amount: break
    return data

#var 2
def requester_with_soup_second_var(base_url, amount, delay_time):
    r = requests.get(BASE_URL)
    soap = BeautifulSoup(r.text, features="html.parser")
    div1 = str(soap.find("div", {"class": "touchcarousel-container"}))
    soap_div1 = BeautifulSoup(str(div1), features="html.parser")
    div2 = soap_div1.find_all("div", {'class':'touchcarousel-item ctg-slider__wrap top_goods'})
    #print(div2)
    data = ["id#url#category#name#description"]
    id = 0
    for i in div2:
        id = id + 1
        print(id)
        url = "https://www.e-katalog.ru" + i.a.get("href")
        if delay_time != 0:
            time.sleep(delay_time)
        r_page = requests.get(url)
        page = BeautifulSoup(r_page.text, features="html.parser")
        header = str.split(page.h1.text)
        categ = header[0]
        name = " ".join(header[1:])
        desc = page.find("div", {'class':'conf-desc-ai-title'}).text
        data.append("{}#{}#{}#{}#{}".format(id, url, categ, name, desc))
        if id == amount: break
    return data

#var 1
#BASE_URL = 'https://lenta.ru/rubrics/economics/'
BASE_URL = 'https://www.e-katalog.ru/k265.htm'
data = requester_with_soup_second_var(BASE_URL, 70, 0)

if __name__ == "__main__":
    data = [i.replace("\\xa0", " ").split("#") for i in data]
    print("OK")
    my_list = []
    print(data)
    fieldnames = data[0]
    for values in data[1:]:
        inner_dict = dict(zip(fieldnames, values))
        my_list.append(inner_dict)
    path = "dict_output.csv"
    csv_dict_writer(path, fieldnames, my_list)


bl_info = {"name":"Icicle Generator",
           "author":"Eoin Brennan (Mayeoin Bread), Tasha Upchurch (Thalia(HMG))",
           "version":(2,5,1),
           "blender":(2,82,0),
           "location":"3D View > Tools",
           "description":"Adds a linear string of icicles of different sizes along selected edges",
           "warning":"",
           "wiki_url":"",
           "tracker_url":"",
           "category":"Development"
           }

import bpy

from bpy.props import (
    BoolProperty,
    FloatProperty,
    IntProperty,
    EnumProperty,
    PointerProperty
)

from bpy.types import PropertyGroup

# import all teh ops and stuff
from . ig_panel import OBJECT_PT_CustomPanel
from . ig_gen_op import WM_OT_GenIcicle
# from . draw_op import OT_draw_operator

# Properties class to hold required parameters
class MyProperties(PropertyGroup):
    max_rad: FloatProperty(
        name='Max Radius',
        description='Maximum radius of a cone',
        default=0.15,
        min=0.01,
        max=1000.0,
        unit='LENGTH'
    )

    min_rad: FloatProperty(
        name='Min Radius',
        description='Minimum radius of a cone',
        default=0.025,
        min=0.01,
        max=1000.0,
        unit='LENGTH'
    )

    max_depth: FloatProperty(
        name='Max Depth',
        description='Maximum depth (height) of a cone',
        default=2.0,
        min=0.0,
        max=1000.0,
        unit='LENGTH'
    )

    min_depth: FloatProperty(
        name='Min Depth',
        description='Minimum depth (height) of a cone',
        default=1.5,
        min=0.0,
        max=1000.0,
        unit='LENGTH'
    )

    num_verts: IntProperty(
        name='Vertices',
        description='Number of vertices at base of cone',
        default=8,
        min=3,
        max=24
    )

    subdivs: IntProperty(
        name='Subdivides',
        description='Max number of kinks on a cone',
        default=3,
        min=0,
        max=128
    )

    max_its: IntProperty(
        name='Iterations',
        description='Number of iterations before giving up, prevents freezing/crashing',
        default=50,
        min=1,
        max=1000
    )

    reselect_base: BoolProperty(
        name='Reselect base mesh',
        description='Reselect the base mesh after adding icicles',
        default=True
    )

    on_selected_edges: BoolProperty(
        name='Only add to selected edges',
        description='Add icicles to selected edges only. Otherwise, applied to all edges in mesh',
        default=False
    )

    add_cap: EnumProperty(
        name='Fill',
        description='Fill the icicle cone base',
        items=[
            ('NGON', 'Ngon', 'Fill with Ngons'),
            ('NOTHING', 'None', 'Do not fill'),
            ('TRIFAN', 'Triangle fan', 'Fill with triangles')
        ],
        default='NGON'
    )

#classes = [MyProperties, OBJECT_PT_CustomPanel, OT_draw_operator, WM_OT_GenIcicle]
classes = [MyProperties, OBJECT_PT_CustomPanel, WM_OT_GenIcicle]

# Register/unregister classes
def register():
    from bpy.utils import register_class
    for cls in classes:
        register_class(cls)

    bpy.types.Scene.my_props = PointerProperty(type=MyProperties)

def unregister():
    from bpy.utils import unregister_class
    for cls in reversed(classes):
        unregister_class(cls)

    del bpy.types.Scene.my_props


# import ray
# ray.init()
from tensorboardX import SummaryWriter
import os

filelist = [ f for f in os.listdir('./experiment/') ]
for f in filelist:
    os.remove(os.path.join('./experiment/', f))

writer = SummaryWriter('./experiment/')

def run_experiment(agents, feeds, exp, num_episodes, r, env_type):
    import time
    from environment import FeedUnit, Feed

    cur_time = float(time.time())
    n = len(feeds)
    num_positive = 0
    for feed in feeds:
        if feed.interest > 0:
            num_positive += 1

#     print('num units: {}, num negative: {}, randomize: {}, experiment: {}'.format(n, n - num_positive, r, exp))

    envs = [
        Feed(feeds, num_positive, env_type) for _ in range(len(agents))
    ]

    agents_episode_reward = {}
    for agent in agents:
        # agents_cumulative_reward[agent.agent_name] = []
        agents_episode_reward[agent.agent_name] = []

    for j in range(num_episodes):
        if j % 100 == 0:
            print('num units: {}, num negative: {}, randomize: {}, experiment: {}'.format(n, n - num_positive, r, exp))
            print('Experiment {} at episode {}, used time {}'.format(exp, j, time.time() - cur_time))
            cur_time = time.time()
            for i, agent in enumerate(agents):
                print(agent.agent_name, agent.cum_rewards)

        for i, agent in enumerate(agents):
            envs[i].reset()
            agents[i].reset()
        for i in range(len(agents)):
            scroll = True
            # count = 0
            while scroll:
                action = agents[i].choose_action()
                # print('current_feed_interest: {}'.format(feeds[count].interest))
                # count += 1
                scroll, reward = envs[i].step(action)
                # print(reward)
                agents[i].update_buffer(scroll, reward)

            agents[i].learn_from_buffer()

            agents_episode_reward[agents[i].agent_name].append(agents[i].cum_rewards)

            if writer:
                if agents[i].agent_name != 'Oracle':
                    writer.add_scalar('running_loss_' + agents[i].agent_name + '_' + str(exp), agents[i].running_loss, j)
                writer.add_scalar('cumulative_reward_interest_unknown' + agents[i].agent_name + '_' + str(exp), agents[i].cum_rewards, j)

    return agents_episode_reward

# @ray.remote
def experiment_wrapper(feed_units, i, num_episodes, randomize, env_type):
    from supervised_agent import SupervisedAgent
    from supervised_agent_one_step import SupervisedAgentOneStep
    from dqn_agent import DQNAgent
    from deep_exp_hyper_agent import DeepExpHyperAgent
    from deep_exp_agent import DeepExpAgent
    from deep_exp_ids_agent import DeepExpIDSAgent
    import numpy as np

    deep_exp_agents = []
    num_positive = 0
    for feed in feed_units:
        if feed.interest > 0:
            num_positive += 1
    for prior in range(0, 1):
        deep_exp_agents.append(
            DeepExpAgent(
                [k for k in range(len(feed_units))],
                'deep_exploration_{}_{}_{}'.format(num_positive, len(feed_units), prior),
                prior_variance=10**prior,
#                     bootstrap=False
            )
        )
    agents = ([
        # OracleAgent(feed_units, session_size),
        SupervisedAgent([k for k in range(len(feed_units))], 'supervised_{}_{}'.format(num_positive, len(feed_units))),
        SupervisedAgentOneStep([k for k in range(len(feed_units))], 'supervised_one_step_{}_{}'.format(num_positive, len(feed_units))),
        DQNAgent([k for k in range(len(feed_units))], 'dqn_{}_{}'.format(num_positive, len(feed_units))),
        # DeepExpHyperAgent([k for k in range(len(feed_units))], 'hyper_deep_exploration_{}_{}'.format(num_positive, len(feed_units))),
        # DeepExpIDSAgent([k for k in range(len(feed_units))], 'ids_{}_{}'.format(num_positive, len(feed_units))),
    ]
    + deep_exp_agents
    )

    cumulative_reward = run_experiment(agents, feed_units, i, num_episodes, randomize, env_type)

    np.save('ids_experiment_{}_{}_{}_{}_{}'.format(
        num_positive, len(feed_units), int(randomize), i, env_type), cumulative_reward)

def generate_feeds(n, k):
    from environment import FeedUnit, Feed

    feed_units = []
    for i in range(k):
        feed_units.append(FeedUnit(-1, i))
    for i in range(n):
        feed_units.append(FeedUnit(1, i))

    return feed_units

def caller(n, k, num_experiment, num_episodes, randomize=False, env_type='sparse_reward'):
    import random

    feed_units = generate_feeds(n, k)

    if randomize:
        random.shuffle(feed_units)
#     writer = SummaryWriter('./experiment/')

    agents_cumulative_reward = []
    agents = []

    futures = []
    for i in range(num_experiment):
        experiment_wrapper(feed_units, i, num_episodes, randomize, env_type)
    #     futures.append(experiment_wrapper.remote(feed_units, i, num_episodes, randomize))
    # ray.get(futures)

#     with Pool(num_experiment) as p:
#         print(p.map(experiment_wrapper, [(feed_units, i, num_episodes, randomize) for i in range(num_experiment)]))

if __name__ == '__main__':
    num_experiments = 5
    inputs = []

    for n in range(5, 6):
        for k in range(2, 3):
            # for r in range(0, 1):
            num_episodes = 2000
            caller(n, k, num_experiments, num_episodes, env_type='sparse_reward')

    writer.close()
#     import torch.multiprocessing as mp
#     processes = []

#     for n in range(7, 8):
#         for k in range(3, 5):
#             for r in range(2):
#                 num_episodes = 10000 * (k + 1)
#                 caller(n, k, num_experiments, num_episodes, bool(r))
#                 p = mp.Process(target=caller, args=(n, k, num_experiments, num_episodes, bool(r)))
#                 p.start()
#                 processes.append(p)

#     print(len(processes))
#     for p in processes:
#         p.join()

#     from multiprocessing import Pool
#     with Pool(30) as p:
#         p.starmap(caller, inputs)


import pandas as pd
import numpy as np
from sklearn.manifold import TSNE as TSNE
import matplotlib.pyplot as plt
import json
import plotly
import plotly.graph_objs as go

def makeComparisonGraph(af,q4):
    merged = pd.merge(q4, af, on="listener_id")
    merged = merged.dropna()

    af_numeric = merged.copy()
    del af_numeric['listener_id']
    del af_numeric['username']

    af_numeric_embedded = TSNE(n_components=2).fit_transform(af_numeric)
    tsne_x = np.zeros(len(af_numeric.index))
    tsne_y = np.zeros(len(af_numeric.index))

    for i in range(len(af_numeric_embedded)):
        tsne_x[i] = af_numeric_embedded[i][0]
        tsne_y[i] = af_numeric_embedded[i][1]

    names = merged['username']

    data3 = [go.Scatter(
        x=tsne_x,
        y=tsne_y,
        mode="markers",
        hovertext=names
    )]

    #fig.update_layout(title_text="Spotify Share Music Taste Universe")
    #fig.update_xaxes(showticklabels=False)
    #fig.update_yaxes(showticklabels=False)

    # data3 = [fig]
    graphJSON3 = json.dumps(data3, cls=plotly.utils.PlotlyJSONEncoder)

    return graphJSON3


"""
Example of a neighbor MARL w/ electron spin
(1D representation w/ 2 neighbors: left and right)
- each agent has its own set of parameters, can be either spin up (1) or down (0)
- greedy policy: always takes Qmax action
- reward is calculated as -1 + sum(0.9 * neighbor's value)
- target value just r_t+1 without the discounted q-val of the next step
- in effect, circular grid, leftmost and rightmost electrons treated as neighbors
"""
import numpy as np
import torch
from matplotlib import pyplot as plt

def init_grid(size=(10,)):
    # pylint: disable=E1103
    """ Creates a random grid of 1/0s representing electron spin """
    grid_ = torch.randn(*size)
    grid_[grid_ > 0] = 1
    grid_[grid_ <= 0] = 0
    grid_ = grid_.byte()  # boolean
    return grid_

def get_rewards(list_binary, value_rewarded):
    """ Reward an electron agent based on its neighbors """
    total_reward = -1
    for neighbor_spin in list_binary:
        if neighbor_spin == value_rewarded:
            total_reward += 0.9
    total_reward *= 2.
    return total_reward

def gen_params(num_agents_, size):
    # pylint: disable=E1103
    """
    Create randomly inited parameters for N agents
    of dimensions <size>  (flattened) each
    """
    ret = []
    for _ in range(num_agents_):
        vec = torch.randn(size) / 10.
        vec.requires_grad = True
        ret.append(vec)
    return ret

def qfunc(state_input, theta, layers=((4, 20), (20, 2)), afn=torch.tanh):
    # pylint: disable=E1101
    """
    Q-model w/ 2 layers + elu, tanh activations
    - network dimensions adjustable as function input
    """
    l1n = layers[0]
    l1s = np.prod(l1n)  # get indices of theta for 1st layer
    theta_1 = theta[0:l1s].reshape(l1n)
    l2n = layers[1]
    l2s = np.prod(l2n)  # get indices of theta for 2nd layer
    theta_2 = theta[l1s:l2s + l1s].reshape(l2n)
    bias = torch.ones((1, theta_1.shape[1]))
    layer1_out = state_input @ theta_1 + bias
    layer1_out = torch.nn.functional.elu(layer1_out)
    layer2_out = afn(layer1_out @ theta_2)
    return layer2_out.flatten()

def get_substate(binary_value):
    # pylint: disable=E1101
    """ Encode binary to OHE """
    ohe_vector = torch.zeros(2)
    if binary_value > 0:
        ohe_vector[1] = 1
    else:
        ohe_vector[0] = 1
    return ohe_vector

def joint_state(neighbors):
    """ Get OH-encode states for neighbors <s>  as single vector """
    s1_ = get_substate(neighbors[0])
    s2_ = get_substate(neighbors[1])
    ret = (s1_.reshape(2, 1) @ s2_.reshape(1, 2)).flatten()
    return ret

def train_and_show():
    """ Run training loop and chart out grid before and after """
    # pylint: disable=E1101
    plt.figure(figsize=(8, 5))
    num_agents = (20,)
    hid_layer = 20
    params = gen_params(num_agents[0], 4 * hid_layer + hid_layer * 2)
    grid = init_grid(size=num_agents)
    shadow_grid = grid.clone()
    print(grid)
    plt.imshow(np.expand_dims(grid, 0))
    plt.show()

    # train
    epochs = 200
    learning_rate = 0.001
    losses = [[] for i in range(num_agents[0])]
    for i in range(epochs):
        for curr_agent in range(num_agents[0]):  # go thru each agent
            # get left neighbor
            left_neighbor = curr_agent - 1 if curr_agent - 1 >= 0 else num_agents[0] - 1
            # get right neighbor
            right_neighbor = curr_agent + 1 if curr_agent + 1 < num_agents[0] else 0
            state_ = grid[[left_neighbor, right_neighbor]]
            state = joint_state(state_)
            q_vals = qfunc(state.float().detach(),
                           params[curr_agent],
                           layers=[(4, hid_layer),
                                   [hid_layer, 2]])
            # policy - just take highest q-value action
            q_max = torch.argmax(q_vals, dim=0).detach().item()
            action = int(q_max)
            # only take action in cloned grid - update only when all agents have taken actions
            shadow_grid[curr_agent] = action
            reward = get_rewards(state_.detach(), action)
            # target is Q-value vector with action taken replaced by reward observed
            with torch.no_grad():
                target = q_vals.clone()
                target[action] = reward
            loss = torch.sum(torch.pow(q_vals - target, 2))
            losses[curr_agent].append(loss.detach().numpy())
            loss.backward()

            with torch.no_grad():  # manual backprop
                params[curr_agent] = params[curr_agent] - learning_rate * params[curr_agent].grad
            params[curr_agent].requires_grad = True

        # update grid state once all agent have taken actions
        with torch.no_grad():
            grid.data = shadow_grid.data

    _, axis = plt.subplots(2, 1)
    for i in range(num_agents[0]):
        axis[0].scatter(np.arange(len(losses[i])), losses[i])
    print(grid, grid.sum())
    axis[1].imshow(np.expand_dims(grid, 0))
    plt.show()

if __name__ == '__main__':
    train_and_show()


# Copyright (c) 2016 Universidade Federal Fluminense (UFF)
# Copyright (c) 2016 Polytechnic Institute of New York University.
# This file is part of noWorkflow.
# Please, consult the license terms in the LICENSE file.
"""'now run' command"""
from __future__ import (absolute_import, print_function,
                        division, unicode_literals)

import argparse
import os
import sys

from ..collection.metadata import Metascript
from ..persistence.models import Tag, Trial
from ..utils import io, metaprofiler

from .command import Command

def non_negative(string):
    """Check if argument is >= 0"""
    value = int(string)
    if value < 0:
        raise argparse.ArgumentTypeError(
            "{} is not a non-negative integer value".format(string))
    return value

class ScriptArgs(argparse.Action):                                               # pylint: disable=too-few-public-methods
    """Action to create script attribute"""
    def __call__(self, parser, namespace, values, option_string=None):
        if not values:
            raise argparse.ArgumentError(
                self, "can't be empty")

        script = os.path.realpath(values[0])

        if not os.path.exists(script):
            raise argparse.ArgumentError(
                self, "can't open file '{}': "
                "[Errno 2] No such file or directory".format(values[0]))

        setattr(namespace, self.dest, script)
        setattr(namespace, "argv", values)

def run(metascript):
    """Execute noWokflow to capture provenance from script"""
    try:
        metascript.trial_id = Trial.store(*metascript.create_trial_args())
        Tag.create_automatic_tag(*metascript.create_automatic_tag_args())

        io.print_msg("collecting definition provenance")
        metascript.definition.collect_provenance()
        metascript.definition.store_provenance()

        io.print_msg("collecting deployment provenance")
        metascript.deployment.collect_provenance()
        metascript.deployment.store_provenance()

        io.print_msg("collection execution provenance")
        metascript.execution.collect_provenance()
        metascript.execution.store_provenance()

        metaprofiler.meta_profiler.save()

    finally:
        metascript.create_last()

class Run(Command):
    """Run a script collecting its provenance"""

    def __init__(self, *args, **kwargs):
        super(Run, self).__init__(*args, **kwargs)
        self.default_context = "main"
        self.default_call_storage_frequency = 10000
        self.default_save_frequency = 0
        self.default_execution_provenance = "Profiler"

    def add_arguments(self):
        add_arg = self.add_argument
        add_cmd = self.add_argument_cmd
        add_arg("--name", type=str,
                help="set branch name used for tracking history")
        add_arg("--dir", type=str,
                help="set project path. The noworkflow database folder will "
                     "be created in this path. Default to script directory")
        # It will create both script and argv var
        add_cmd("script", nargs=argparse.REMAINDER, action=ScriptArgs,
                help="Python script to be executed")

        add_cmd("--create_last", action="store_true")
        add_arg("-v", "--verbose", action="store_true",
                help="increase output verbosity")
        add_arg("--meta", action="store_true",
                help="execute noWorkflow meta profiler")

        # Deployment
        add_arg("-b", "--bypass-modules", action="store_true",
                help="bypass module dependencies analysis, assuming that no "
                     "module changes occurred since last execution")

        # Definition
        add_arg("--disasm0", action="store_true",
                help="show script disassembly before noWorkflow change it")
        add_arg("--disasm", action="store_true",
                help="show script disassembly")

        # Execution
        add_arg("-d", "--depth", type=non_negative,
                default=sys.getrecursionlimit(),
                help="depth for capturing function activations (defaults to "
                     "recursion limit)")
        add_arg("-D", "--non-user-depth", type=non_negative, default=1,
                help="depth for capturing function activations outside the "
                     "selected context (defaults to 1)")
        add_arg("-e", "--execution-provenance",
                default=self.default_execution_provenance,
                choices=["Profiler", "Tracer"],
                help="execution provenance provider. (defaults to Profiler)")
        add_arg("-c", "--context", choices=["main", "package", "all"],
                default=self.default_context,
                help="functions subject to depth computation when capturing "
                     "activations (defaults to main)")
        add_arg("-s", "--save-frequency", type=non_negative,
                default=self.default_save_frequency,
                help="frequency (in ms) to save partial provenance")
        add_arg("-S", "--call-storage-frequency", type=non_negative,
                default=self.default_call_storage_frequency,
                help="frequency (in calls) to save partial provenance")

    def execute(self, args):
        if args.meta:
            metaprofiler.meta_profiler.active = True
            metaprofiler.meta_profiler.data["cmd"] = " ".join(sys.argv)

        io.verbose = args.verbose
        io.print_msg("removing noWorkflow boilerplate")

        # Create Metascript with params
        metascript = Metascript().read_cmd_args(args)

        # Set __main__ namespace
        import __main__
        metascript.namespace = __main__.__dict__

        # Clear boilerplate
        metascript.clear_sys()
        metascript.clear_namespace()

        # Run script
        run(metascript)


import os
import sys
from config import content_path
from subprocess import call

file_list = os.listdir(content_path)

print(file_list)

def replace_space(filename):
	return filename.replace(' ','\ ')

def convert_wmvfile(filename):
	filename2 = filename.replace('.wmv','.mp4')
	print(filename2)

	command = "ffmpeg -i "+replace_space(filename)+" "+replace_space(filename2)+" -hide_banner"
	print(command)
	os.system(command)
	os.remove(filename)
	print("**Deleted ",filename)

def convert_docx_pptfile(filename):
	command = "unoconv -f pdf "+replace_space(filename)
	print(command)
	os.system(command)
	os.remove(filename)
	print("**Deleted ",filename)

def conversion_call(filename):
	if filename.endswith('.wmv'):
		convert_wmvfile(filename)

	elif filename.endswith('.docx') or filename.endswith('.pptx'):
		convert_docx_pptfile(filename)

for file in os.listdir(content_path):
	path = content_path +"/"+ file
	for file1 in os.listdir(path):
		path2 = path+"/"+file1
		if os.path.isdir(path2):
			for file2 in os.listdir(path2):
				path3 = path2+"/"+file2
				conversion_call(path3)

		else:
			conversion_call(path2)

print("Converted")


import pygame
import random

pygame.init()
SCREEN_WIDTH = 700
SCREEN_HEIGHT = 500

# screen = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))

pygame.display.set_caption("Glass Ceiling")

background = pygame.image.load ("C:\Desktop\Adobe-SIP-2016\week6/background.jpg")
screen = pygame.display.set_mode(background_size)
done = False

# clock = pygame.time.clock()

# -------- Main Program Loop -----------
while not done:
	for event in pygame.event.get():
		if event.type == pygame.QUIT:
			done = True

	# screen.fill(WHITE)
	# gameDisplay.blit(bg, (0, 0))

	pygame.display.flip()

	# clock.tick(60)

pygame.quit()
exit()


import json

from marshmallow import Schema, fields, post_load, pre_load

from paynlsdk.api.requestbase import RequestBase
from paynlsdk.api.responsebase import ResponseBase
from paynlsdk.objects import ErrorSchema, RefundFailInfoSchema, RefundSuccessInfoSchema
from paynlsdk.validators import ParamValidator

class Response(ResponseBase):
    """
    Response object for the Transaction::refund API

    :param dict refunded_transactions: refunded transactions information
    :param dict failed_transactions: failed refunds information
    :param int amount_refunded: amount that was refunded
    :param str description: refunds description
    :param str refund_id: Refund ID. Please note this is NOT guaranteed. Value is only returned for IBAN based payments
    """
    def __init__(self,
                 refunded_transactions: dict={},
                 failed_transactions: dict={},
                 amount_refunded: int=None,
                 description: str=None,
                 refund_id: str=None,
                 *args, **kwargs):
        self.refunded_transactions = refunded_transactions
        self.failed_transactions = failed_transactions
        self.amount_refunded = amount_refunded
        self.description = description
        self.refund_id = refund_id
        super().__init__(**kwargs)

    def __repr__(self):
        return str(self.__dict__)

    def get_refunded_amount(self):
        """
        Get refunded amount
        :return: refunded amount
        :rtype: float
        """
        return self.response.amount_refunded / 100

class ResponseSchema(Schema):
    request = fields.Nested(ErrorSchema)
    refunded_transactions = fields.List(fields.Nested(RefundSuccessInfoSchema), allow_none=True, load_from='refundedTransactions')
    failed_transactions = fields.List(fields.Nested(RefundFailInfoSchema), allow_none=True, load_from='failedTransactions')
    amount_refunded = fields.Integer(load_from='amountRefunded', allow_none=True)
    description = fields.String(allow_none=True)
    refund_id = fields.String(load_from='refundId', allow_none=True, required=False)

    @pre_load
    def pre_processor(self, data):
        # Again, the API returns an empty string where it SHOULD return null or an empty list.
        if 'refundedTransactions' in data and ParamValidator.is_empty(data['refundedTransactions']):
            data['refundedTransactions'] = []
        elif 'refundedTransactions' in data and ParamValidator.not_empty(data['refundedTransactions']):
            #  v2.x has NO fields.Dict implementation like fields.List, so we'll have to handle this ourselves
            list = []
            for item in data['refundedTransactions']:
                list.append(item)
            data['refundedTransactions'] = list
        if 'failedTransactions' in data and ParamValidator.is_empty(data['failedTransactions']):
            data['failedTransactions'] = []
        elif 'failedTransactions' in data and ParamValidator.not_empty(data['failedTransactions']):
            #  v2.x has NO fields.Dict implementation like fields.List, so we'll have to handle this ourselves
            list = []
            for item in data['failedTransactions']:
                list.append(item)
            data['failedTransactions'] = list
        return data

    @post_load
    def create_response(self, data):
        #  This is NASTY. Perform conversion due to fields.Dict NOT taking nesteds in 2.x (aka undo preprocessing).
        #  This should be fixed in 3.x but that's a pre-release
        if 'refunded_transactions' in data:
            rs = {}
            for item in data['refunded_transactions']:
                print(item)
                rs[item.order_id] = item
            data['refunded_transactions'] = rs
        #  This is NASTY. Perform conversion due to fields.Dict NOT taking nesteds in 2.x (aka undo preprocessing).
        #  This should be fixed in 3.x but that's a pre-release
        if 'failed_transactions' in data:
            rs = {}
            for item in data['failed_transactions']:
                rs[item.order_id] = item
            data['failed_transactions'] = rs
        return Response(**data)

class Request(RequestBase):
    """
    Request object for the Refund::transaction API

    :param str transaction_id: Refund ID
    :param int amount: refund amount
    :param str description: refunds description
    :param str process_date: process date
    """
    def __init__(self,
                 transaction_id: str=None,
                 amount: int=None,
                 description: str=None,
                 process_date: str=None,
                 ):
        self.transaction_id = transaction_id
        self.amount = amount
        self.description = description
        self.process_date = process_date
        super().__init__()

    def requires_api_token(self):
        return True

    def requires_service_id(self):
        return False

    def get_version(self):
        return 13

    def get_controller(self):
        return 'Transaction'

    def get_method(self):
        return 'refund'

    def get_query_string(self):
        return ''

    def get_parameters(self):
        # Validation
        ParamValidator.assert_not_empty(self.transaction_id, 'transaction_id')
        # Get default api parameters
        rs = self.get_std_parameters()
        # Add own parameters
        rs['transactionId'] = self.transaction_id
        if ParamValidator.not_empty(self.amount):
            rs['amount'] = self.amount
        if ParamValidator.not_empty(self.description):
            rs['description'] = self.description
        if ParamValidator.not_empty(self.process_date):
            rs['processDate'] = self.process_date
        return rs

    @RequestBase.raw_response.setter
    def raw_response(self, raw_response):
        self._raw_response = raw_response
        # Do error checking.
        rs = json.loads(self.raw_response)
        schema = ResponseSchema(partial=True)
        response, errors = schema.load(rs)
        self.handle_schema_errors(errors)
        self._response = response

    @property
    def response(self) -> Response:
        """
        Return the API :class:`Response` for the validation request

        :return: The API response
        :rtype: paynlsdk.api.refund.transaction.Response
        """
        return self._response

    @response.setter
    def response(self, response: Response):
        # print('{}::respone.setter'.format(self.__module__ + '.' + self.__class__.__qualname__))
        self._response = response


import numpy as np
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt

def plotValdPerformances():
    x_vals      = np.array([6, 9, 13, 18, 27, 60])
    constant    = np.array([2.976, 3.334, 3.408, 3.150, 3.159, 3.397])
    constant_err = np.array([0.212, 0.170, 0.368, 0.222, 0.421, 0.179])
    depth       = np.array([3.080, 3.025, 3.265, 3.366, 3.321, 3.320])
    depth_err   = np.array([0.495, 0.363, 0.397, 0.530, 0.147, 0.277])
    reverse     = np.array([3.462, 3.141, 2.897, 2.902, 2.797, 3.196])
    reverse_err = np.array([0.462, 0.429, 0.131, 0.180, 0.260, 0.128])

    const_line = plt.errorbar(x_vals, constant, yerr=constant_err, fmt='-bo', capsize=4)
    depth_line = plt.errorbar(x_vals + 0.5, depth, yerr=depth_err, fmt='-ro', capsize=4)
    rever_line = plt.errorbar(x_vals + 1.0, reverse, yerr=reverse_err, fmt='-go', capsize=4)

    const_line.set_label('Constant Filters')
    depth_line.set_label('Normal Filters')
    rever_line.set_label('Reverse Filters')
    ax = plt.gca()
    ax.legend()
    plt.title('Comparison of 3D models on MRI imaging data')
    plt.xlabel('Stride Size')
    plt.ylabel('Mean Square Error')
    plt.show()

def plotDownsamplingPerformances():
    x_vals      = np.array([1, 2, 3])
    baseline    = np.array([3.541, 3.229, 3.848])
    baseline_err= np.array([0.279, 0.338, 0.486])
    depth       = np.array([3.359, 3.531, 3.923])
    depth_err   = np.array([0.274, 0.408, 0.356])
    reverse     = np.array([2.692, 2.884, 2.878])
    reverse_err = np.array([0.326, 0.171, 0.121])

    base_line = plt.errorbar(x_vals, baseline, yerr=baseline_err, fmt='-bo', capsize=4)
    depth_line = plt.errorbar(x_vals + 0.05, depth, yerr=depth_err, fmt='-ro', capsize=4)
    rever_line = plt.errorbar(x_vals - 0.05, reverse, yerr=reverse_err, fmt='-go', capsize=4)

    base_line.set_label('Baseline')
    depth_line.set_label('Depth Slicing')
    rever_line.set_label('Reverse Slicing')
    ax = plt.gca()
    ax.legend()
    plt.title('Comparison of 3D models on MRI imaging data')
    plt.xlabel('Downsampling rate')
    plt.ylabel('Mean Square Error')
    plt.show()

def plotTestPerformances():
    y_pos=np.arange(6)
    names=np.array(['depth_stride9', 'constant_stride6', 'reverse_stride_27', 'baseline', 'r_baseline', 'c_baseline'])
    title='Test Performance of best 3D models'
    p=np.array([4.257, 4.758, 3.652, 4.150, 4.137, 4.481])
    err = np.array([0.635, 1.602, 0.328, 0.407, 0.339, 0.815])
    plt.xticks(y_pos, names, rotation=45)
    plt.title(title)
    plt.ylabel('Mean Square Error')
    plt.bar(y_pos, p, align='center', alpha=0.5, yerr=err, capsize=30)
    plt.show()

def plotTrainingTime():
    x_vals      = np.array([1, 2, 3])
    baseline    = np.array([480, 65, 30])
    depth       = np.array([130, 23, 15])
    reverse     = np.array([270, 40, 21])

    [base_line]       = plt.plot(x_vals, baseline, '-bo')
    [depth_line]      = plt.plot(x_vals, depth, '-ro')
    [reverse_line]    = plt.plot(x_vals, reverse, '-go')

    base_line.set_label('Baseline')
    depth_line.set_label('Depth Slicing')
    reverse_line.set_label('Reverse Slicing')
    ax = plt.gca()
    ax.legend()

    for xy in zip(x_vals, baseline):
        ax.annotate('{}'.format(xy[1]), xy=(xy[0], xy[1]+7), textcoords='data')
    for xy in zip(x_vals, depth):
        ax.annotate('{}'.format(xy[1]), xy=(xy[0], xy[1]-13), textcoords='data')
    for xy in zip(x_vals, reverse):
        ax.annotate('{}'.format(xy[1]), xy=(xy[0]+0.01, xy[1]), textcoords='data')

    plt.title('Comparison of time taken by 3D models on MRI imaging data')
    plt.xlabel('Downsampling Rate')
    plt.ylabel('Time (minutes) / 50k iterations')
    plt.show()

def plotNumberParams():
    x_vals      = np.array([1,2,3,4,5,6])
    constant_fc     = np.array([16640, 16640, 131328, 131328, 442624, 1310976])
    constant_conv   = np.array([2313792, 1209600, 912384, 820800, 787968, 775782])
    depth_fc        = np.array([16640, 16640, 131328, 131328, 442624, 1310976])
    depth_conv      = np.array([411912, 273888, 236736, 225228, 221184, 219672])
    reverse_fc      = np.array([2304, 2304, 16640, 16640, 55552, 164096])
    reverse_conv    = np.array([1759104, 654912, 357696, 266112, 233280, 221184])

    constant_plot_bot = plt.bar(x_vals-0.25, constant_conv, 0.25)
    constant_plot_top = plt.bar(x_vals-0.25, constant_fc, 0.25, bottom=constant_conv)
    depth_plot_bot = plt.bar(x_vals, depth_conv, 0.25)
    depth_plot_top = plt.bar(x_vals, depth_fc, 0.25, bottom=depth_conv)
    reverse_plot_bot = plt.bar(x_vals+0.25, reverse_conv, 0.25)
    reverse_plot_top = plt.bar(x_vals+0.25, reverse_fc, 0.25, bottom=reverse_conv)
    plt.legend((constant_plot_bot, constant_plot_top, depth_plot_bot, depth_plot_top, reverse_plot_bot, reverse_plot_top),
               ('Constant: Conv.', 'Constant: FC', 'Depth: Conv.', 'Depth: FC', 'Reverse: Conv.', 'Reverse: FC'))
    tick_ind = []
    tick_names = []
    strides = [6, 9, 13, 18, 27, '_baseline']
    for i in range(6):
        tick_ind.append(i + 1 - 0.25)
        tick_ind.append(i + 1)
        tick_ind.append(i + 1 + 0.25)
        tick_names.append('constant{}'.format(strides[i]))
        tick_names.append('depth{}'.format(strides[i]))
        tick_names.append('reverse{}'.format(strides[i]))
    plt.xticks(tick_ind, tick_names, rotation=60)

    plt.title('Comparison of number of parameters of 3D models on MRI imaging data')
    plt.xlabel('Stride Size')
    plt.ylabel('Number of Parameters')
    plt.show()

def scatterParamPerformance():
    constant_fc     = np.array([16640, 16640, 131328, 131328, 442624, 1310976])
    constant_conv   = np.array([2313792, 1209600, 912384, 820800, 787968, 775782])
    depth_fc        = np.array([16640, 16640, 131328, 131328, 442624, 1310976])
    depth_conv      = np.array([411912, 273888, 236736, 225228, 221184, 219672])
    reverse_fc      = np.array([2304, 2304, 16640, 16640, 55552, 164096])
    reverse_conv    = np.array([1759104, 654912, 357696, 266112, 233280, 221184])
    constant_test   = np.array([4.758, 5.057, 5.439, 3.910, 3.983, 4.481])
    depth_test      = np.array([4.276, 4.257, 3.902, 4.312, 4.058, 4.150])
    reverse_test    = np.array([8.591, 4.023, 3.865, 3.609, 3.652, 4.137])
    constant_err    = np.array([1.602, 0.977, 1.942, 0.399, 0.490, 0.815])
    depth_err       = np.array([0.084, 0.635, 0.496, 0.452, 0.317, 0.407])
    reverse_err     = np.array([1.539, 0.640, 0.736, 0.614, 0.328, 0.339])
    const_line = plt.errorbar(constant_conv + constant_fc, constant_test, fmt='bo', capsize=4)
    depth_line = plt.errorbar(depth_conv + depth_fc, depth_test, fmt='ro', capsize=4)
    rever_line = plt.errorbar(reverse_conv + reverse_fc, reverse_test, fmt='go', capsize=4)
    const_line.set_label('Constant Filters')
    depth_line.set_label('Normal Filters')
    rever_line.set_label('Reverse Filters')
    ax = plt.gca()
    ax.legend()
    plt.title('Comparison of 3D models on MRI imaging data')
    plt.xlabel('Number of Parameters')
    plt.ylabel('Mean Square Error')
    plt.show()

def alignmentScatter():
    inFile = '/home/psturm/brain_age_prediction/summaries/alignmentComp/reverseScale2DataPNCBatch4Rate0.0001AlignedDropout0.6/performance.txt'
    with open(inFile) as fin:
        content = fin.readlines()
    content = [x.strip() for x in content]
    content = [content[i] for i in [1,2,3,5,6,7]]
    content = [(x.split('['))[1].split(']')[0] for x in content]
    content = [[float(y) for y in x.split(', ')] for x in content]
    vald_loss = content[0]
    vald_dice = content[2]
    test_loss = content[3]
    test_dice = content[5]

    plt.scatter(test_dice, test_loss)
    plt.plot(np.unique(test_dice), np.poly1d(np.polyfit(test_dice, test_loss, 1))(np.unique(test_dice)))
    plt.title('Scatter Plot of Test Loss vs. Dice Coefficient')
    plt.xlabel('Dice Coefficient')
    plt.ylabel('Test Loss')
    plt.savefig('TestDice.png', bbox_inches='tight')

if __name__ == '__main__':
    alignmentScatter()


from os.path import dirname, abspath
from pathlib import Path
from typing import Any
from ntpath import split
from os import makedirs
from json import dumps, load
from paraview.simple import *

SRC_DIR = Path(dirname(abspath(__file__)))
PROJ_DIR = SRC_DIR.parent

class SharedState(object):
    case_list = []
    m_widget = None
    cam_props_for_del = []
    slices_for_del = []
    cases_for_del = []

class FileHandling(object):
    @classmethod
    def write_json(cls, obj, path: str) -> None:
        def get_object_dict(d):
            return d.__dict__

        filepath, _ = split(path)
        if filepath:
            makedirs(filepath, exist_ok=True)
        with open(path, "w") as out:
            json_string = dumps(obj, default=get_object_dict)
            out.write(json_string)

    @classmethod
    def read_json(cls, inp: Path, object_hook_=None) -> Any:
        with open(inp, "r") as fin:
            data = load(fin, object_hook=object_hook_)

        return data

    @classmethod
    def read_foamcase(cls, inp: Path):
        foamcase_path = inp / "temp.foam"
        cls.write_file(foamcase_path)
        foamcase = OpenFOAMReader(FileName=str(foamcase_path))
        Path.unlink(foamcase_path)
        return foamcase

    @classmethod
    def write_file(cls, file_path: Path, text: str = "", mode: str = 'w'):
        dir_path = file_path.parent
        dir_path.mkdir(parents=True, exist_ok=True)
        with open(file_path, mode) as out:
            out.write(text)

if __name__ == "__main__":
    print(PROJ_DIR)


import pandas as pd
import numpy as np
import pandas_datareader.data as web
import empyrical as ep
import scipy.stats as stats
import warnings
import dash_html_components as html
import pickle
import time
from functools import lru_cache

with warnings.catch_warnings():
    warnings.filterwarnings(
        'ignore',
        'axes.color_cycle is deprecated',
        UserWarning,
        'matplotlib',
    )
    import seaborn as sns

@lru_cache(typed=False) ##### the single most important line in the app
def get_raw_backtest(backtest_file):
    with open(backtest_file, 'rb') as f:
        history, run_config = pickle.load(f)
    return history, run_config

@lru_cache(typed=False)
def get_backtest(file_name, data_dir ="/home/user/Backtests/processed/"):
    with open(data_dir+ file_name, 'rb') as f:
        data = pickle.load(f)
    return data

def get_summary(backtest_files):

    sum_headers = ["Backtest name", "Average return, %", "Sharpe ratio", "Max drawdown from start, %"]

    names_list = [x[:len(x) - 7] for x in backtest_files]

    data_list = [get_backtest(x) for x in backtest_files]
    returns_list = [get_common_returns(data) for data in data_list]
    sharpe_list = ["{0:.3g}".format(ep.sharpe_ratio(x)) for x in returns_list]

    AR_list = []
    dd_from_start_list = []
    for k in data_list:
        AR, dd_from_start = stat_from_start(k.Rcum)
        AR_list.append("{0:.3g}".format(AR))
        dd_from_start_list.append("-{0:.3g}".format(dd_from_start))

    return sum_headers, pd.Series(names_list), pd.Series(AR_list), pd.Series(sharpe_list), pd.Series(dd_from_start_list)

# processing backtest results
def get_common_returns(data):
    dailyCumReturns = data.Rcum.resample("B").last()
    returns = ep.simple_returns(dailyCumReturns)
    returns = returns.fillna(method="ffill")

    return returns

def stat_from_start(dailyCumReturns):

    initial_capital = 50000

    # drawdown manually
    local_max = dailyCumReturns.cummax()
    dd_from_start = (np.max((local_max - dailyCumReturns)) / initial_capital *100)

    dailyCumReturns = dailyCumReturns.resample("B").last()
    R = (dailyCumReturns.iloc[1:] - dailyCumReturns.iloc[:-1].values) / initial_capital
    return_yearly = R.mean() * 252
    AR = return_yearly * 100

    return AR, dd_from_start

def get_benchmark_info(benchmark_ticker='SPX'):

    #since we are aware of the period of the backtests from the very beginning
    #this part is hard-coded
    start = pd.Timestamp(2015, 1, 14)
    end = pd.Timestamp(2018, 4, 6)
    benchmark = web.DataReader("%s" % benchmark_ticker, 'morningstar', start, end)

    prices = pd.Series(data=benchmark.Close.reset_index().Close.values,
                       index=benchmark.Close.reset_index().Date)
    benchmark_rets = prices.pct_change().iloc[1:]
    benchmark_rets = benchmark_rets.fillna(method="bfill")

    benchmark_rets = benchmark_rets.tz_localize("UTC")

    return benchmark_rets

def process_bckst_for_risk_analysis(data):

    '''
    @returns: non-cumulative returns,
              margin_contract,
              margin_MValue,
              AR, dd_from_start,
              benchmark_rets
    '''

    initial_capital = 50000
    BPS = 10000
    multiplier = 1000

    #total_ret_pct = 100 * (data.Rcum[-1] - initial_capital) / initial_capital
    total_ret_usd = data.Rcum[-1] - initial_capital

    margin_contract = np.around(total_ret_usd / data.TrdVolume.abs().sum(), 4)  # margin in usd/contracts
    margin_MValue = np.around(total_ret_usd / (data.TrdMVal.abs().sum() * multiplier) * BPS, 4)  # margin in bps

    AR, dd_from_start = stat_from_start(data.Rcum)

    returns = get_common_returns(data)

    return returns, margin_contract, margin_MValue, AR, dd_from_start

#for performance graphs
def get_info_for_performance_graph(data, benchmark_ticker='SPX'):

    BacktestReturns = data.Rcum.resample("B").last()

    BacktestReturns = BacktestReturns.fillna(method="ffill")

    return BacktestReturns

def get_info_for_underwater_graph(data):

    returns = get_common_returns(data)

    df_cum_rets = ep.cum_returns(returns, starting_value=1.0)
    running_max = np.maximum.accumulate(df_cum_rets)
    underwater = -100 * ((running_max - df_cum_rets) / running_max)
    underwater = underwater[:len(returns)]

    return underwater

#for graphs depending on margin
def calc_ret(data, var_cost, start_capital=50000):
    table = data.copy()

    costs = np.cumsum(var_cost * (table.TrdMVal.abs()) * 1000)
    Rcum = start_capital + np.cumsum(table.R * 1000) - costs

    dailyCumReturns = Rcum.resample("B").last()
    returns = dailyCumReturns.pct_change().iloc[1:]

    return returns

def margin_stat(data):

    initial_capital = 50000

    total_ret_usd = data.Rcum[-1] - initial_capital

    BPS = 10000
    multiplier = 1000
    margin_MValue = round(total_ret_usd / (data.TrdMVal.abs().sum() * multiplier) * BPS, 4)  # margin in bps

    costs = np.arange(0.0, margin_MValue / BPS, 0.01)

    #calculating different returns depending on different costs
    returns_list = [calc_ret(data, var_cost) for var_cost in costs]

    an_ret_list = [ep.annual_return(returns) for returns in returns_list]
    sharpe_list = [ep.sharpe_ratio(returns) for returns in returns_list]
    dd_list = [ep.max_drawdown(returns) for returns in returns_list]

    an_returns = pd.Series(an_ret_list, index=costs) * 100
    sharpe = pd.Series(sharpe_list, index=costs)
    dd = pd.Series(dd_list, index=costs) * 100

    return an_returns, sharpe, dd

#for stats table depending on costs
def calc_costs_stat(data, commission, slippage, pip = 15.625):

    trade_data = data[data.TrdVolume != 0]
    total_trades = len(trade_data)
    # number of trades in-position
    #number_in = len([trade_data.index[i] for i in range(len(trade_data.index)) if
         #      time.strftime(str(trade_data.index[i]))[11:] == time.strftime("17:30:00+00:00")])
    # number of no trades - when just waiting for expiration
    wait_1 = len([trade_data.index[i] for i in range(len(trade_data.index)) if
                time.strftime(str(trade_data.index[i]))[11:] == time.strftime("21:50:00+00:00")])
    wait_2 = len([trade_data.index[i] for i in range(len(trade_data.index)) if
                  time.strftime(str(trade_data.index[i]))[11:] == time.strftime("22:10:00+00:00")])
    wait_3 = len([trade_data.index[i] for i in range(len(trade_data.index)) if
                  time.strftime(str(trade_data.index[i]))[11:] == time.strftime("20:50:00+00:00")])
    wait_4 = len([trade_data.index[i] for i in range(len(trade_data.index)) if
                  time.strftime(str(trade_data.index[i]))[11:] == time.strftime("21:00:00+00:00")])
    wait_5 = len([trade_data.index[i] for i in range(len(trade_data.index)) if
                  time.strftime(str(trade_data.index[i]))[11:] == time.strftime("21:10:00+00:00")])

    num_need_to_pay = total_trades - (wait_1 + wait_2 + wait_3 + wait_4 + wait_5)
    contract_multiplier = num_need_to_pay / total_trades
    strangle_multiplier = contract_multiplier / 2

    cumRet = data.Rcum
    slip_costs = np.cumsum((data.TrdVolume * strangle_multiplier) * (slippage*pip))
    com_costs = np.cumsum((data.TrdVolume * contract_multiplier) * commission)
    costs = slip_costs + com_costs
    ret = cumRet - costs
    ret = ret.resample("B").last()
    returns = ret.pct_change().iloc[1:]

    AR, dd_from_start = stat_from_start(ret)

    return returns, AR, dd_from_start

######################################################################################################################
#################################------MAKING TABLES------############################################################
######################################################################################################################

func_list = [ep.annual_return, ep.cum_returns_final, ep.annual_volatility, ep.sharpe_ratio,
             ep.calmar_ratio, ep.stability_of_timeseries, ep.max_drawdown, ep.omega_ratio,
             ep.sortino_ratio, stats.skew, stats.kurtosis, ep.tail_ratio]

stat_name_col = ["Annual return, %", "Cum_returns, %", "Annual volatility, %", "Sharpe ratio", "Calmar ratio",
                  "Stability", "Max dd, %", "Omega ratio", "Sortino ratio", "Skew", "Kurtosis", "Tail ratio",
                  "Average return, %", "Max dd from start, %"]

def make_costs_def(returns, AR, dd_from_start):

    data_list = ["{0:.3g}".format(k(returns)*100) if i <= 2 or i == 6 else "{0:.3g}".format(k(returns))
                 for i, k in enumerate(func_list)]
    data_list.append("{0:.3g}".format(AR))
    data_list.append("-{0:.3g}".format(dd_from_start))

    df = pd.DataFrame(data={
        "Stats": stat_name_col,
        "Tes": data_list
    })

    return df

def make_pandas_df(returns, benchmark_rets, margin_contract, margin_MValue, AR, dd_from_start):

    stat_list = stat_name_col.copy()
    added_func = ["Alpha", "Beta", "Margin (1 contract), usd/contract", "Margin (Mvalue), bps"]
    stat_list.extend(added_func)

    data_list = ["{0:.3g}".format(k(returns) * 100) if i <= 2 or i == 6 else "{0:.3g}".format(k(returns))
                 for i, k in enumerate(func_list)]

    added_values = [AR, -dd_from_start, ep.alpha(returns, benchmark_rets), ep.beta(returns, benchmark_rets)]
    values = rnd(added_values)
    values.extend([round(margin_contract,2), round(margin_MValue,2)])
    data_list.extend(values)

    df = pd.DataFrame(data= {
        "Stats" : stat_list,
        "Tes" : data_list
    })

    return df

# beautiful digit output
def rnd(col):
    col = ["{0:.3g}".format(x) for x in col]
    return col

#calculating stats
def comparison_table(returns_list):

    cols = []
    for b in returns_list:
        back_col = [k(b)*100 if i <= 2 or i == 6 else k(b)  for i, k in enumerate(func_list)]
        cols.append(back_col)

    res_cols = [rnd(x) for x in cols]

    return res_cols

#Strategy descriptions
description_dict = {
    "51WednesdayAll" : [
        html.P('STRATEGY: 5:1 Wednesday'),
        html.P('ASSET: U.S. Treasury Bond Options'),
        html.P('1. Selling 5 and buying 1 strangles at the distance of 1.5 SD'),
        html.P('2. Closing all positions if spot goes 0.5 figure out of our strangle '),
        html.P('3. Hedge is used'),
        html.P('4. Trades are being executed on Wednesday'),
        html.P('5. Trading options with 9 days to expiration')
    ],
    "1WednesdayAll" : [
        html.P('STRATEGY: 1 Wednesday'),
        html.P('ASSET: U.S. Treasury Bond Options'),
        html.P('1. Selling 1 strangle at the distance of 1.5 SD'),
        html.P('2. Closing all positions if spot goes 0.5 figure out of our strangle '),
        html.P('3. No hedge is used'),
        html.P('4. Trades are being executed on Wednesday'),
        html.P('5. Trading options with 9 days to expiration')
    ],
    "51TuesdayAll" : [
        html.P('STRATEGY: 5:1 Tuesday'),
        html.P('ASSET: U.S. Treasury Bond Options'),
        html.P('1. Selling 5 and buying 1 strangles at the distance of 1.5 SD'),
        html.P('2. Closing all positions if spot goes 0.5 figure out of our strangle '),
        html.P('3. Hedge is used'),
        html.P('4. Trades are being executed on Tuesday'),
        html.P('5. Trading options with 10 days to expiration')
    ],
    "1TuesdayAll" : [
        html.P('STRATEGY: 1 Tuesday'),
        html.P('ASSET: U.S. Treasury Bond Options'),
        html.P('1. Selling 1 strangle at the distance of 1.5 SD'),
        html.P('2. Closing all positions if spot goes 0.5 figure out of our strangle '),
        html.P('3. No hedge is used'),
        html.P('4. Trades are being executed on Tuesday'),
        html.P('5. Trading options with 10 days to expiration')
    ],
    "51FridayAll" : [
        html.P('STRATEGY: 5:1 Friday'),
        html.P('ASSET: U.S. Treasury Bond Options'),
        html.P('1. Selling 5 and buying 1 strangles at the distance of 1.5 SD'),
        html.P('2. Closing all positions if spot goes 0.5 figure out of our strangle '),
        html.P('3. Hedge is used'),
        html.P('4. Trades are being executed on Friday'),
        html.P('5. Trading options with 14 days to expiration')
    ],
    "1FridayAll" : [
        html.P('STRATEGY: 1 Friday'),
        html.P('ASSET: U.S. Treasury Bond Options'),
        html.P('1. Selling 1 strangle at the distance of 1.5 SD'),
        html.P('2. Closing all positions if spot goes 0.5 figure out of our strangle '),
        html.P('3. No hedge is used'),
        html.P('4. Trades are being executed on Friday'),
        html.P('5. Trading options with 14 days to expiration')
    ]
}


# -*- coding: utf-8 -*-
import sys
import os
import tempfile
from datetime import datetime
import subprocess

import ipaddress
import requests

LOG_TS_FORMAT = "%d/%b/%Y:%H:%M:%S"

def load_aws_networks():
    try:
        r = requests.get('https://ip-ranges.amazonaws.com/ip-ranges.json')
    except Exception as e:
        print('Error making https request : {}'.format(e))
        sys.exit(1)
    if r.status_code != 200:
        return []
    return [ipaddress.ip_network(item.get('ip_prefix')) for item in r.json().get('prefixes')]

AWS_NETWORKS = load_aws_networks()

def from_aws(ip):
    is_aws = False
    for nw in AWS_NETWORKS:
        is_aws = is_aws or (ipaddress.ip_address(ip) in nw)
    return is_aws

def _scraped_off_part_not_i_need(origin_log):
    # *.gz と分ける
    cat_cmd = "cat"
    decompress = " | gunzip " if os.path.splitext(origin_log)[1] == ".gz" else ""
    no_bot_no_assets = '| grep "GET /.*/ HTTP" | grep -v -i "bot"'
    i_need_column = '| cut -d " " -f 1,4,14-' #IP, 日時, User-Agentだけ
    tmp_path = tempfile.mkstemp(prefix='eachip', suffix='.log')[1]
    cmd = '{0} {1} {2} {3} {4} > {5}'.format(
        cat_cmd,
        os.path.abspath(origin_log),
        decompress,
        no_bot_no_assets,
        i_need_column,
        tmp_path)
    print (cmd)
    subprocess.call(cmd, shell=True)
    return tmp_path

def open_log():
    if len(sys.argv) < 2:
        return []
    _, fname = tuple(sys.argv)
    return open(_scraped_off_part_not_i_need(fname))

def timedeltas_each_ip(stream):
    hits_each_ips = {}
    for line in stream:
        ip, ts = tuple(line.replace("[", "").split(" ")[0:2])
        # IP毎にtsの配列化を行う
        if ip not in hits_each_ips.keys():
            hits_each_ips[ip] = []
        hits_each_ips[ip].append(datetime.strptime(ts, LOG_TS_FORMAT))
    # IP毎のアクセスタイムスタンプが得られた
    # アクセスの間隔をIP毎に計算
    return [(ip, [end - begin for begin, end in zip(times[:-1], times[1:])])
                for ip, times in hits_each_ips.items()]

def list_of_ips(timedeltas_each_ip):
    for ip, deltas in timedeltas_each_ip:
        delta_seconds = [d.seconds for d in deltas]
        aws = 'AWS' if from_aws(ip) else ''
        if len(delta_seconds) > 5:
            yield ('{0} {1:.1f} {2} {3}'.format(
                ip,
                sum(delta_seconds)/len(delta_seconds),
                len(delta_seconds),
                aws))

def main():
    print ("ip_address | avg interval(sec) | counts | AWS?")
    print ("----------------------------------------------")
    for out in list_of_ips(timedeltas_each_ip(open_log())):
        print(out)

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
# _*_ encoding: utf-8 _*_

import socket
import struct
import json

tcp_server = socket.socket()
ip_port = ('127.0.0.1', 8080)
tcp_server.bind(ip_port)

tcp_server.listen()
conn, addr = tcp_server.accept()

# 首先接受到的文件信息长度转换出来的4个字节的数据
file_info_stru = conn.recv(4)

#解包文件信息的长度
file_info_len = struct.unpack('i', file_info_stru)[0]

client_file_info = conn.recv(file_info_len).decode('utf-8')

abc_file_info = json.loads(client_file_info)
print('abc_file_infp>>>', abc_file_info)

client_file_size = abc_file_info['file_size']


"""Furcifar Processing Module

This module contains functions to perform a preprocessing of whole-slide images of the
CAMELYON data sets to be used  to train a convolutional neural network for metastasis
localisation.
"""

import logging
import math
from typing import Tuple, Iterator
from datetime import datetime

import numpy as np
from skimage import filters
from skimage.draw import polygon as ski_polygon
from skimage.measure import label as ski_label

from .datamodel import Slide
from .util import ProgressBar, LogStyleAdapter

logger = LogStyleAdapter(logging.getLogger('preprocessing.processing'))

def remove_alpha_channel(image: np.ndarray) -> np.ndarray:
    """Remove the alpha channel of an image.

    Parameters
    ----------
    image : np.ndarray
        RGBA image as numpy array with W×H×C dimensions.

    Returns
    -------
    np.ndarray
        RGB image as numpy array
    """
    if len(image.shape) == 3 and image.shape[2] == 4:
        return image[::, ::, 0:3:]
    else:
        return image

def rgb2gray(rgb: np.ndarray) -> np.ndarray:
    """Convert RGB color image to a custom gray scale for HE-stained WSI

    by Jonas Annuscheit

    Parameters
    ----------
    rgb : np.ndarray
        Color image.

    Returns
    -------
    np.ndarray
        Gray scale image as float64 array.
    """
    gray = 1.0 * rgb[::, ::, 0] + rgb[::, ::, 2] - (
        (1.0 * rgb[::, ::, 0] + rgb[::, ::, 1] + rgb[::, ::, 2])
        / 1.5)
    gray[gray < 0] = 0
    gray[gray > 255] = 255
    return gray

def create_otsu_mask_from_slide(slide: Slide, level) -> np.ndarray:
    """Creates an otsu mask from of a slide layer given the level.

    Parameters
    ----------
    slide : Slide
        Input slide.

    level : int
        Level of the layer used to create the mask.

    Returns
    -------
    np.ndarray
        Binary Otsu mask with value 1 as foreground and 0 as background.
    """
    image = np.asarray(slide.get_full_slide(level))
    image = remove_alpha_channel(image)

    return _create_otsu_mask_by_image(image)

def _create_otsu_mask_by_image(image) -> np.ndarray:
    """Create a binary mask of an image based on the Otsu methode.

    by Jonas Annuscheit

    Parameters
    ----------
    image : PIL.Image.Image or np.ndarray
        Image used to create the mask.

    Returns
    -------
    np.ndarray
        The generated binary masks has value 1 in foreground areas and 0s everywhere
        else (background).
    """
    if not isinstance(image, np.ndarray):
        slide_img = np.asarray(image)
    else:
        slide_img = image

    gray_slide = 1.0 * slide_img[::, ::, 0] + slide_img[::, ::, 2] - (
        (1.0 * slide_img[::, ::, 0] + slide_img[::, ::, 1] + slide_img[::, ::, 2]) / 1.5)
    gray_slide[gray_slide < 0] = 0
    gray_slide[gray_slide > 255] = 255

    threshold = filters.threshold_otsu(gray_slide, nbins=256)
    otsu_mask = gray_slide > threshold
    otsu_mask2 = gray_slide > threshold * 0.25

    otsu_mask2_labeled = ski_label(otsu_mask2)
    for i in range(1, otsu_mask2_labeled.max()):
        if otsu_mask[otsu_mask2_labeled == i].sum() == 0:
            otsu_mask2_labeled[otsu_mask2_labeled == i] = 0
    otsu_mask3 = otsu_mask2_labeled
    otsu_mask3[otsu_mask3 > 0] = 1

    return otsu_mask3.astype(np.uint8)

def create_otsu_mask_by_threshold(image: np.ndarray, threshold) -> np.ndarray:
    """Create a binary mask separating fore and background based on the otsu threshold.

    Parameters
    ----------
    image : np.ndarray
        Gray scale image as array W×H dimensions.

    threshold : float
        Upper Otsu threshold value.

    Returns
    -------
    np.ndarray
        The generated binary masks has value 1 in foreground areas and 0s everywhere
        else (background)
    """
    otsu_mask = image > threshold
    otsu_mask2 = image > threshold * 0.25

    otsu_mask2_labeled = ski_label(otsu_mask2)
    for i in range(1, otsu_mask2_labeled.max()):
        if otsu_mask[otsu_mask2_labeled == i].sum() == 0:
            otsu_mask2_labeled[otsu_mask2_labeled == i] = 0
    otsu_mask3 = otsu_mask2_labeled
    otsu_mask3[otsu_mask3 > 0] = 1

    return otsu_mask3.astype(np.uint8)

def _otsu_by_hist(hist, bin_centers) -> float:
    """Return threshold value based on Otsu's method using an images histogram.

    Based on skimage's threshold_otsu method without histogram generation.

    Parameters
    ----------
    hist : np.ndarray
        Histogram of a gray scale input image.

    bin_centers: np.ndarray
        Centers of the histogram's bins.

    Returns
    -------
    threshold : float
        Upper threshold value. All pixels with an intensity higher than
        this value are assumed to be foreground.

    References
    ----------
    Wikipedia, http://en.wikipedia.org/wiki/Otsu's_Method

    See Also
    --------
    skimage.filters.threshold_otsu
    """
    hist = hist.astype(float)

    # class probabilities for all possible thresholds
    weight1 = np.cumsum(hist)
    weight2 = np.cumsum(hist[::-1])[::-1]

    # class means for all possible thresholds
    mean1 = np.cumsum(hist * bin_centers) / weight1
    mean2 = (np.cumsum((hist * bin_centers)[::-1]) / weight2[::-1])[::-1]

    # Clip ends to align class 1 and class 2 variables:
    # The last value of `weight1`/`mean1` should pair with zero values in
    # `weight2`/`mean2`, which do not exist.
    variance12 = weight1[:-1] * weight2[1:] * (mean1[:-1] - mean2[1:]) ** 2

    idx = np.argmax(variance12)
    threshold = bin_centers[:-1][idx]
    return threshold

def add_dict(left, right):
    """Merge two dictionaries by adding common items.

    Parameters
    ----------
    left: dict
        Left dictionary.

    right
        Right dictionary

    Returns
    -------
    dict
        Resulting dictionary
    """
    return {k: left.get(k, 0) + right.get(k, 0) for k in left.keys() | right.keys()}

def get_otsu_threshold_chunked(slide: Slide, level=0, step_size=1000,
                               verbose=False) -> float:
    """Calculate the otsu threshold by reading in the slide in chunks.

    To avoid memory overflows the slide image will be loaded in by chunks of the size
    $slide width × `step_size`$. A histogram will be generated of these chunks that will
    be used to calculate the otsu threshold based on skimage's `threshold_otsu` function.

    Parameters
    ----------
    slide : Slide
        Whole slide image slide

    level : int
        Level/layer of the `slide` to be used. Use of level ≠ 0 is not advised, see notes.

    step_size : int
        Each chunk loaded will have the size $slide-width × `step_size`$ on the level 0
        slide. For higher levels the step will be downsampled accordingly (e.g.: with a
        `step_size` of 1000 and `level` of 1 and a downsample factor of 2 the actual size
        of each chunk is $level-1-slide width × 500$.

    verbose : bool
        If true print progress information.

    Returns
    -------
    otsu_threshold : float
        Upper threshold value. All pixels with an intensity higher than
        this value are assumed to be foreground.
    """
    logger.info('Calculating chunked otsu threshold from {} on level {}.', slide.name,
                level)
    size = slide.level_dimensions[0]
    downsample = slide.level_downsamples[level]

    # dictionary with all unique values and counts of the whole slide
    slide_count_dict = {}

    total_chunks = math.ceil(size[1] / step_size)
    logger.debug('{} chunks to load with chunk size of {:,}×{:,}.',
                 total_chunks, size[0], step_size)
    if verbose:
        bar_suffix = '%(percent)3d%% | Chunks %(index)3d / %(max)d ' \
                     '[%(elapsed_fmt)s | eta: %(remaining_fmt)s]'

        bar = ProgressBar(f'Otsu: {slide.name:20}',
                          max=total_chunks, suffix=bar_suffix)

    for i, y in enumerate(range(0, size[1], step_size)):
        if verbose:
            bar.next()

        # check if next step exceeds the image height and adjust it if needed
        cur_step = step_size if size[1] - y > step_size else size[1] - y

        # read in the image and transform to gray scale
        start, cut_size = (0, y), (int(size[0] / downsample), int(cur_step / downsample))
        a_img_cut = np.asarray(slide.read_region(start, level, cut_size))
        a_img_cut = rgb2gray(a_img_cut)

        # get unique values and their count
        chunk_count_dict = dict(zip(*np.unique(a_img_cut, return_counts=True)))

        # add those values and count to the dictionary
        slide_count_dict = add_dict(slide_count_dict, chunk_count_dict)

    if verbose:
        bar.finish()

    # transform dictionary back to a arrays and calculate otsu threshold
    unique_values, counts = tuple(np.asarray(x) for x in zip(*slide_count_dict.items()))
    threshold = _otsu_by_hist(counts, unique_values)

    return threshold

def get_otsu_threshold(slide: Slide, level=3) -> float:
    """Calculate the otsu threshold of a slide on layer `level`.

    Parameters
    ----------
    slide: Slide
        Whole image slide
    level: int
        Whole image slide layer number

    Returns
    -------
    otsu_threshold: float
        Upper threshold value. All pixels with an intensity higher than
        this value are assumed to be foreground.

    References
    ----------
    Wikipedia, http://en.wikipedia.org/wiki/Otsu's_Method

    See Also
    --------
    skimage.filters.threshold_otsu
    """
    if level < 3:
        logger.warning('Level under 3 might cause memory overflows!')

    img = remove_alpha_channel(np.asarray(slide.get_full_slide(level)))
    gray_slide = rgb2gray(img)
    return filters.threshold_otsu(gray_slide, nbins=256)

def split_negative_slide(slide: Slide, level, otsu_threshold,
                         poi_threshold=0.01, tile_size=128,
                         overlap=5, verbose: bool = False):
    """Create tiles from a negative slide.

    Iterator over the slide in `tile_size`×`tile_size` Tiles. For every tile an otsu mask
    is created and summed up. Only tiles with sums over the percental threshold
    `poi_threshold` will be yield.

    Parameters
    ----------
    slide : Slide
        Input Slide.

    level : int
        Layer to produce tiles from.

    otsu_threshold : float
        Otsu threshold of the whole slide on layer `level`.

    poi_threshold : float, optional
        Minimum percentage, 0 to 1, of pixels with tissue per tile. (Default 0.01; 1%)

    tile_size : int
        Pixel size of one side of a square tile the image will be split into.
        (Default: 128)

    overlap : int, optional
        Count of pixel overlapping between two tiles. (Default: 30)

    verbose : Boolean, optional
        If set to True a progress bar will be printed to stdout. (Default: False)

    Yields
    -------
    image_tile : np.ndarray
        Array of (`tile_size`, `tile_size`) shape containing tumorous tissue.

    bounds : tuple
        Tile boundaries on layer 0: ((x, y), (width, height))
    """
    if tile_size <= overlap:
        raise ValueError("Overlap has to be smaller than the tile size.")
    if overlap < 0:
        raise ValueError("Overlap can not be negative.")
    if otsu_threshold < 0:
        raise ValueError("Otsu threshold can not be negative.")
    if not 0.0 <= poi_threshold <= 1.0:
        raise ValueError("PoI threshold has to be between 0 and 1")

    width0, height0 = slide.level_dimensions[0]
    downsample = slide.level_downsamples[level]

    # tile size on level 0
    tile_size0 = int(tile_size * downsample + 0.5)
    overlap0 = int(overlap * downsample + 0.5)

    if verbose:
        # amount of tiles horizontally and vertically
        count_horitonzal = int(
            math.ceil((width0 - (tile_size0 - overlap0)) / (tile_size0 - overlap0) + 1))
        count_vertical = int(
            math.ceil((height0 - (tile_size0 - overlap0)) / (tile_size0 - overlap0) + 1))

        bar_suffix = '%(percent)3d%% | Tiles: %(index)d / %(max)d ' \
                     '[%(elapsed_fmt)s | eta: %(remaining_fmt)s]'
        bar = ProgressBar(f'Processing: {slide.name:20}',
                          max=count_horitonzal * count_vertical,
                          suffix=bar_suffix)
        print('verbose on')
        print('count_vertical:' , count_vertical)
        print('count_horitonzal:' , count_horitonzal)

    min_poi_count = tile_size ** 2 * poi_threshold

    for yi, y in enumerate(range(0, height0, tile_size0 - overlap0)):
        if verbose: print('row', yi, 'of', count_vertical, ' -- time: ', datetime.now())
        for xi, x in enumerate(range(0, width0, tile_size0 - overlap0)):
            tile = np.asarray(slide.read_region((x, y), level, (tile_size, tile_size)))
            mask = create_otsu_mask_by_threshold(rgb2gray(tile), otsu_threshold)
            poi_count = np.sum(mask)
            if poi_count >= min_poi_count:
                yield remove_alpha_channel(tile), ((x, y), (tile_size0, tile_size0))
            if verbose:
                bar.next()
    if verbose:
        bar.finish()

def create_tumor_mask(slide: Slide, level, bounds=None):
    """Create a tumor mask for a slide or slide section.

    If `bounds` is given the tumor mask of only the section of the slide will be
    calculated.

    Parameters
    ----------
    slide : Slide
        Tissue slide.

    level : int
        Slide layer.

    bounds : tuple, optional
        Boundaries of a section as: ((x, y), (width, height))
        Where x and y are coordinates of the top left corner of the slide section on
        layer 0 and width and height the dimensions of the section on the specific
        layer `level`.  (Default: None)

    Returns
    -------
    tumor_mask : np.ndarray
        Binary tumor mask of the specified section. Healthy tissue is represented by 0,
        cancerous by 1.
    """
    if bounds is None:
        start_pos = (0, 0)
        size = slide.level_dimensions[level]
    else:
        start_pos, size = bounds

    mask = np.zeros((size[1], size[0]), dtype=np.uint8)
    downsample = slide.level_downsamples[level]

    for i, annotation in enumerate(slide.annotations):
        c_values, r_values = list(zip(*annotation.polygon))
        r = np.array(r_values, dtype=np.float32)
        r -= start_pos[1]
        r /= downsample
        r = np.array(r + 0.5, dtype=np.int32)

        c = np.array(c_values, dtype=np.float32)
        c -= start_pos[0]
        c /= downsample
        c = np.array(c + 0.5, dtype=np.int32)

        rr, cc = ski_polygon(r, c, shape=mask.shape)
        mask[rr, cc] = 1

    return mask

def split_positive_slide(slide: Slide, level, tile_size=128, overlap=5,
                         poi_threshold=None,
                         verbose=False) -> Iterator[Tuple[np.ndarray, Tuple]]:
    """Create tiles from a positive slide.

    Iterator over the slide in `tile_size`×`tile_size` Tiles. For every tile a tumor mask
    is created and summed up.

    Parameters
    ----------
    slide : Slide
        Input Slide.

    level : int
        Layer to produce tiles from.

    tile_size : int, optional
        Pixel size of one side of a square tile the image will be split into.
        (Default: 128)

    overlap : int, optional
        Count of pixel overlapping between two tiles. (Default: 30)

    poi_threshold : float or None, optional
        Minimum percentage, 0 to 1, of pixels with metastasis per tile or None for tiles
        with at least one tumor pixel. (Default: None)

    verbose : Boolean, optional
        If set to True a progress bar will be printed to stdout. (Default: False)

    Yields
    -------
    image_tile : np.ndarray
        Array of (`tile_size`, `tile_size`) shape containing tumorous tissue.

    bounds : tuple
        Tile boundaries on layer 0: ((x, y), (width, height))

    verbose : Boolean, optional
        If set to True a progress bar will be printed to stdout. (Default: False)
    """
    if not slide.annotations:
        raise ValueError("Slide {} has no annotations.".format(slide.name))
    if tile_size <= overlap:
        raise ValueError("Overlap has to be smaller than tile_size.")
    if overlap < 0:
        raise ValueError("Overlap can not be negative.")

    width0, height0 = slide.level_dimensions[0]
    downsample = slide.level_downsamples[level]

    # tile size on level 0
    tile_size0 = int(tile_size * downsample + 0.5)
    overlap0 = int(overlap * downsample + 0.5)

    if verbose:
        count_horitonzal = int(
            math.ceil((width0 - (tile_size0 - overlap0)) / (tile_size0 - overlap0) + 1))
        count_vertical = int(
            math.ceil((height0 - (tile_size0 - overlap0)) / (tile_size0 - overlap0) + 1))

        bar_suffix = '%(percent)3d%% | Tiles: %(index)d / %(max)d ' \
                     '[%(elapsed_fmt)s | eta: %(remaining_fmt)s]'
        bar = ProgressBar(f'Processing: {slide.name:20}',
                          max=count_horitonzal * count_vertical,
                          suffix=bar_suffix)
    min_poi_count = tile_size ** 2 * poi_threshold if poi_threshold is not None else 1
    for yi, y in enumerate(range(0, height0, tile_size0 - overlap0)):
        mask_row = create_tumor_mask(slide, level, ((0, y), (width0, tile_size)))
        if mask_row.sum() > 0: ### lets skip rows without any tumor
            for xi, x in enumerate(range(0, width0, tile_size0 - overlap0)):

                if level != 0:
                    mask = create_tumor_mask(slide, level, ((x, y), (tile_size, tile_size)))
                    poi_count = np.sum(mask)
                if level == 0:
                    poi_count = np.sum(mask_row[:,x:(x+tile_size)])

                logger.debug('Tile ({:2},{:2}) PoI count: {:6,}', yi, xi, poi_count)
                if poi_count >= min_poi_count:
                    tile = slide.read_region((x, y), level, (tile_size, tile_size))

                    tile = remove_alpha_channel(np.asarray(tile))
                    yield tile, ((x, y), (tile_size0, tile_size0))

                if verbose:
                    bar.next()

    if verbose:
        bar.finish()