import urllib.request
import os
import re
import sys

def url_open(url):
    req = urllib.request.Request(url)
    req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36')
    response = urllib.request.urlopen(url)
    html = response.read()

    return html

def find_imgs(url):
    html = url_open(url).decode('gbk')
    img_addres = []

    reg = r'http://ymz.qqwmb.com/allimg/c.*\.gif'
    for each in re.findall(reg,html):
        url = each
        img_addres.append(url)
    return img_addres

def save_imgs(folder,img_addres):
    for each in img_addres:
        filename = each.split('/')[5]
        with open(filename,'wb') as f:
            img = url_open(each)
            f.write(img)

def download_mm(folder='xxoo'):
    try:
        os.mkdir(folder)
        os.chdir(folder)
    except:
        os.chdir(folder)
    url = 'http://www.youmzi.com/meinvgif.html'
    reg = 'meinvgif_[0-9][0-9]?.html' ##找到最后一页的网址的正则
    pages = re.findall(reg, url_open(url).decode('gbk'))

    reg ='[0-9]+'
    lastpage = re.findall(reg,pages[-1])[0]

    for page in range(5,int(lastpage)):
        print('正在下载第%d页...'%(page+1))
        url = "http://www.youmzi.com/meinvgif_"+str(page+1)+".html"
        img_adress = find_imgs(url)
        save_imgs(folder,img_adress)

if __name__ == "__main__":
    download_mm()


class Stack:
    def __init__(self, size):
        self._data = []
        self._size = size

    def push(self, data):
        if (len(self._data) + 1) > self._size:
            print('stack is full')
            return

        self._data.append(data)

    def pop(self):
        if len(self._data) == 0:
            print('stack is empty')
            return None

        return self._data.pop()  # 默认返回最后一个

    def length(self):
        print('current stack length is {}/{}'.format(len(self._data), self._size))

    def __iter__(self):
        return iter(reversed(self._data))

def main():
    stack = Stack(3)
    stack.push(1)
    stack.push(2)
    stack.push(3)
    stack.length()
    # stack.push(4) # stack is full
    print(stack.pop())
    print(stack.pop())
    print(stack.pop())
    stack.length()
    # print(stack.pop())  # stack is empty

    print()

    stack = Stack(100)  # iterable

    for i in range(10):
        stack.push(i + 1) # init data
    stack.length()

    for data in stack:
        print(data, end=' ')
    print()
    stack.length()

if __name__ == '__main__':
    main()


import stft
import cluster
import numpy as np

import matplotlib.pyplot as plt

if __name__ == "__main__":

    in_dir = "/Users/bergamaschi/Documents/HumpbackSong/dataset/"
    stft.main(in_dir=in_dir)

    k_vals = np.logspace(1, 4, 10, base=10, dtype=int)

    inertia_scores = []

    for k in k_vals:
        inertia = cluster.main(k=k, return_inertia=True)
        inertia_scores.append(inertia)

    fig, ax = plt.subplots(figsize=(5, 5))

    print(k_vals)
    print(inertia_scores)

    plt.plot(k_vals, inertia_scores, marker="x")
    plt.title("Inertia vs. K Value")
    plt.xlabel("K Value")
    plt.ylabel("Inertia")
    plt.show()


import os
import tempfile
import unittest

from itolapi import Itol

class TestIntegration(unittest.TestCase):
    def test_upload_download(self):
        current_dir = os.path.dirname(os.path.realpath(__file__))
        examples_dir = os.path.join(current_dir, '..', '..', 'examples')

        test = Itol()
        tree = os.path.join(examples_dir, 'tree_of_life.tree.txt')
        test.add_file(tree)
        dataset = os.path.join(examples_dir, 'colors_tol.txt')
        test.add_file(dataset)
        upload_status = test.upload()

        self.assertTrue(upload_status)

        exporter = test.get_itol_export()
        exporter.set_export_param_value('format', 'pdf')
        with tempfile.NamedTemporaryFile() as temp:
            exporter.export(temp.name)


from graphthreading import Controller,GraphLock,GraphThread,GraphCondition
from threading import Thread,current_thread
from time import sleep
import graphthreading

class Structure:
    def __init__(self):
        self.lock = GraphLock()
        self.condition = GraphCondition(self.lock)
        self.lock.setName('test')

    def get(self):
        self.lock.acquire()
        self.lock.release()

class MyThread(GraphThread):
    def __init__(self,structure,sctructure2,structure3):
        super().__init__()
        self.structures=structure
        self.structure2 = sctructure2
        self.structure3 = structure3

    def run(self):
        while True:
            self.structures.get()
            sleep(5)
            self.structure2.get()
            sleep(5)
            self.structure3.get()
            sleep(5)

structure = Structure()
structure2 = Structure()
structure3 = Structure()

threads = []

for i in range(4):
    t = MyThread(structure3,structure2,structure)
    threads.append(t)

for t in threads:
    t.start()

graphthreading.startGraph()


# coding:utf-8
"""
create on May 12, 2020 By Wayne YU

Function:

V2：
在研究俄罗斯断网事件的过程，对出口AS网络做了详细的研究
需进一步分析，我国出口AS互联关系中，美国占比的情况
分为两类，一类是全部互联关系；一类是AS TOP 1000的互联关系占比情况

V3:
新需求，需要研究三大运营商出口AS互联关系中，美国占比的情况
分为两类，一类是全部互联关系；一类是AS TOP 1000的互联关系占比情况

经统计三大运营商出口AS(v4部分)为：

电信:4134、4809、4812、4813
联通:4808、4837、9929、17621、17623
移动（含铁通）：9394、9808、24400、56041

V4:
我国互联网网络关键基础设施布局研究-国际通信网络部分数据分析支撑
我国电信运营商AS网络对外互联方向分国家统计图

经统计我国三大运营商主要出口AS为：

电信:4134、4809、4812、4813
联通:4808、4837、9929、17621、17623
移动（含铁通）：9394、9808、24400、56041

V5:
继续支撑我国互联网网络关键基础设施布局研究-国际通信网络部分数据分析支撑
我国电信运营商AS网络对外互联方向分国家统计图

增加输出Excel表，并将国家信息和缩写对应

V6:
继续支撑我国互联网网络关键基础设施布局研究-国际通信网络部分数据分析支撑
限定运营商

1）中国电信，4134（163网）、4809（CN2网）
2）中国联通，4837（169网）、9929（IP承载A网）
3）中国移动，9808（CMNET网）

一次互联，分别统计各运营商出口至全球各国的互联关系
二次互联，分别统计各运营商及其互联单位出口至全球各国的互联关系

V7:
统计4134、4809、4837、9929、9808五张网，分别的出口互联关系（分为一次互联和二次互联）

"""

import time
import csv
import os
import matplotlib.pyplot as plt

def write_to_csv(res_list, des_path):
    """
    把给定的List，写到指定路径的文件中
    :param res_list:
    :param des_path:
    :return: None
    """
    print("write file <%s> ..." % des_path)
    csvFile = open(des_path, 'w', newline='', encoding='gbk')
    try:
        writer = csv.writer(csvFile, delimiter=",")
        for i in res_list:
            writer.writerow(i)
    except Exception as e:
        print(e)
    finally:
        csvFile.close()
    print("write finish!")

def gain_as2country(as_info_file, country):
    """
    根据传入的as info file信息获取AS与国家的对应字典及该国家的所有的AS Info
    :param as_info_file:
    :param country:
    :return country_as_info:
    :return as2country:
    """
    country_as_info = []  # 存储country as 信息
    as2country = {}  # 存储as号到country的映射关系
    file_read = open(as_info_file, 'r', encoding='utf-8')
    # for line in file_read.readlines():
    #     line = line.strip().split("|")
    #     as2country[line[0]] = line[8]  # 生成字典
    #     temp_list = []
    #     if line[8] == country:
    #         temp_list.append(line[0])  # AS Number
    #         temp_list.append(line[1])  # AS All Relationships
    #         temp_list.append(line[5])  # AS Name
    #         temp_list.append(line[7])  # Source
    #         temp_list.append(line[8])  # Country
    #         country_as_info.append(temp_list)

    for line in file_read.readlines():
        line = line.strip().split("\t")
        # print(line)
        as_number = line[0]
        as_name = line[1].strip().split(",")[0].strip()
        as_country = line[1].strip().split(",")[-1].strip()
        as2country[as_number] = as_country
        temp_list = []
        if as_country == country:
            temp_list.append(as_number)
            temp_list.append(as_name)
            temp_list.append(as_country)
            country_as_info.append(temp_list)

    return country_as_info, as2country

def external_as_analysis(country, as2country, as_list, org_name):
    """
    根据输入的国家，统计该国家的出口AS数量及其互联方向的统计分析
    :param country:
    :param as2country:
    :param as_list:
    :param org_name:
    :return external_country_as:
    """
    print(country)
    external_country_as_return = []
    # 获取1998-2020年间全球BGP互联关系的存储文件
    file_path = []
    for root, dirs, files in os.walk("..\\000LocalData\\as_relationships\\serial-1"):
        for file_item in files:
            file_path.append(os.path.join(root, file_item))

    for path_item in file_path[-5:-4]:
        print(path_item)
        # 遍历一次文件，获取该国出口AS的数量
        file_read = open(path_item, 'r', encoding='utf-8')
        external_cnt = 0  # 存储该国出口连边的数量
        external_as_list = []  # 存储出口AS
        external_country_list = []  # 存储该国出口方向的国家
        external_country_as = []  # 存储该国出口互联关系中，国外AS网络
        for line in file_read.readlines():
            if line.strip().find("#") == 0:
                continue
            try:
                if str(line.strip().split('|')[0]) in as_list:
                    if as2country[str(line.strip().split('|')[1])] != country:
                        external_cnt += 1
                        external_as_list.append(str(line.strip().split('|')[0]))
                        external_country_list.append(as2country[str(line.strip().split('|')[1])])
                        external_country_as.append(str(line.strip().split('|')[1]))

                if as2country[str(line.strip().split('|')[0])] != country:
                    if str(line.strip().split('|')[1]) in as_list:
                        external_cnt += 1
                        external_as_list.append(str(line.strip().split('|')[1]))
                        external_country_list.append(as2country[str(line.strip().split('|')[0])])
                        external_country_as.append(str(line.strip().split('|')[0]))
            except Exception as e:
                pass
        external_country_as_return = external_country_as
        # 统计出口AS的排名
        external_as_rank = {}
        for item in list(set(external_as_list)):
            external_as_rank[item] = 0
        for item in external_as_list:
            external_as_rank[item] += 1
        # 将字典转换为列表
        external_as_rank_list = []
        temp_list = []
        for item in external_as_rank.keys():
            temp_list.append(item)
            temp_list.append(external_as_rank[item])
            external_as_rank_list.append(temp_list)
            temp_list = []
        external_as_rank_list.sort(reverse=True, key=lambda elem: int(elem[1]))
        external_as_list = list(set(external_as_list))
        print("All External Edges Count:", external_cnt)
        print("All External AS Count(Abroad):", len(list(set(external_country_as))))
        external_country_as_country = {}  # 统计三家运营商直联国际网络数量的国家分布
        external_country_as_us = []  # 存储美国AS网络
        for item_as in list(set(external_country_as)):
            # print(as2country[item_as])
            if as2country[item_as] == "US":
                external_country_as_us.append(item_as)

            if as2country[item_as] not in external_country_as_country.keys():
                external_country_as_country[as2country[item_as]] = 1
            else:
                external_country_as_country[as2country[item_as]] += 1
        # 将字典转化为列表
        external_country_as_country_list = []  # 存储三家运营商直联国际网络数量的国家分布
        for key in external_country_as_country.keys():
            external_country_as_country_list.append([key, external_country_as_country[key]])
        external_country_as_country_list.sort(reverse=True, key=lambda elem: elem[1])
        # print(external_country_as_country_list)
        # print("All External AS Count(Abroad(US)):", external_country_as_country["US"])
        # print(external_country_as_us)

        print("All External AS Count(Internal):", len(external_as_list))
        # print(external_as_rank_list)
        print("All External Country Count:", len(list(set(external_country_list))))

        # 统计互联国家方向的排名
        external_country_rank = {}
        for item in list(set(external_country_list)):
            external_country_rank[item] = 0
        for item in external_country_list:
            external_country_rank[item] += 1
        # print(len(external_country_rank))

        # 将字典转为列表
        external_country_rank_list = []
        temp_list = []
        for item in external_country_rank.keys():
            temp_list.append(item)
            temp_list.append(external_country_rank[item])
            external_country_rank_list.append(temp_list)
            temp_list = []
        # print(external_country_rank_list)
        external_country_rank_list.sort(reverse=True, key=lambda elem: int(elem[1]))
        gain_external_country_rank_list_plus(external_country_rank_list, country, org_name)
        draw_bar(external_country_rank_list, country, org_name)
    return external_country_as_return

def gain_external_country_rank_list_plus(rank_list, country, org_name):
    """
    根据传入的rank_list，将其国家缩写与国家信息、大洲信息相对应
    :param rank_list:
    :return:
    """
    # print(rank_list)
    geo_file = "..\\000LocalData\\as_geo\\GeoLite2-Country-Locations-zh-CN.csv"
    code2country = {}  # 存储country_iso_code到国家信息的字典
    geo_file_read = open(geo_file, 'r', encoding='utf-8')
    temp_list = []
    for line in geo_file_read.readlines():
        line = line.strip().split(",")
        if line[4]:  # 当国家代码存在
            # print(line)
            temp_list.append(line[5].strip("\""))
            temp_list.append(line[3].strip("\""))
            code2country[line[4]] = temp_list
            temp_list = []
    # print(code2country)
    external_country_rank_list_save = []  # 存储最终输出的带国家详细的排名表
    temp_list = []
    for item in rank_list:
        try:
            # print(item)
            temp_list.append(item[0])
            temp_list.append(item[1])
            temp_list.extend(code2country[item[0]])
            external_country_rank_list_save.append(temp_list)
            temp_list = []
        except Exception as e:
            # print(e)
            external_country_rank_list_save.append(temp_list)
            temp_list = []
    # print(external_country_rank_list_save)
    # 将信息存储到文件中
    save_path = "..\\000LocalData\\RUNet\\External_BGP_Rel" + country + "-ISP(20200201)_" + org_name + ".csv"
    write_to_csv(external_country_rank_list_save, save_path)

def draw_bar(rank_list, country, orgname):
    """
    根据传入的rank_list信息绘制直方图
    :param rank_list:
    :return:
    """
    x_list = []
    y_list = []

    for item in rank_list:
        x_list.append(item[0])
        y_list.append(item[1])
    # 开始绘图
    fig, ax = plt.subplots(1, 1, figsize=(50, 10))
    title_string = "External BGP Relationships Analysis-ISP:" + country + "( Date:20200201 )"
    ax.set_title(title_string)
    color = ['blue']
    plt.bar(x_list, y_list, width=0.7, color=color)
    for x, y in zip(x_list, y_list):
        plt.text(x, y + 0.05, '%.0f' % y, ha='center', va='bottom', fontsize=11)
    ax.set_xlabel('Country')
    ax.set_ylabel('Relationships Nums')
    # plt.grid(True, linestyle=':', color='r', alpha=0.9)
    # plt.show()
    plt.savefig("..\\000LocalData\\RUNet\\External_BGP_Rel" + country + "-ISP(20200201)" + orgname + ".jpg")

if __name__ == "__main__":
    time_start = time.time()  # 记录启动时间
    country = "CN"
    as_info_file_top = '..\\000LocalData\\as_map\\as_core_map_data_new20200201.csv'
    as_info_file_in = '..\\000LocalData\\as_Gao\\asn_info.txt'
    country_as_info, as2country_dict = gain_as2country(as_info_file_in, country)
    as_list = ['4134', '4809', '4837', '9929', '9808']  # 存储需要统计的AS列表
    for item_as in as_list:
        item_as_direct_name = str(item_as) + "_direct"
        external_country_as_direct = external_as_analysis(country,
                                                          as2country_dict,
                                                          [item_as],
                                                          item_as_direct_name)

        item_as_indirect_name = str(item_as) + "_indirect"
        external_country_as_direct.extend([item_as])
        external_country_as_indirect = external_as_analysis(country,
                                                            as2country_dict,
                                                            external_country_as_direct,
                                                            item_as_indirect_name)

    time_end = time.time()
    print("=>Scripts Finish, Time Consuming:", (time_end - time_start), "S")


alien_0 = {'color': 'green',  'points': 5}
print( alien_0['color'] )
print( alien_0['points'] )
# Значения в словарях могут быть в любой момент добавлены или изменены
alien_0['x_position'] = 0
alien_0['y_position'] = 25
alien_0['y_position'] = 10
print ( alien_0 )

alien_1 = {}
print( type(alien_1) )

del alien_0['color']
print( alien_0 )

# Можно задавать числовой индекс в словаре
alien_0[0] = 'aliens'
print( alien_0[0] )

python = {
  'types':  [
  'string',
  'integer',
  'float',
  'dictionaries',
  'tuples',
  'lists'
  ],
  'paradigms':
  [ 'functions',
  'object-orientation'
  ]
}

# print( python.items() )

# Перебор словаря в цикле for
# for k, v in  alien_0.items():
#   print( 'Key: ' + str( k ) )
#   print ( 'Value: ' + str(v) + '\n\n' )

for key in  set(alien_0.keys()): # то же самое что просто aliens
# перебор ключей работает по умолчанию
  print( '\nKey is "' + str(key) + '"' )

print( alien_0.values() )
print('points' in alien_0.values() )

print( set( alien_0 ) )


class BureauDeChange:
    def exchange_base(rates_data):
        currency_base = rates_data['base']
        print(f"Base currency: {currency_base}")

    def exchange_list(rates_data):
        currency_base = rates_data['base']
        currency_list = rates_data['rates']
        currency_updated = rates_data['date']
        print(f"1 {currency_base} is equivalent to:\n")
        for key, value in currency_list.items():
            print(f"{key}: {value}")
        print(f"\nData last updated: {currency_updated}\n")

    def exchange_convert(rates_data, currency1, amount, currency2):
        currency_list = rates_data['rates']
        value1 = currency_list.get(currency1.strip().upper())
        value2 = currency_list.get(currency2.strip().upper()) / value1
        rate = value2 / value1
        print(f"{amount} {currency1.strip().upper()} is worth {value1 * rate * amount:.2f} {currency2.strip().upper()}.")


#1.
for onehundredfifty in range(151):
    print(onehundredfifty)
#2.
for thousandFives in range(5,1001,5):
    print(thousandFives)
#3.
for hundred in range(1,101):
    if hundred%5==0:
        print("Coding")
    elif hundred%5!=0:
        print("CodingDojo!")
    else:
        print(hundred)
#4.
sum =0
for halfmillion in range(500001):
    if halfmillion%2!=0:
        sum+=halfmillion
print(sum)
#5.
count = 2018
while count > 0:
    print( count)
    count -= 4
#6.
lowNum=2
highNum=9
mult=3
res=0
for i in range(lowNum,highNum):
        if res < highNum:
            res+=mult
            print(res)
        else:
            break


# 给一棵树和一个数组 看这个数组是否是这棵树或者这棵树的子树的后序遍历
from leetcode.tree import TreeNode

class Solution:
    def helper(self, root, arr):
        temp = []

        def afterTraverse(node):
            if not node:
                temp.append(-1)
                return
            afterTraverse(node.left)
            afterTraverse(node.right)
            temp.append(node.val)

        afterTraverse(root)

        t = temp[temp.index(arr[0]):temp.index(arr[-1])+1]
        if t.count(-1) >= len(t)-t.count(-1): #该序列在此树中不是一颗子树
            return False
        t= list(filter(lambda a:a!=-1,t))
        if t:
            if set(arr).issubset(t):
                return True
        return False

s = Solution()
root = TreeNode(1)
root.left = TreeNode(2)
root.right = TreeNode(3)
root.left.left = TreeNode(4)
root.right.left = TreeNode(5)
root.left.left.left = TreeNode(6)
arr = [2,3,1]
print(s.helper(root,arr))


import tkinter as tk
import re
from init import *
import os, sys

FULL_ITEM_LIST = ["Aldur's Advance", "Aldur's Deception", "Aldur's Rhythm", "Aldur's Stony Gaze", "Alma Negra",
                  "Amn Rune", "Andariel's Visage", "Angelic Halo", "Angelic Mantle", "Angelic Sickle",
                  "Angelic Wings", "Annihilus", "Arachnid Mesh", "Arcanna's Deathwand", "Arcanna's Flesh",
                  "Arcanna's Head", "Arcanna's Sign", "Arctic Binding", "Arctic Furs", "Arctic Horn",
                  "Arctic Mitts", "Arioc's Needle", "Arkaine's Valor", "Arm of King Leoric", "Arreat's Face",
                  "Astreon's Iron Ward", "Athena's Wrath", "Atma's Scarab", "Atma's Wail", "Axe of Fechmar",
                  "Azurewrath", "Baezil's Vortex", "Bane Ash", "Baranar's Star", "Bartuc's Cut-Throat",
                  "Ber Rune", "Berserker's Hatchet", "Berserker's Hauberk", "Berserker's Headgear",
                  "Biggin's Bonnet", "Bing Sz Wang", "Black Hades", "Blackbog's Sharp", "Blackhand Key",
                  "Blackhorn's Face", "Blackleach Blade", "Blackoak Shield", "Blacktongue", "Blade of Ali Baba",
                  "Bladebone", "Bladebuckle", "Blastbark", "Blinkbat's Form", "Blood Crescent",
                  "Blood Raven's Charge", "Bloodfist", "Bloodletter", "Bloodmoon", "Bloodrise", "Bloodthief",
                  "Bloodtree Stump", "Boneflame", "Boneflesh", "Bonehew", "Boneshade", "Boneslayer Blade",
                  "Bonesnap", "Brainhew", "Bul-Kathos' Sacred Charge", "Bul-Kathos' Tribal Guardian",
                  "Bul-Kathos' Wedding Band", "Buriza-Do Kyanon", "Butcher's Pupil", "Bverrit Keep",
                  "Carin Shard", "Carrion Wind", "Cathan's Mesh", "Cathan's Rule", "Cathan's Seal",
                  "Cathan's Sigil", "Cathan's Visage", "Cerebus' Bite", "Cham Rune", "Chance Guards",
                  "Chromatic Ire", "Civerb's Cudgel", "Civerb's Icon", "Civerb's Ward", "Cleglaw's Claw",
                  "Cleglaw's Pincers", "Cleglaw's Tooth", "Cliffkiller", "Cloudcrack", "Coif of Glory",
                  "Coldkill", "Coldsteel Eye", "Corpsemourn", "Cow King's Hide", "Cow King's Hooves",
                  "Cow King's Horns", "Crainte Vomir", "Cranebeak", "Credendum", "Crescent Moon", "Crow Caw",
                  "Crown of Ages", "Crown of Thieves", "Crushflange", "Culwen's Point", "Dangoon's Teaching",
                  "Dark Adherent", "Dark Clan Crusher", "Darkforce Spawn", "Darkglow", "Darksight Helm",
                  "Death Cleaver", "Deathbit", "Death's Fathom", "Death's Guard", "Death's Hand", "Death's Touch",
                  "Death's Web", "Deathspade", "Demon Limb", "Demon Machine", "Demonhorn's Edge", "Demon's Arch",
                  "Dimoak's Hew", "Djinn Slayer", "Dol Rune", "Doombringer", "Doomslinger", "Dracul's Grasp",
                  "Dragonscale", "Duriel's Shell", "Duskdeep", "Dwarf Star", "Eaglehorn", "Earth Shifter",
                  "Earthshaker", "El Rune", "Eld Rune", "Endlesshail", "Eschuta's Temper", "Eth Rune",
                  "Ethereal Edge", "Executioner's Justice", "Fal Rune", "Felloak", "Firelizard's Talons",
                  "Flamebellow", "Fleshrender", "Fleshripper", "Frostburn", "Frostwind", "Gargoyle's Bite",
                  "Gerke's Sanctuary", "Gheed's Fortune", "Ghostflame", "Ghoulhide", "Giant Skull", "Gimmershred",
                  "Ginther's Rift", "Gleamscythe", "Gloom's Trap", "Goblin Toe", "Goldskin", "Goldstrike Arch",
                  "Goldwrap", "Gore Rider", "Gorefoot", "Goreshovel", "Gravenspine", "Gravepalm", "Greyform",
                  "Griffon's Eye", "Grim's Burning Dead", "Griswold's Edge", "Griswold's Heart",
                  "Griswold's Honor", "Griswold's Redemption", "Griswold's Valor", "Guardian Angel",
                  "Guardian Naga", "Guillaume's Face", "Gul Rune", "Gull", "Gut Siphon", "Haemosu's Adamant",
                  "Halaberd's Reign", "Hand of Blessed Light", "Harlequin Crest", "Hawkmail",
                  "Head Hunter's Glory", "Headstriker", "Heart Carver", "Heavenly Garb", "Heaven's Light",
                  "Hel Rune", "Hellcast", "Hellclap", "Hellfire Torch", "Hellmouth", "Hellplague", "Hellrack",
                  "Hellslayer", "Herald of Zakarum", "Hexfire", "Highlord's Wrath", "Homunculus", "Hone Sundan",
                  "Horizon's Tornado", "Hotspur", "Howltusk", "Hsarus' Iron Fist", "Hsarus' Iron Heel",
                  "Hsarus' Iron Stay", "Humongous", "Husoldal Evo", "Hwanin's Blessing", "Hwanin's Justice",
                  "Hwanin's Refuge", "Hwanin's Splendor", "Iceblink", "Ichorsting", "Immortal King's Detail",
                  "Immortal King's Forge", "Immortal King's Pillar", "Immortal King's Soul Cage",
                  "Immortal King's Stone Crusher", "Immortal King's Will", "Infernal Cranium", "Infernal Sign",
                  "Infernal Torch", "Infernostride", "Io Rune", "Iratha's Coil", "Iratha's Collar",
                  "Iratha's Cord", "Iratha's Cuff", "Iron Pelt", "Ironstone", "Isenhart's Case",
                  "Isenhart's Horns", "Isenhart's Lightbrand", "Isenhart's Parry", "Islestrike", "Ist Rune",
                  "Ith Rune", "Jade Talon", "Jah Rune", "Jalal's Mane", "Kelpie Snare", "Kinemil's Awl",
                  "Kira's Guardian", "Knell Striker", "Ko Rune", "Kuko Shakaku", "Lacerator", "Lance Guard",
                  "Lance of Yaggai", "Langer Briser", "Lava Gout", "Laying of Hands", "Leadcrow", "Lem Rune",
                  "Lenymo", "Leviathan", "Lidless Wall", "Lightsabre", "Lo Rune", "Lum Rune", "Lycander's Aim",
                  "Lycander's Flank", "Maelstrom", "Magefist", "Magewrath", "Magnus' Skin", "Mal Rune",
                  "Manald Heal", "Mang Song's Lesson", "Mara's Kaleidoscope", "Marrowwalk", "M'avina's Caster",
                  "M'avina's Embrace", "M'avina's Icy Clutch", "M'avina's Tenet", "M'avina's True Sight",
                  "Medusa's Gaze", "Messerschmidt's Reaver", "Metalgrid", "Milabrega's Diadem", "Milabrega's Orb",
                  "Milabrega's Robe", "Milabrega's Rod", "Moonfall", "Moser's Blessed Circle", "Nagelring",
                  "Naj's Circlet", "Naj's Light Plate", "Naj's Puzzler", "Natalya's Mark", "Natalya's Shadow",
                  "Natalya's Soul", "Natalya's Totem", "Nature's Peace", "Nef Rune", "Nightsmoke",
                  "Nightwing's Veil", "Nokozan Relic", "Nord's Tenderizer", "Nosferatu's Coil", "Ohm Rune",
                  "Ondal's Almighty", "Ondal's Wisdom", "Orb of Corruption", "Ormus' Robes", "Ort Rune",
                  "Peasant Crown", "Pelta Lunata", "Pierre Tombale Couant", "Plague Bearer", "Pluckeye",
                  "Pompeii's Wrath", "Pul Rune", "Pus Spitter", "Que-Hegan's Wisdom", "Radament's Sphere",
                  "Rainbow Facet (Cold Die)", "Rainbow Facet (Cold Level Up)", "Rainbow Facet (Fire Die)",
                  "Rainbow Facet (Fire Level Up)", "Rainbow Facet (Light Die)", "Rainbow Facet (Light Level Up)",
                  "Rainbow Facet (Poison Die)", "Rainbow Facet (Poison Level Up)", "Rakescar", "Ral Rune", "Rattlecage",
                  "Raven Claw", "Raven Frost", "Ravenlore", "Razor's Edge", "Razorswitch", "Razortail", "Razortine",
                  "Ribcracker", "Riphook", "Ripsaw", "Rite of Passage", "Rixot's Keen", "Rockfleece", "Rockstopper",
                  "Rogue's Bow", "Rune Master", "Rusthandle", "Sander's Paragon", "Sander's Riprap",
                  "Sander's Superstition", "Sander's Taboo", "Sandstorm Trek", "Saracen's Chance",
                  "Sazabi's Cobalt Redeemer", "Sazabi's Ghost Liberator", "Sazabi's Mental Sheath", "Schaefer's Hammer",
                  "Seraph's Hymn", "Serpent Lord", "Shadow Dancer", "Shadow Killer", "Shadowfang", "Shael Rune",
                  "Shaftstop", "Sigon's Gage", "Sigon's Guard", "Sigon's Sabot", "Sigon's Shelter", "Sigon's Visor",
                  "Sigon's Wrap", "Silks of the Victor", "Silkweave", "Skewer of Krintiz",
                  "Skin of the Flayed One", "Skin of the Vipermagi", "Skull Collector", "Skull Splitter",
                  "Skullder's Ire", "Skystrike", "Snakecord", "Snowclash", "Sol Rune", "Soul Drainer",
                  "Soul Harvest", "Soulfeast Tine", "Soulflay", "Sparking Mail", "Spectral Shard", "Spellsteel",
                  "Spike Thorn", "Spineripper", "Spire of Honor", "Spire of Lazarus", "Spirit Forge",
                  "Spirit Keeper", "Spirit Ward", "Stealskull", "Steel Carapace", "Steel Pillar", "Steel Shade",
                  "Steelclash", "Steeldriver", "Steelgoad", "Steelrend", "Stone Crusher", "Stone of Jordan",
                  "Stoneraven", "Stormchaser", "Stormeye", "Stormguild", "Stormlash", "Stormrider",
                  "Stormshield", "Stormspike", "Stormspire", "Stormstrike", "Stoutnail", "String of Ears",
                  "Suicide Branch", "Sur Rune", "Sureshrill Frost", "Swordback Hold", "Swordguard",
                  "Taebaek's Glory", "Tal Rasha's Adjudication", "Tal Rasha's Fine Spun Cloth",
                  "Tal Rasha's Guardianship", "Tal Rasha's Horadric Crest", "Tal Rasha's Lidless Eye", "Tal Rune",
                  "Tancred's Crowbill", "Tancred's Hobnails", "Tancred's Skull", "Tancred's Spine",
                  "Tancred's Weird", "Tarnhelm", "Tearhaunch", "Telling of Beads", "Templar's Might",
                  "The Atlantean", "The Battlebranch", "The Cat's Eye", "The Centurion", "The Chieftain",
                  "The Cranium Basher", "The Diggler", "The Dragon Chang", "The Eye of Etlich",
                  "The Face of Horror", "The Fetid Sprinkler", "The Gavel of Pain", "The General's Tan Do Li Ga",
                  "The Gladiator's Bane", "The Gnasher", "The Grandfather", "The Grim Reaper",
                  "The Hand of Broc", "The Impaler", "The Iron Jang Bong", "The Jade Tan Do",
                  "The Mahim-Oak Curio", "The Meat Scraper", "The Minotaur", "The Oculus", "The Patriarch",
                  "The Reaper's Toll", "The Redeemer", "The Rising Sun", "The Salamander", "The Scalper",
                  "The Spirit Shroud", "The Tannr Gorerod", "The Vile Husk", "The Ward", "Thul Rune",
                  "Thundergod's Vigor", "Thunderstroke", "Tiamat's Rebuke", "Tir Rune", "Titan's Revenge",
                  "Todesfaelle Flamme", "Tomb Reaver", "Toothrow", "Torch of Iro", "Trang-Oul's Claws",
                  "Trang-Oul's Girth", "Trang-Oul's Guise", "Trang-Oul's Scales", "Trang-Oul's Wing",
                  "Treads of Cthon", "Twitchthroe", "Tyrael's Might", "Um Rune", "Umbral Disk",
                  "Ume's Lament", "Undead Crown", "Valkyrie Wing", "Vampire Gaze", "Veil of Steel",
                  "Venom Grip", "Venom Ward", "Verdungo's Hearty Cord", "Vex Rune", "Vidala's Ambush",
                  "Vidala's Barb", "Vidala's Fetlock", "Vidala's Snare", "Viperfork", "Visceratuant",
                  "Wall of the Eyeless", "War Traveler", "Warlord's Trust", "Warpspear", "Warshrike", "Waterwalk",
                  "Whitstan's Guard", "Widowmaker", "Wilhelm's Pride", "Windforce", "Windhammer",
                  "Wisp Projector", "Witchwild String", "Witherstring", "Wizardspike", "Wizendraw", "Woestave",
                  "Wolfhowl", "Wormskull", "Wraith Flight", "Zakarum's Hand", "Zod Rune",
                  "Key of Destruction", "Key of Hate", "Key of Terror"]

ITEM_ALIASES = {"Aldur's Boots": "Aldur's Advance",
                "Aldur's Weapon": "Aldur's Rhythm",
                "Aldur's Armor": "Aldur's Deception",
                "Aldur's Helm": "Aldur's Stony Gaze",
                "Andy's Face": "Andariel's Visage",
                'Angelic Amu': 'Angelic Wings',
                'Angelic Ring': 'Angelic Halo',
                'Angelic Armor': 'Angelic Mantle',
                'Angelic Weapon': 'Angelic Sickle',
                'BK ring': "Bul-Kathos' Wedding Band",
                'CoA': 'Crown of Ages',
                'DF': "Death's Fathom",
                'Dkey': 'Key of Destruction',
                'G Angel': 'Guardian Angel',
                'GAngel': 'Guardian Angel',
                'GF': 'The Grandfather',
                "Griswold's Armor": "Griswold's Heart",
                "Griswold's Helm": "Griswold's Valor",
                "Griswold's Shield": "Griswold's Honor",
                "Griswold's Weapon": "Griswold's Redemption",
                "Griswold's Caduceus": "Griswold's Redemption",
                "Gris Caduceus": "Griswold's Redemption",
                "Gris Armor": "Griswold's Heart",
                "Gris Helm": "Griswold's Valor",
                "Gris Shield": "Griswold's Honor",
                "Gris Weapon": "Griswold's Redemption",
                'Gull Dagger': 'Gull',
                'HoZ': 'Herald of Zakarum',
                'Hkey': 'Key of Hate',
                'IK Armor': "Immortal King's Soul Cage",
                'IK Weapon': "Immortal King's Stone Crusher",
                'IK Helm': "Immortal King's Will",
                'IK Belt': "Immortal King's Detail",
                'IK Boots': "Immortal King's Pillar",
                'IK Gloves': "Immortal King's Forge",
                "Immortal King's Armor": "Immortal King's Soul Cage",
                "Immortal King's Weapon": "Immortal King's Stone Crusher",
                "Immortal King's Helm": "Immortal King's Will",
                "Immortal King's Belt": "Immortal King's Detail",
                "Immortal King's Boots": "Immortal King's Pillar",
                "Immortal King's Gloves": "Immortal King's Forge",
                'LoH': 'Laying of Hands',
                "M'avina's Bow": "M'avina's Caster",
                "M'avina's Armor": "M'avina's Embrace",
                "M'avina's Gloves": "M'avina's Icy Clutch",
                "M'avina's Belt": "M'avina's Tenet",
                "M'avina's Diadem": "M'avina's True Sight",
                "M'avina's Helm": "M'avina's True Sight",
                'Mahim Oak': 'The Mahim-Oak Curio',
                'Nagel Ring': 'Nagelring',
                'OOC': 'Orb of Corruption',
                'Occu': 'The Oculus',
                'Occy': 'The Oculus',
                "Que Hegan's Wisdom": "Que-Hegan's Wisdom",
                'RBF cold die': 'Rainbow Facet (Cold Die)',
                'RBF cold level up': 'Rainbow Facet (Cold Level Up)',
                'RBF fire die': 'Rainbow Facet (Fire Die)',
                'RBF fire level up': 'Rainbow Facet (Fire Level Up)',
                'RBF light die': 'Rainbow Facet (Light Die)',
                'RBF light level up': 'Rainbow Facet (Light Level Up)',
                'RBF poison die': 'Rainbow Facet (Poison Die)',
                'RBF poison level up': 'Rainbow Facet (Poison Level Up)',
                'Rainbow Facet cold die': 'Rainbow Facet (Cold Die)',
                'Rainbow Facet cold level up': 'Rainbow Facet (Cold Level Up)',
                'Rainbow Facet fire die': 'Rainbow Facet (Fire Die)',
                'Rainbow Facet fire level up': 'Rainbow Facet (Fire Level Up)',
                'Rainbow Facet light die': 'Rainbow Facet (Light Die)',
                'Rainbow Facet light level up': 'Rainbow Facet (Light Level Up)',
                'Rainbow Facet poison die': 'Rainbow Facet (Poison Die)',
                'Rainbow Facet poison level up': 'Rainbow Facet (Poison Level Up)',
                'Shako': 'Harlequin Crest',
                'SoE': 'String of Ears',
                'SoJ': 'Stone of Jordan',
                'SS': 'Stormshield',
                "Sigon's Armor": "Sigon's Shelter",
                "Sigon's Belt": "Sigon's Wrap",
                "Sigon's Boots": "Sigon's Sabot",
                "Sigon's Gloves": "Sigon's Gage",
                "Sigon's Helm": "Sigon's Visor",
                "Sigon's Shield": "Sigon's Guard",
                'Tkey': 'Key of Terror',
                'TGS': "Thundergod's Vigor",
                'TGods': "Thundergod's Vigor",
                'Tal Amu': "Tal Rasha's Adjudication",
                'Tal Armor': "Tal Rasha's Guardianship",
                'Tal Belt': "Tal Rasha's Fine Spun Cloth",
                'Tal Helm': "Tal Rasha's Horadric Crest",
                'Tal Orb': "Tal Rasha's Lidless Eye",
                'Tal Weapon': "Tal Rasha's Lidless Eye",
                "Tancred's Amu": "Tancred's Weird",
                'Trang Armor': "Trang-Oul's Scales",
                'Trang Belt': "Trang-Oul's Girth",
                'Trang Gloves': "Trang-Oul's Claws",
                'Trang Helm': "Trang-Oul's Guise",
                'Trang Shield': "Trang-Oul's Wing",
                'Vgaze': 'Vampire Gaze',
                'WForce': 'Windforce',
                'WTrav': 'War Traveler',
                'WWs': 'Waterwalk',
                'Wizzy': 'Wizardspike'}

class AutocompleteEntry:
    def __init__(self, master, width, textvariable, enable=True):
        self.chosen = None
        self.enable = enable
        self.master = master
        self.width = width
        self.var = textvariable
        self.entry = tk.Entry(master, textvariable=self.var)
        self.entry.pack(fill=tk.X, padx=4)
        self.entry.focus()

        self.var.trace_add('write', lambda name, index, mode: self.changed(name, index, mode))
        self.entry.bind("<Up>", self.move_up)
        self.entry.bind("<Down>", self.move_down)
        self.entry.bind("<Tab>", self.selection)

        self.listboxUp = False

    def changed(self, name=None, index=None, mode=None):
        var = self.var.get()
        if var == '':
            if self.listboxUp:
                self.listbox.destroy()
                self.listboxUp = False
        else:
            if self.enable:
                words = self.comparison(var)
                if var.lower().startswith('eth '):
                    words.extend(self.comparison(var[4:], eth=True))
            else:
                words = []

            if words:
                if self.listboxUp:
                    self.listbox.destroy()
                self.listbox = tk.Listbox(self.master, width=self.width, height=min(len(words), 6))
                self.listbox.bind("<Double-Button-1>", self.selection)
                self.listbox.bind("<Tab>", self.selection)
                self.listbox.place(relx=0, rely=0.3)
                self.listbox.tkraise()
                self.listboxUp = True

                self.listbox.delete(0, tk.END)
                for w in words:
                    self.listbox.insert(tk.END, w)
            else:
                if self.listboxUp:
                    self.listbox.destroy()
                    self.listboxUp = False

    def selection(self, event=None):
        if self.listboxUp:  # and self.listbox.curselection():
            self.chosen = self.listbox.get(tk.ACTIVE)
            self.var.set(self.chosen + ' ')
            self.listbox.destroy()
            self.listboxUp = False
            self.entry.icursor(tk.END)

    def move_up(self, event):
        if self.listboxUp:
            if self.listbox.curselection() == ():
                index = tk.END
                hl_idx = tk.END
            else:
                index = self.listbox.curselection()[0]
                hl_idx = str(int(index) - 1)

            if index != '0':
                self.listbox.selection_clear(first=index)

                self.listbox.see(hl_idx)  # Scroll!
                self.listbox.selection_set(first=hl_idx)
                self.listbox.activate(hl_idx)

    def move_down(self, event):
        if self.listboxUp:
            if self.listbox.curselection() == ():
                index = '0'
                hl_idx = '0'
            else:
                index = self.listbox.curselection()[0]
                hl_idx = str(int(index) + 1)

            if index != tk.END:
                self.listbox.selection_clear(first=index)

                self.listbox.see(hl_idx)  # Scroll!
                self.listbox.selection_set(first=hl_idx)
                self.listbox.activate(hl_idx)

    @staticmethod
    def comparison(var, eth=False):
        out = set()
        # regex to append a [']? after all letters, which is an optional argument for adding a hyphen
        # this means that for example typing in "mavinas" and "m'avina's" will yield the same results
        hyphen_escape = re.sub('([^a-zA-Z]*)', "\\1[']?", re.escape(var))
        # encapsulating with \b ensures that searches are done only at the start of each word
        # ".*" allows anything to follow after the already typed letters
        pattern = re.compile(r"\b" + hyphen_escape + r".*\b", flags=re.IGNORECASE)

        for w in FULL_ITEM_LIST + list(ITEM_ALIASES.keys()):
            if re.search(pattern, w):
                # Append true entry from the alias list - if none are found, add the match from original list
                i_name = ITEM_ALIASES.get(w, w)
                if eth:
                    if 'Rune' not in i_name and 'Orb of Corruption' not in i_name:
                        out.add('Eth ' + i_name)
                else:
                    out.add(i_name)
        return sorted(out)

class ACMbox(object):
    def __init__(self, title, enable=True):
        self.root = tk.Toplevel()
        self.root.geometry(
            '200x145+%s+%s' % (self.root.winfo_screenwidth() // 2 - 100, self.root.winfo_screenheight() // 2 - 72))
        self.root.update_idletasks()
        self.root.focus_set()
        self.root.iconbitmap(os.path.join(getattr(sys, '_MEIPASS', os.path.abspath('.')), media_path + 'icon.ico'))
        self.root.title(title)
        self.root.wm_attributes("-topmost", True)
        self.root.resizable(False, False)

        frm_1 = tk.Frame(self.root)
        frm_1.pack(ipadx=4, ipady=2, fill=tk.BOTH, expand=tk.Y)

        tk.Label(frm_1, text='Input your drop...').pack()

        tw = tk.StringVar()
        self.entry = AutocompleteEntry(frm_1, width=32, textvariable=tw, enable=enable)

        frm_2 = tk.Frame(frm_1)
        frm_2.pack(padx=4, pady=4)

        tk.Button(frm_2, width=8, text='OK', command=self.b1_action).pack(side=tk.LEFT)
        tk.Button(frm_2, width=8, text='Cancel', command=self.close_mod).pack(side=tk.LEFT)

        self.root.bind('<KeyPress-Return>', func=self.b1_action)
        self.root.bind('<KeyPress-Escape>', func=self.close_mod)

        # call self.close_mod when the close button is pressed
        self.root.protocol("WM_DELETE_WINDOW", self.close_mod)

        # a trick to activate the window (on windows 7)
        self.root.deiconify()

    def b1_action(self, event=None):
        if self.entry.listboxUp:
            self.entry.selection(event)
        else:
            item_name = self.entry.chosen
            if item_name is not None and item_name.startswith('Eth ') and item_name != 'Eth Rune':
                item_name = item_name[4:]
            user_input = self.entry.var.get().strip()
            extra_input = user_input.replace(item_name, '').strip().replace('  ', ' ') if item_name is not None else ''
            self.returning = {'item_name': item_name, 'input': user_input, 'extra': extra_input}
            self.root.quit()

    def close_mod(self, event=None):
        if self.entry.listboxUp:
            self.entry.listbox.destroy()
            self.entry.listboxUp = False
        else:
            self.returning = None
            self.root.quit()

def acbox(title='Drop', enable=True):
    msgbox = ACMbox(title, enable=enable)
    msgbox.root.mainloop()

    # the function pauses here until the mainloop is quit
    msgbox.root.destroy()
    return msgbox.returning

if __name__ == '__main__':
    print(acbox(enable=True))


# https://leetcode.com/problems/string-compression/

# Modify in-place:

class Solution:
    def compress(self, chars):
        read = 0
        write = 0
        cur_count = 1
        for read in range(len(chars)):
            if read + 1 == len(chars) or chars[read + 1] != chars[read]:
                chars[write] = chars[read]
                if cur_count > 1:
                    for i in range(len(str(cur_count))):
                        chars[write + 1 + i] = str(cur_count)[i]
                    write = write + 1 + len(str(cur_count))
                else:
                    write = write + 1
                cur_count = 1
            else:
                cur_count += 1
        return (write)

# Use extra space:

class Solution:
    def compress(self, chars):
        """
        :type chars: List[str]
        :rtype: int
        """
        res = []
        cur_count = 1
        for i in range(len(chars)):
            if i + 1 == len(chars) or chars[i + 1] != chars[i]:
                res.append(chars[i])
                if cur_count > 1:
                    for x in str(cur_count):
                        res.append(x)
                cur_count = 1
            else:
                cur_count += 1

        for i in range(len(res)):
            chars[i] = res[i]

        return len(res)


import Models
import keras
import numpy as np
import pandas as pd

# df = pd.read_csv("/home/amithbn/Desktop/TABL/fi_2010_dataset/FI2010_train.csv")
# arr = df["148"].values

# n = len(df)

# n /= 10
# n = int(n)
# arr = arr[0:n]

# up = pd.DataFrame(data={"UP":np.where(arr == 1, 1, 0)})
# dn = pd.DataFrame(data={"DN":np.where(arr == 3, 1, 0)})
# #eq = pd.DataFrame(data={"EQ":np.where(arr == 2, 1, 0)})

# df = df.iloc[:, :40]
# lst = np.array_split(df,n)
# lst = np.array(lst)

# lbls = up.join(dn)
# #lbls = lbls.join(eq)

# 1 hidden layer network with input: 40x10, hidden 120x5, output 3x1
# template = [[10,40], [120,5], [2,1]]
template = [[40,10], [120,5], [3,1]]

x = np.random.rand(1000,40,10)
y = keras.utils.to_categorical(np.random.randint(0,3,(1000,)),3)

# random data
#y = keras.utils.to_categorical(np.random.randint(0,3,(n,)),3) # - low, high,

# get Bilinear model
projection_regularizer = None
projection_constraint = keras.constraints.max_norm(3.0,axis=0)
attention_regularizer = None
attention_constraint = keras.constraints.max_norm(5.0, axis=1)
dropout = 0.1

model = Models.TABL(template, dropout, projection_regularizer, projection_constraint,
                    attention_regularizer, attention_constraint)
model.summary()

# create class weight
class_weight = {0 : 1e6/300.0,
                1 : 1e6/400.0,
                2 : 1e6/300.0}

# training
model.fit(x,y, batch_size=256, epochs=100, class_weight=class_weight)

# model.fit(lst,lbls, batch_size=256, epochs=10, class_weight=class_weight)
# model.save('tabl_model.h5')


import pickle
# import apriori
#
# dataset = apriori.load_dataset('feedback_csr.pkl')
#
# L, support_data = apriori.apriori(dataset, minsupport=0.02)
#
# op = open('tmp.txt', 'wb')
# op.write(str(apriori.generateRules(L, support_data)))

import scipy.sparse as sp
import pprint

clusters ={}
data = pickle.load(open('feedback_csr.pkl', 'rb'))
user_map = {v: k for k, v in pickle.load(open('user_map.pkl', 'rb')).iteritems()}

comps = sp.cs_graph_components(data)[1]

for i in range(len(comps)):
    if comps[i] in clusters:
        clusters[comps[i]].add(user_map[i])
    else:
        clusters[comps[i]] = {user_map[i]}
clusters.pop(-2)
pprint.pprint(clusters)
open('graph_networks.txt', 'wb').write(pprint.pformat(clusters))


from MapParser import MapParser as parser

class MapObjectUtils:
    @staticmethod
    def get_adm_level(word_array, word_array_from_db, comment):
        result_array = dict()
        print("Пробуем найти ссылки слов на " + comment + ". Точный поиск")
        if len(word_array) > 1:
            print("у нас больше чем одно слово попробуем найти пересечение ссылок этих слов на объекты")
            word_id = -1
            object_array_by_word_id = dict()
            all_object_array = dict()
            for word in word_array:
                word_id += 1
                true_search = word in word_array_from_db.keys()
                set_id = set()
                if true_search:
                    row_list = word_array_from_db.get(word)
                    for row in row_list:
                        all_object_array.update({row[2]: (row[3], row[4], row[5], row[6], row[2], row[7])})
                        set_id.add(row[2])
                object_array_by_word_id.update({word_id: set_id})
            set_array = []
            result_set = object_array_by_word_id.get(word_id)
            while word_id >= 1:
                key = word_id - 1
                result_set = result_set.intersection(object_array_by_word_id.get(key))
                if len(result_set) > 0:
                    set_array.append(result_set)
                word_id -= 1
                result_set = object_array_by_word_id.get(word_id)
            if len(set_array):
                for set_el in set_array:
                    for set_value in set_el:
                        print("Нашли пересечения " + str(all_object_array.get(set_value)))
                        result_array.update({set_value: all_object_array.get(set_value)})
            else:
                print("HELL не нашли пересечения")
                result_array = all_object_array
        else:
            for word in word_array:
                true_search = word in word_array_from_db.keys()
                if true_search:
                    row_list = word_array_from_db.get(word)
                    for row in row_list:
                        result_array.update({row[2]: (row[3], row[4], row[5], row[6], row[2], row[7])})
        if len(result_array) == 0:
            print("{0} не определили".format(comment))
        #print(result_array)
        return result_array

    @staticmethod
    def get_adm_level_similar(table_in, comment):
        table_out = dict()
        for row in table_in:
            table_out.update({row[0]: (row[1], row[3], row[2], row[4], row[0], row[7])})
        print("Получено {0} нечеткий поиск: {1}".format(comment, len(table_out)))
        print(table_out)
        return table_out

    @staticmethod
    def lev(s, t):
        if s == t:
            return 0
        elif len(s) == 0:
            return len(t)
        elif len(t) == 0:
            return len(s)
        v0 = [None] * (len(t) + 1)
        v1 = [None] * (len(t) + 1)
        for i in range(len(v0)):
            v0[i] = i
        for i in range(len(s)):
            v1[0] = i + 1
            for j in range(len(t)):
                cost = 0 if s[i] == t[j] else 1
                v1[j + 1] = min(v1[j] + 1, v0[j + 1] + 1, v0[j] + cost)
            for j in range(len(v0)):
                v0[j] = v1[j]
        return v1[len(t)]

    @staticmethod
    def get_adm_level_lev(word_array, word_array_from_db, comment):
        result_array = dict()
        for word in word_array:
            distance = int(len(word) / 4) + 1
            for word_from_db in word_array_from_db.keys():
                if MapObjectUtils.lev(word, word_from_db) <= distance:
                    row_list = word_array_from_db.get(word_from_db)
                    for row in row_list:
                        result_array.update({row[2]: (row[3], row[4], row[5], row[6], row[2], row[7])})
        if len(result_array) == 0:
            print("{0} не определили".format(comment))
        return result_array

    @staticmethod
    def get_street_from_index_array(street_id_array, street):
        tmp_street = None
        if len(street) == 0 or len(street_id_array) == 0:
            return tmp_street
        for el_array in street_id_array:
            if el_array in street.keys():
                print(el_array)
                tmp_street = street.get(el_array)
        return tmp_street

    @staticmethod
    def get_street_type(new_address_str, type_of_adm_level):
        key_word = set()
        separators = set()
        separators.add(" ")
        word_array = parser.get_word_array(new_address_str, key_word, separators, False)
        result_type = None
        for word in word_array:
            for type_key, type_value in type_of_adm_level.items():
                if word in type_value:
                    result_type = type_key
        return result_type

    @staticmethod
    def filter_street_by_type(streets, street_type):
        filtered_streets = dict()
        for street_key, street_value in streets.items():
            if street_value[3] == street_type:
                filtered_streets.update({street_key: street_value})

        return filtered_streets


import numpy as np
import pandas as pd
import sys
from sklearn.model_selection import train_test_split
from sklearn import preprocessing, linear_model
df1 = pd.read_csv(sys.argv[1],encoding = "ISO-8859-1")
df2 = pd.read_csv(sys.argv[2],encoding = "ISO-8859-1")
df3 = pd.read_csv(sys.argv[3],encoding = "ISO-8859-1")
df4 = pd.read_csv(sys.argv[4],encoding = "ISO-8859-1")
df5 = pd.read_csv(sys.argv[5],encoding = "ISO-8859-1")

#edu_num
edu_num = np.array(df4['education_num'])
edu_num = edu_num.reshape(32561,1)
edu_num_test = np.array(df5['education_num'])
edu_num_test = edu_num_test.reshape(16281,1)

df1 = df1.drop(df1.columns[[i for i in range(-42,-4)]+[i for i in range(15,31)]+[-3,-2,10,45,55]+[-47,-46,-45,-44]], axis=1)
df3 = df3.drop(df3.columns[[i for i in range(-42,-4)]+[i for i in range(15,31)]+[-3,-2,10,45,55]+[-47,-46,-45,-44]], axis=1)

# some feature to x**40
X = np.array(df1,dtype = float)
X = np.concatenate((edu_num,X),axis = 1)
front = X[:,:3]
front = np.concatenate( (front,X[:,4:7]),axis = 1)
for i in range(2,41):
	X = np.concatenate((X,front**i),axis = 1)

#normalize
mean = np.mean(X, axis=0)
std  = np.std(X, axis=0)
X = (X - mean)/std

Y = np.array(df2,dtype = float)
Y = np.squeeze(Y)

# X_train, X_train_test, y_train, y_test = train_test_split(X, Y, test_size=0.25, random_state=int(sys.argv[1]))

X_test = np.array(df3,dtype = float)
X_test = np.concatenate((edu_num_test,X_test),axis =1)
front = X_test[:,:3]
front = np.concatenate( (front,X_test[:,4:7]),axis = 1)
for i in range(2,41):
	X_test = np.concatenate((X_test,front**i),axis = 1)
X_test = (X_test - mean)/std

logistic_regr = linear_model.LogisticRegression(C=10, penalty='l2')
logistic_regr.fit(X,Y)
# logistic_regr.fit(X_train,y_train)

ans = logistic_regr.predict(X_test)
ans = np.array(ans,dtype = int)
ans_df = pd.DataFrame({'id':range(1,len(ans)+1),'label':ans})
ans_df.to_csv(sys.argv[6],index = False)


import boto3
import os

ACCESS_KEY = '*'
SECRET_ACCESS_KEY = '*'

client = boto3.client(
    's3',
    aws_access_key_id=ACCESS_KEY,
    aws_secret_access_key=SECRET_ACCESS_KEY
)

print(vars(client))

upload_file_bucket = 'upload-file-bucket-jpd'
for file in os.listdir():
    if '.py' in file:
        upload_file_key = 'fold1/' + str(file)
        client.upload_file(file, upload_file_bucket, upload_file_key)


# Define bodies that are propagated.
bodies_to_propagate = ["Vehicle"]

# Define central bodies.
central_bodies = ["Earth"]

# Define settings for propagator
termination_condition = propagation_setup.propagator.time_termination(
    simulation_end_epoch )
propagator_settings = propagation_setup.propagator.translational(
    central_bodies,
    acceleration_models,
    bodies_to_propagate,
    initial_state,
    termination_condition )


from django.shortcuts import redirect, render, get_object_or_404
from django.contrib.auth.decorators import login_required
from django.utils import timezone
from .models import Recipe, Comment
from .forms import RecipeForm, CommentForm

def index(request):
	return render(request, 'blog/index.html',{})

def recipes(request):
	recipes = Recipe.objects.filter(pub_date__lte=timezone.now()).order_by('-pub_date')
	return render(request, 'blog/recipes.html',{'recipes': recipes})

def recipe(request, pk):
	recipe = get_object_or_404(Recipe, pk=pk)
	return render(request, 'blog/recipe.html',{'recipe': recipe})

def add_comment(request, pk):
	recipe = get_object_or_404(Recipe, pk=pk)
	if request.method == "POST":
		form = CommentForm(request.POST)
		if form.is_valid():
			comment = form.save(commit=False)
			comment.recipe = recipe
			comment.save()
			return redirect('recipe', pk=recipe.pk)
	else:
		form = CommentForm()
	return render(request, 'blog/add_comment.html', {'form': form})

@login_required
def comment_approve(request, pk):
    comment = get_object_or_404(Comment, pk=pk)
    comment.approve()
    return redirect('blog.views.recipe', pk=comment.recipe.pk)

@login_required
def comment_remove(request, pk):
    comment = get_object_or_404(Comment, pk=pk)
    recipe_pk = comment.recipe.pk
    comment.delete()
    return redirect('blog.views.recipe', pk=recipe_pk)

@login_required
def add_recipe(request):
	if request.method == "POST":
		form = RecipeForm(request.POST, request.FILES)
		if form.is_valid():
			recipe = form.save(commit=False)
			recipe.author = request.user
			recipe.save()
			return redirect('recipe', pk=recipe.pk)
	else:
		form = RecipeForm()
	return render(request, 'blog/add_recipe.html', {'form': form})

@login_required
def edit_recipe(request, pk):
	recipe = get_object_or_404(Recipe, pk=pk)
	if request.method == "POST":
		form = RecipeForm(request.POST, request.FILES, instance=recipe)
		if form.is_valid():
			if not recipe.thumbnail:
				recipe.create_thumbnail()
			recipe = form.save(commit=False)
			recipe.author = request.user
			recipe.save()
			return redirect('recipe', pk=recipe.pk)
	else:
		form = RecipeForm(instance=recipe)
	return render(request, 'blog/edit_recipe.html', {'form': form})

@login_required
def drafts(request):
    recipes = Recipe.objects.filter(pub_date__isnull=True).order_by('created_date')
    return render(request, 'blog/drafts.html', {'recipes': recipes})

@login_required
def recipe_publish(request, pk):
    recipe = get_object_or_404(Recipe, pk=pk)
    recipe.publish()
    return redirect('blog.views.recipe', pk=pk)

@login_required
def recipe_delete(request, pk):
    recipe = get_object_or_404(Recipe, pk=pk)
    recipe.delete()
    return redirect('blog.views.recipes')


import nltk
nltk.download('stopwords')
nltk.download('punkt')
nltk.download('wordnet')
from nltk.corpus import stopwords
from nltk import word_tokenize
import string
from nltk.stem.wordnet import WordNetLemmatizer

lemmatizer = WordNetLemmatizer()
punct = set(string.punctuation)
stop_word_list = set(stopwords.words('english'))

def nlp_pipeline(text_list):
    '''
    this pipeline aims to do a series of preprocessing steps:
    tokenizing, punctuations and numbers removal, lemmatization
    :param:text_list
    :type: a list of str
    :return: a list of preprocessed str
    '''
    string_list = []
    for text in text_list:
        text = text.lower()
        text = word_tokenize(text)
        text = [token for token in text if token not in string.punctuation and token.isalpha()]
        text = [token for token in text if token not in stop_word_list]
        text = [lemmatizer.lemmatize(token) for token in text]
        wordstring = ' '.join(text)
        string_list.append(wordstring)
    return string_list


""" Generating Database and Graphs for Test set """

import os
import cv2
import pickle
import threading
import numpy as np
import matplotlib.pyplot as plt
# User defined Modules
from createDB import createDB
from plots import plot_matches, plot_EER
from classify import classify_matched, thread_helper

############################### Helper Functions ##############################
def display_images(image):
    cv2.imshow('Showing image', image)
    cv2.waitKey()

def storeData(dat, filename):
    dbfile = open(filename, 'wb')
    pickle.dump(dat, dbfile)
    dbfile.close()

def loadData(filename):
    dbfile = open(filename, 'rb')
    return pickle.load(dbfile)
###############################################################################

# Load/Save DB
DB = createDB('../TestSet/Reference/')
storeData(DB, '../Pickles/database_test.pkl')
DB = loadData('../Pickles/database_test.pkl')
threshold = 0.11

# Matched Points Percentage for each type of signature
print("Questioned Signatures")
match = thread_helper('../TestSet/Questioned/', DB, threshold)
genuine_match = match[[48, 51, 65]]
disguise_match = match[[5, 14, 27, 28, 33, 86, 89]]
simulated_match = match[[0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 17,
18, 19, 20, 21, 22, 23, 24, 25, 26, 29, 30, 31, 32, 34, 35, 36, 37, 38, 39, 40,
41, 42, 43, 44, 45, 46, 47, 49, 50, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62,
63, 64, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83,
84, 85, 87, 88, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99]]

# Save/Load percentages (for convenience)
storeData([genuine_match, disguise_match, simulated_match], '../Pickles/vals_test.pkl')
genuine_match, disguise_match, simulated_match = loadData('../Pickles/vals_test.pkl')
plot_matches(genuine_match, disguise_match, simulated_match, '_test')

# Calculate FAR and FRR
far = []
frr = []
theta_range = np.arange(0.05, 0.25, 0.005)
for theta in theta_range:
    ind1 = np.where(genuine_match < theta)
    ind2 = np.where(disguise_match < theta)
    ind3 = np.where(simulated_match > theta)
    frr.append((ind1[0].shape[0] + ind2[0].shape[0]) / (genuine_match.shape[0] + disguise_match.shape[0]) * 100)
    far.append(ind3[0].shape[0] / simulated_match.shape[0] * 100)
plot_EER(theta_range, far, frr, threshold)

# Compute Confidence Matrix
eer = 0.224
ind = np.where(genuine_match < eer)
ind1 = np.where(disguise_match < eer)
ind2 = np.where(simulated_match > eer)
print('\t\tGenuine\tFake')
print('Genuine\t\t' + str(genuine_match.shape[0] - ind[0].shape[0]) + '\t' + str(ind[0].shape[0]))
print('Disguise\t' + str(disguise_match.shape[0] - ind1[0].shape[0]) + '\t' + str(ind1[0].shape[0]))
print('Simulated\t' + str(ind2[0].shape[0]) + '\t' + str(simulated_match.shape[0] - ind2[0].shape[0]))


from SPARQLWrapper import SPARQLWrapper, JSON
import requests
import json

class DomainGetter():

    def query_wikidata(self, entity, limit_articles_number=None):
        articles_list = []
        sparql = SPARQLWrapper("http://query.wikidata.org/sparql")
        query = """
            SELECT ?item ?domain ?article
            WHERE
            {
              wd:%s wdt:P31 ?domain .
              ?item wdt:P31 ?domain .
              ?article schema:about ?item .
              ?article schema:isPartOf <https://en.wikipedia.org/> .
            }
        """ % (entity)
        if limit_articles_number:
            query += " LIMIT " + str(limit_articles_number)
        sparql.setQuery(query)
        sparql.setReturnFormat(JSON)
        results = sparql.query().convert()

        for result in results["results"]["bindings"]:
            articles_list.append(result["article"]["value"])
        return list(set(articles_list))

    def run(self, entity, limit_articles_number=None):
        article_list = self.query_wikidata(entity, limit_articles_number)
        return article_list

class EntityDomainGetter():

    def get_entity_from_keyword(self, keyword):
        url = "https://www.wikidata.org/w/api.php?action=wbsearchentities&language=en&format=json&limit=3&search=" + keyword
        req = requests.get(url)
        data = json.loads(req.text)
        entities = []
        for x in data['search']:
            print(x['label'])
            entities.append(x['id'])
        return entities

    def get_domain_for_entity(self, entity):
        url = 'http://www.wikidata.org/wiki/Special:EntityData/' + entity + '.json'
        req = requests.get(url)
        data = json.loads(req.text)
        domains = set()
        for x in data['entities'][entity]['claims']['P31']:
            domains.add(x['mainsnak']['datavalue']['value']['id'])
        return list(domains)

class EntityGetter():

    def get_entity_from_keyword(self, keyword):
        url = "https://www.wikidata.org/w/api.php?action=wbsearchentities&language=en&format=json&limit=3&search=" + keyword
        req = requests.get(url)
        data = json.loads(req.text)
        entities = []
        for x in data['search']:
            print(x['label'])
            entities.append(x['id'])
        return entities

    def run(self,keyword):
        return self.get_entity_from_keyword(keyword)


# -*- coding: utf-8 -*-
from __future__ import print_function
import sys
import os
import time
import subprocess
import logging
import shlex
import re
from application import const

deco = logging.getLogger('decoration')

def case_1():
    logging.critical('请在已安装源码检索工具的平台上运行！')

def case_2():
    logging.critical('检查：系统默认安装之后的账号管理')

    blacklist = ['test']

    # 扫描组账户
    if os.path.exists(os.path.abspath('/etc/group')):
        with open(os.path.abspath('/etc/group'), 'r') as f:
            illegal = []
            for line in f:
                # 特征关键字检查
                for v in blacklist:
                    if re.search(r'' + v, line, re.I | re.U):
                        illegal.append(line.strip('\n'))

                # GID检查
                gid = int(line.strip('\n').split(':')[2])
                if gid > 1000:
                    illegal.append(line.strip('\n'))

            illegal = list(set(illegal))

            if len(illegal) == 0:
                logging.info('结果：无非法组账户')
            else:
                logging.error('结果：发现可疑组账户')
                for v in illegal:
                    logging.warning(v)

def case_3():
    logging.critical('扫描：账号登录权限')

    # 扫描组账户
    if os.path.exists(os.path.abspath('/etc/passwd')):
        with open(os.path.abspath('/etc/passwd'), 'r') as f:
            illegal = []
            normal = []
            for line in f:
                record = line.strip('\n').split(':')
                user = record[0]
                uid = int(record[2])
                shell = record[len(record) - 1]

                if user != 'root' and uid == 0:
                    illegal.append(user)

                if uid > 1000 and shell != '/sbin/nologin':
                    normal.append(user)

            illegal = list(set(illegal))

            if len(illegal) != 0:
                logging.error('发现非法root权限账户')
                for v in illegal:
                    logging.info(v)

            if len(normal) == 0:
                logging.info('结果：无可疑账户')
            else:
                logging.error('结果：发现多个拥有登录权限的账户')
                for v in normal:
                    logging.warning(v)

@const.run(module=globals(),feature='账户管理')
def run(module, case):
    module[case]()


from mysite.views import *
from django.conf.urls.defaults import *
from django.conf import settings

urlpatterns = patterns('',
    (r'^$', main_page),

    # Login / logout.
    (r'^login/$', 'django.contrib.auth.views.login'),
    (r'^accounts/login/$', 'django.contrib.auth.views.login'),
    (r'^logout/$', logout_page),

    # Web portal.
    (r'^myapp/', include('myapp.urls')),

    # Serve static content.
    (r'^static/(?P<path>.*)$', 'django.views.static.serve',
        {'document_root': 'static'}),
)

if settings.DEBUG:
    urlpatterns += patterns('',
        url(r'^media/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.MEDIA_ROOT, 'show_indexes': True}),
)


import seaborn as sns
import pandas as pd
import csv

def age_parser(age_txt):
	mirna2age = {}
	age2mirna = {}

	age_lst = [alpha for alpha in list(csv.reader(age_txt,delimiter='\t')) if alpha[0][0] != '#' ]

	for i in age_lst:
		mirna2age[i[0]] = float(i[1])
		age2mirna.setdefault(float(i[1]),[]).append(i[0])

	with open('../relevant_data/hsa_all_mir_lst.txt','w') as mirlst:
		for mirna in mirna2age:
			mirlst.write(mirna + '\n')

	return mirna2age, age2mirna

def sortmir(mirna2age, mir_expdb):
	newlst = []

	for val in mir_expdb.index:
		alpha =  sum(mir_expdb.loc[val].tolist())
		if val in mirna2age:
			if float(mirna2age[val]) < 100.0:
				newlst.append([alpha, 'Under 100.0 MY'])
			else:
				newlst.append([alpha, 'Over 100.0 MY'])

	return pd.DataFrame(newlst, columns=['Number of Tissues Expressed In', 'miRNA Age'])

def genfig(db):
	sns.violinplot(x='miRNA Age', y='Number of Tissues Expressed In',data=db, cut=0)
	sns.plt.gca().set_ylim([0,20])
	sns.plt.savefig('../figures/mir_old_yung_tisexp_violinplot.pdf',bbox_inches='tight')

	sns.plt.close()

	sns.boxplot(x='miRNA Age', y='Number of Tissues Expressed In',data=db)
	sns.plt.gca().set_ylim([0,20])
	sns.plt.savefig('../figures/mir_old_yung_tisexp_boxplot.pdf',bbox_inches='tight')

	sns.plt.close()

mir_expdb = pd.read_csv('../relevant_data/exp_data_alldmir.txt', sep='\t',index_col=[0])
agedb = '../relevant_data/allmir_ages/hsa_family_file_ph_allmir_dollo_age-time.protein_list'
mirna2age, age2mirna = age_parser(open(agedb, 'r'))

fig_db = sortmir(mirna2age, mir_expdb)

genfig(fig_db)


# coding:utf-8

import numpy as np
import matplotlib.pyplot as plt
import pdb

"旅行商问题 ( TSP , Traveling Salesman Problem )"
coordinates = np.array([[0, 0]])

def initData():
  global coordinates
  fr = open('../tc/qa194.txt')
  for line in fr.readlines():
    lineArr = line.strip().split()
    temp = np.array([[float(lineArr[1]), float(lineArr[2])]])
    coordinates = np.vstack((coordinates, temp))
  fr.close()
  coordinates = coordinates[1:]
  print(coordinates.shape[0])

def show_path(cl, cost, T):
  plt.clf()
  plt.scatter(coordinates[:, 1], coordinates[:, 0], s=5)
  for i in range(cl.shape[0] - 1):
    plt.plot([coordinates[cl[i]][1], coordinates[cl[i + 1]][1]], [coordinates[cl[i]][0], coordinates[cl[i + 1]][0]])
  plt.plot([coordinates[cl[0]][1], coordinates[cl[cl.shape[0] - 1]][1]], [coordinates[cl[0]][0], coordinates[cl[cl.shape[0] - 1]][0]])
  plt.title('T: ' + str(T) + ', cost: ' + str(cost))
  plt.draw()
  plt.pause(0.1)

#得到距离矩阵的函数
def getdistmat(coordinates):
    num = coordinates.shape[0] #194个坐标点
    distmat = np.zeros((194,194)) #194X194距离矩阵
    for i in range(num):
        for j in range(i,num):
            distmat[i][j] = distmat[j][i]=np.linalg.norm(coordinates[i]-coordinates[j])
    return distmat

def initpara():
    alpha = 0.99
    t = (1,200)
    markovlen = 1000

    return alpha,t,markovlen

initData()
num = coordinates.shape[0]
distmat = getdistmat(coordinates) #得到距离矩阵

solutionnew = np.arange(num)
#valuenew = np.max(num)

solutioncurrent = solutionnew.copy()
valuecurrent =99000  #np.max这样的源代码可能同样是因为版本问题被当做函数不能正确使用，应取一个较大值作为初始值
#print(valuecurrent)

solutionbest = solutionnew.copy()
valuebest = 99000 #np.max

alpha,t2,markovlen = initpara()
t = t2[1]

count = 0
result = [] #记录迭代过程中的最优解
while t > t2[0]:
    for i in np.arange(markovlen):

        #下面的两交换和三角换是两种扰动方式，用于产生新解
        if np.random.rand() > 0.5:# 交换路径中的这2个节点的顺序
            # np.random.rand()产生[0, 1)区间的均匀随机数
            while True:#产生两个不同的随机数
                loc1 = np.int(np.ceil(np.random.rand()*(num-1)))
                loc2 = np.int(np.ceil(np.random.rand()*(num-1)))
                ## print(loc1,loc2)
                if loc1 != loc2:
                    break
            solutionnew[loc1],solutionnew[loc2] = solutionnew[loc2],solutionnew[loc1]
        else: #三交换
            while True:
                loc1 = np.int(np.ceil(np.random.rand()*(num-1)))
                loc2 = np.int(np.ceil(np.random.rand()*(num-1)))
                loc3 = np.int(np.ceil(np.random.rand()*(num-1)))

                if((loc1 != loc2)&(loc2 != loc3)&(loc1 != loc3)):
                    break

            # 下面的三个判断语句使得loc1<loc2<loc3
            if loc1 > loc2:
                loc1,loc2 = loc2,loc1
            if loc2 > loc3:
                loc2,loc3 = loc3,loc2
            if loc1 > loc2:
                loc1,loc2 = loc2,loc1

            #下面的三行代码将[loc1,loc2)区间的数据插入到loc3之后
            tmplist = solutionnew[loc1:loc2].copy()
            solutionnew[loc1:loc3-loc2+1+loc1] = solutionnew[loc2:loc3+1].copy()
            solutionnew[loc3-loc2+1+loc1:loc3+1] = tmplist.copy()

        valuenew = 0
        for i in range(num-1):
            valuenew += distmat[solutionnew[i]][solutionnew[i+1]]
        valuenew += distmat[solutionnew[0]][solutionnew[193]]
       # print (valuenew)
        if valuenew<valuecurrent: #接受该解

            #更新solutioncurrent 和solutionbest
            valuecurrent = valuenew
            solutioncurrent = solutionnew.copy()

            if valuenew < valuebest:
                valuebest = valuenew
                solutionbest = solutionnew.copy()
        else:#按一定的概率接受该解
            if np.random.rand() < np.exp(-(valuenew-valuecurrent)/t):
                valuecurrent = valuenew
                solutioncurrent = solutionnew.copy()
            else:
                solutionnew = solutioncurrent.copy()
    t = alpha*t
    result.append(valuebest)
    print (("%d : %d") % (t, valuebest))
    count = count + 1
    if count == 10:
        show_path(solutionbest, valuebest, t)
        count = 0

plt.clf()
plt.scatter(coordinates[:, 1], coordinates[:, 0], s=5)
for i in range(solutionbest.shape[0] - 1):
  plt.plot([coordinates[solutionbest[i]][1], coordinates[solutionbest[i + 1]][1]], [coordinates[solutionbest[i]][0], coordinates[solutionbest[i + 1]][0]])
plt.plot([coordinates[solutionbest[0]][1], coordinates[solutionbest[solutionbest.shape[0] - 1]][1]], [coordinates[solutionbest[0]][0], coordinates[solutionbest[solutionbest.shape[0] - 1]][0]])
plt.title('T: ' + str(t) + ', cost: ' + str(valuebest))

plt.show()


"""
Delete round transformation
"""

from __future__ import print_function
import random
import os

import torch
import torch.nn.parallel
import torch.backends.cudnn as cudnn
cudnn.benchmark = True
cudnn.fastest = True

import torchvision.transforms as T

from trainer import Trainer
from config import get_config
from dataloader import get_loader

def main(config):
    # make directory
    if config.ckpt_folder is None:
        config.ckpt_folder = 'checkpoints'
    os.system('mkdir {0}'.format(config.ckpt_folder))
    print("[*] Make checkpoints folder!")

    if config.sample_folder is None:
        config.sample_folder = 'samples'
    os.system('mkdir {0}'.format(config.sample_folder))
    print("[*] Make samples folder!")

    config.manual_seed = random.randint(1, 10000)
    print("Random Seed: ", config.manual_seed)

    random.seed(config.manual_seed)
    torch.manual_seed(config.manual_seed)
    torch.cuda.manual_seed_all(config.manual_seed)

    # for faster training
    cudnn.benchmark = True

    # define train/test transformation
    transform = T.Compose([
        T.ToTensor(),
        T.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),
    ])

    # define train/test dataloader
    train_loader, test_loader = get_loader(dataroot=config.dataroot,
                                           batch_size=config.batch_size,
                                           shuffle=config.shuffle,
                                           transform=transform,
                                           num_workers=config.num_workers)

    # define trainer class
    trainer = Trainer(train_loader, test_loader, config)

    if config.mode == 'train':
        trainer.train()
    elif config.mode == 'test':
        trainer.eval()

if __name__ == "__main__":
    config = get_config()
    main(config)


import re

input_text = input('Please put in a text containing your pin: ')
pin_pattern = re.compile('[0-9]+')
pin_match = pin_pattern.match(input_text)
try:
    pin = pin_match.group()
    print('Your pin contains',len(pin),'numbers: ', pin)
except:
    print('No pin given!')


# test file for splineS function

def bspline_splineS():
    import numpy as np
    from xspline.__init__ import bspline

    ok = True
    # setup test problem
    # -------------------------------------------------------------------------
    knots = np.linspace(0.0, 1.0, 3)
    degree = 3

    bs = bspline(knots, degree)

    # test splineS function
    # -------------------------------------------------------------------------
    a = len(bs.knots) - 2
    tol = 1e-10

    tr_cl = np.repeat(bs.knots[:-1], [bs.degree + 1] + [1]*a)
    tr_cr = np.repeat(bs.knots[1:], [1]*a + [bs.degree + 1])
    tr_c = np.vstack((tr_cl, tr_cr)).T
    my_c = np.array([bs.splineS(i) for i in range(bs.num_spline_bases)])

    ok = ok and np.linalg.norm(my_c - tr_c) < tol

    if not ok:
        print('tr_c', tr_c)
        print('my_c', my_c)

    return ok


#!/usr/bin/env python

import rospy
import sys
import csv
import glob
import sys
import numpy as np
import os

from kinova_scripts.srv import Joint_angles

if __name__ == '__main__':
    ##########################################################################################
    # Reads the joint angles and sends them to the plannar to create
    ##########################################################################################

    try:  # get input from command line for which file to use
	    n = sys.argv[1]
    except:
	    n = 6

    rospy.init_node('joint_angles', argv=sys.argv, disable_signals=True)

    directory = os.path.dirname(os.path.realpath(__file__))

    angles = np.zeros((1,13))

    # set the file number parameter so other programs update
    rospy.set_param('file_number', str(n))

    # get angles
    # with open(directory + '/final_test/test_data/Matrices/Angles_Optimized.2.csv') as f:  # '/final_test/test_data/Matrices/Angles_' + str(n) +'.0.csv'
    # with open(directory + '/final_test/test_data/Matrices/Angles_' + str(n) +'.0.csv') as f:
    #     reader = csv.reader(f)
    #     for j, row in enumerate(reader):
    #         for i, col in enumerate(row):
    #             angles[j][i] = float(col)

    arm_angles = angles[0]
    print(arm_angles)
    rospy.wait_for_service('joint_angles')

    set_angles = rospy.ServiceProxy('joint_angles', Joint_angles)

    # send the angles to the plannar to move the arm
    try:
        answer = set_angles(arm_angles)
    except rospy.ServiceException as e:
        rospy.logwarn('Service call failed for: {0}'.format(e))

    rospy.loginfo( '{0}'.format(answer.success))

    rospy.signal_shutdown('{0}'.format(answer.success))


X = int(input())
Y = int(input())

maior = X if X > Y else Y
menor = Y if Y < X else X

maior += 1
total = 0

for i in range(menor, maior):
  if i%13 != 0:
    total += i

print(total)


import re
import os
#-----------------------------------------step0 prepare all the textfile--------------------------------------------------------------------------------------------
train_dir = r'./dataset/20news-bydate/20news-bydate-train'
dic = []
# print(os.listdir(train_dir))
# root_dir = r'.\dataset\20news-bydate\20news-bydate-train\comp.os.ms-windows.misc'
# print(os.path.abspath(root_dir))
# print(os.listdir(root_dir))
for train_var in os.listdir(train_dir):
    print(len(dic))
    temp_dir = train_dir + '/' + train_var
    for data_file in os.listdir(temp_dir):
        file_addr = temp_dir + '/' + data_file
        # read file and cutting head
        fp = open(file_addr, encoding='gb18030', errors = 'ignore') #tag:maybe error later
        lines = fp.readlines()
        fp.close()
        head = 0
        for item in enumerate(lines):
            # print(item[1][0:5])
            if item[1][0:5].strip() == 'Lines':
                head = item[0]
                break
        del lines[0:head+1]
        # splitting the words
        # print(lines)
        for sentence in lines:
            temp_list = list(filter(None, re.split(r'[0-9!:@#$%^&*?/<>,.+=_;|{}~`\-\s\n\(\)\\\'\[\]\"]\s*', sentence)))
            for word in temp_list:
                if word == '':
                    continue
                if word not in dic:
                    dic.append(word)
# finish and minimize and deliminate
for i in range(0,len(dic)):
    dic[i] = dic[i].lower()
dic = (list(set(dic)))
dic.sort()#bad idea
# delete stopword
fp = open('stopwords.txt')
try:
    alltext = fp.read()
    # print(alltext)
finally:
    fp.close()
stop_list = alltext.split('\n')
# print(stop_list)
temp = 0
for i in range(0, len(dic)):
    if dic[temp] in stop_list:
        del dic[temp]
    else:
        temp += 1
print (dic)

"""
cutting head, spliting the words altogether and then lowerrize and deliminate
it is about the space and the time
 """

"""
#-----------------------------------------step1 cut the head--------------------------------------------------------------------------------------------
fp = open('37915')
lines = fp.readlines()
#print(lines)
fp.close()
head = 0
for item in enumerate(lines):
#    print(item[1][0:5])
    if item[1][0:5].strip() == 'Lines':
        head = item[0]
        break
del lines[0:head+1]
#print(lines)

#-----------------------------------------step2 split it up and deleting--------------------------------------------------------------------
print ("step2-------------------------------------------------\n")
dic = []
for item in lines:
    temp_list = list(filter(None, re.split(r'[!@#$%^&*?\-,.\s\n\(\)]\s*', item)))
    # please remeber there is a ......issue
    # print(temp_list)
    for word in temp_list:
        if word == '':
            continue
        if word not in dic:
            dic.append(word)
print(dic)

#-----------------------------------------step3 lowerize the characters--------------------------------------------------------------------
print ("step3-------------------------------------------------\n")
for i in range(0,len(dic)):
    dic[i] = dic[i].lower()
dic = (list(set(dic)))
# can't use the sort directly in the above line.
dic.sort()#bad idea
print(dic)

#-----------------------------------------step4 dealing with the stopwords--------------------------------------------------------------------
print ("step 4-----------------------------------------------\n")

fp = open('stopwords.txt')
try:
    alltext = fp.read()
    # print(alltext)
finally:
    fp.close()
stop_list = alltext.split('\n')
# print(stop_list)
temp = 0
for i in range(0, len(dic)):
    if dic[temp] in stop_list:
        del dic[temp]
    else:
        temp += 1
print (dic)

//AFTER PREPROCESS

 """

#-----------------------------------------preparing the stopwords list----------------------------------------------------------
'''
fp = open('stopwords.txt')
try:
    alltext = fp.read()
    print(alltext)
finally:
    fp.close()
stop_list = alltext.split('\n')
print(stop_list)

'''
#------------------------------------------testing the split and resplit--------------------------------------------------
'''
temp = "like me anyway!"
temp_list = list(filter(None, re.split(r'[;,\s!]\s*', temp)))
#temp_list = temp.split(' ')
print(temp_list)
sentence = "are you love me or hate me anyway bitch like "
sentence_list = sentence.split(' ')
print(sentence_list)
output = []
for w in sentence_list:
    if w in temp_list:
        output.append(w)
#output = [w for w in sentence_list if w in temp_list]
print(output)
'''

#------------------------------------------the original module------------------------------------------
'''
print(temp.split())
print(temp)
print("hello world")
'''
'''
sentence = "this is a apple"
filter_sentence= [w for w in sentence.split(' ') if w not in stopwords.words('english')]
print(filter_sentence)
'''


import pandas as pd
import numpy as np
import time
import matplotlib.pyplot as plt
import pickle
import os
import params as p
from concurrent.futures import ProcessPoolExecutor
from functools import partial
import datalib as dl
import trade as t
import exchange as ex

# Init Q table with small random values
def init_q():
    qt = pd.DataFrame()
    if p.train:
        qt = pd.DataFrame(np.random.normal(scale=p.random_scale, size=(p.feature_bins**p.features,p.actions)))
        # Use Optimistic Values: 1
#        qt = pd.DataFrame(np.ones((p.feature_bins**p.features,p.actions)))
        qt['visits'] = 0
        qt['conf'] = 0
        qt['ratio'] = 0.0
    else:
        if os.path.isfile(p.q): qt = pickle.load(open(p.q, "rb" ))
    return qt

# Calculate Discretised State based on features
def get_state(row):
    bins = p.feature_bins
    if p.version == 1:
        state = int(bins**3*row.binrsi+bins**2*row.binadr+bins*row.binhh+row.binll)
    elif p.version == 2:
        state = int(bins**3*row.binrsma+bins**2*row.bindsma+bins*row.binrsi+row.binhhll)
    visits = qt.at[state, 'visits']
    conf = qt.at[state, 'conf']
    return state, visits, conf

# P Policy: P[s] = argmax(a)(Q[s,a])
def get_action(state, test=True):
    if (not test) and (np.random.random() < p.epsilon):
        # choose random action with probability epsilon
        max_action = np.random.randint(0,p.actions)
    else:
        #choose best action from Q(s,a) values
        max_action = int(qt.iloc[state,0:p.actions].idxmax(axis=1))

    max_reward = qt.iat[state, max_action]
    return max_action, max_reward

# Execute Action: buy or sell
def take_action(pf, action, dr):
    old_total = pf.total
    target = pf.total*actions.iat[action,0] # Target portfolio
    if target >= 0: # Long
        if pf.short > 0: t.sell_lot(pf, pf.short, True) # Close short positions first
        diff = target - pf.equity
        if diff > 0: t.buy_lot(pf, diff)
        elif diff < 0: t.sell_lot(pf, -diff)
    else: # Short
        if pf.equity > 0: t.sell_lot(pf, pf.equity) # Close long positions first
        diff = -target - pf.short
        if diff > 0: t.buy_lot(pf, diff, True)
        elif diff < 0: t.sell_lot(pf, -diff, True)

    # Calculate reward as a ratio to maximum daily return
    # reward = 1 - (1 + abs(dr))/(1 + dr*(equity-cash)/total)

    # Update Balance
    pf.equity = pf.equity*(1 + dr)
#    pf.short = pf.short*(1 - dr) This calculation is incorrect
    pf.upd_total()
    reward = pf.total/old_total - 1
    # Calculate Reward as pnl + cash dr
#    reward = (1+pnl)*(1-dr*pf.cash/old_total) - 1
    return reward

# Update Rule Formula
# The formula for computing Q for any state-action pair <s, a>, given an experience tuple <s, a, s', r>, is:
# Q'[s, a] = (1 - α) · Q[s, a] + α · (r + γ · Q[s', argmaxa'(Q[s', a'])])
#
# Here:
#
# r = R[s, a] is the immediate reward for taking action a in state s,
# γ ∈ [0, 1] (gamma) is the discount factor used to progressively reduce the value of future rewards,
# s' is the resulting next state,
# argmaxa'(Q[s', a']) is the action that maximizes the Q-value among all possible actions a' from s', and,
# α ∈ [0, 1] (alpha) is the learning rate used to vary the weight given to new experiences compared with past Q-values.
#
def update_q(s, a, s1, r):
    action, reward = get_action(s1)
    q0 = qt.iloc[s, a]
    q1 = (1 - p.alpha)*q0 + p.alpha*(r + p.gamma*reward)
    qt.iloc[s, a] = q1
    qt.at[s1, 'visits'] += 1

# Iterate over data => Produce experience tuples: (s, a, s', r) => Update Q table
# In test mode do not update Q Table and no random actions (epsilon = 0)
def run_model(df, test=False):
    global qt
    df = df.assign(state=-1, visits=1, conf=0, action=0, equity=0.0, cash=0.0, total=0.0, pnl=0.0)
    pf = t.Portfolio(p.start_balance)

    for i, row in df.iterrows():
        if i == 0:
            state, visits, conf = get_state(row) # Observe New State
            action = 0 # Do not take any action in first day
        else:
            old_state = state
            if test and conf == 0: # Use same action if confidence is low
                action = action
            else:
                # Find Best Action based on previous state
                action, _ = get_action(old_state, test)
            # Take an Action and get Reward
            reward = take_action(pf, action, row.dr)
            # Observe New State
            state, visits, conf = get_state(row)
            # If training - update Q Table
            if not test: update_q(old_state, action, state, reward)
            df.at[i, 'pnl'] = reward

        df.at[i, 'visits'] = visits
        df.at[i, 'conf'] = conf
        df.at[i, 'action'] = action
        df.at[i, 'state'] = state
        df.at[i, 'equity'] = pf.equity
        df.at[i, 'cash'] = pf.cash
        df.at[i, 'total'] = pf.total

    if not test:
        qt['r'] = qt.visits * (qt.iloc[:,:p.actions].max(axis=1) - qt.iloc[:,:p.actions].min(axis=1))
        qt['ratio'] = qt.r / qt.r.sum()
        qt['conf'] = (qt['ratio'] > p.ratio).astype('int')

    return df

def train_model(df, tdf):
    global qt
    print("*** Training Model using "+p.ticker+" data. Epochs: %s ***" % p.epochs)

    max_r = 0
    max_q = qt
    for ii in range(p.epochs):
        # Train Model
        df = run_model(df)
        # Test Model
        tdf = run_model(tdf, test=True)
        if p.train_goal == 'R':
            r = dl.get_ret(tdf.total)
        else:
            r = dl.get_sr(tdf.pnl)
#        print("Epoch: %s %s: %s" % (ii, p.train_goal, r))
        if r > max_r:
            max_r = r
            max_q = qt.copy()
            print("*** Epoch: %s Max %s: %s" % (ii, p.train_goal, max_r))

    qt = max_q
    if max_r > p.max_r:
        print("*** New Best Model Found! Best R: %s" % (max_r))
        # Save Model
        pickle.dump(qt, open(p.cfgdir+'/q'+str(int(1000*max_r))+'.pkl', "wb" ))

def show_result(df, title):
    # Thanks to: http://benalexkeen.com/bar-charts-in-matplotlib/
    if p.result_size > 0: df = df.tail(p.result_size).reset_index(drop=True)
    df['nclose'] = dl.normalize(df.close) # Normalise Price
    df['ntotal'] = dl.normalize(df.total) # Normalise Price
    if p.charts:
        d = df.set_index('date')
        d['signal'] = d.action-d.action.shift(1)
        fig, ax = plt.subplots()
        ax.plot(d.nclose, label='Buy and Hold')
        ax.plot(d.ntotal, label='QL', color='red')

        # Plot buy signals
        ax.plot(d.loc[d.signal == 1].index, d.ntotal[d.signal == 1], '^',
                markersize=10, color='m', label='BUY')
        # Plot sell signals
        ax.plot(d.loc[d.signal == -1].index, d.ntotal[d.signal == -1], 'v',
                markersize=10, color='k', label='SELL')

        fig.autofmt_xdate()
        plt.title(title+' for '+p.conf)
        plt.ylabel('Return')
        plt.legend(loc='best')
        plt.grid(True)
        plt.show()

    qlr = dl.get_ret(df.ntotal)
    qlsr = dl.get_sr(df.pnl)
    bhr = dl.get_ret(df.nclose)
    bhsr = dl.get_sr(df.dr)
    print("R: %.2f SR: %.3f QL/BH R: %.2f QL/BH SR: %.2f" % (qlr, qlsr, qlr/bhr, qlsr/bhsr))
    print("AVG Confidence: %.2f" % df.conf.mean())
    print('QT States: %s Valid: %s Confident: %s' %
          (len(qt), len(qt[qt.visits > 0]), len(qt[qt.conf >= 1])))

def get_today_action(tdf):
    action = 'HOLD'
    if tdf.action.iloc[-1] != tdf.action.iloc[-2]:
        action = 'BUY' if tdf.action.iloc[-1] > 0 else 'SELL'
    return action

def print_forecast(tdf):
    print()
    position = p.currency if tdf.cash.iloc[-1] > 0 else p.ticker
    print('Current position: '+position)
    print('Today: '+get_today_action(tdf))

    state = tdf.state.iloc[-1]
    next_action, reward = get_action(state)
    conf = qt.conf.iloc[state]
    action = 'HOLD'
    if next_action != tdf.action.iloc[-1] and conf >= 1:
        action = 'BUY' if next_action > 0 else 'SELL'
    print('Tomorrow: '+action)

def init(conf):
    global actions
    global tl
    global qt

    p.load_config(conf)

    qt = init_q() # Initialise Model
    actions = pd.DataFrame(np.linspace(-1 if p.short else 0, 1, p.actions))
    if os.path.isfile(p.tl):
        tl = pickle.load(open(p.tl, "rb" ))
    else:
        tl = t.TradeLog()

def execute_action():
    print('!!!EXECUTE MODE!!!')
    action = get_today_action(tdf)
    if action == 'HOLD': return
    amount = tl.cash if action == 'buy' else tl.equity
    cash, equity = ex.market_order(action, amount)
    tl.log_trade(action, cash, equity) # Update trade log
    pickle.dump(tl, open(p.tl, "wb" ))

def run_forecast(conf, seed = None):
    global tdf
    global df

    if seed is not None: np.random.seed(seed)
    init(conf)

    dl.load_data() # Load Historical Price Data
    # This needs to run before test dataset as it generates bin config
    if p.train: df = dl.get_dataset() # Read Train data.
    tdf = dl.get_dataset(test=True) # Read Test data
    if p.train: train_model(df, tdf)

    tdf = run_model(tdf, test=True)
    if p.stats: show_result(tdf, "Test") # Test Result
    print_forecast(tdf) # Print Forecast
    if p.execute: t.execute_action()

def run_batch(conf, instances = 1):
    if instances == 1:
        run_forecast(conf)
        return
    ts = time.time()
    run_forecast_a = partial(run_forecast, conf) # Returning a function of a single argument
    with ProcessPoolExecutor() as executor: # Run multiple processes
        executor.map(run_forecast_a, range(instances))

    print('Took %s', time.time() - ts)

#run_batch('ETHUSD')


#!/usr/bin/env python
# coding: utf-8

# In[443]:

import numpy as np
import torch
import sys
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

from torch.utils import data
from torchvision import transforms
from torchvision.datasets import MNIST

import matplotlib.pyplot as plt
import time
import scipy.io

# In[444]:

cuda = torch.cuda.is_available()
cuda
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# In[445]:

train_data = scipy.io.loadmat('../data/nist36_train.mat')
valid_data = scipy.io.loadmat('../data/nist36_valid.mat')
test_data = scipy.io.loadmat('../data/nist36_test.mat')

train_x, train_y = train_data['train_data'], train_data['train_labels']
valid_x, valid_y = valid_data['valid_data'], valid_data['valid_labels']
test_x, test_y = test_data['test_data'], test_data['test_labels']

print(train_x.shape)
print(train_x[0])

# In[ ]:

# In[446]:

class MyDataset(data.Dataset):
    def __init__(self, X, Y):
        self.X = X
        self.Y = Y

    def __len__(self):
        return self.Y.shape[0]

    def __getitem__(self,index):
        X = self.X[index].float().reshape(-1)
        Y = self.Y[index].long()
        return X,Y

# In[447]:

class FC_model(nn.Module):
    def __init__(self):
        super(FC_model,self).__init__()
        layers = []
        layers.append(nn.Linear(1024,64))
        layers.append(nn.Sigmoid())
        layers.append(nn.Linear(64,36))
        self.net = nn.Sequential(*layers)

    def forward(self, x):
        return self.net(x)
FC_model()

# In[448]:

model = FC_model()
criterion = nn.CrossEntropyLoss()
learning_rate = 1e-2
optimizer = optim.SGD(model.parameters(), lr=learning_rate)
def init_xavier(m):
    if type(m) == nn.Linear:
        fan_in = m.weight.size()[1]
        fan_out = m.weight.size()[0]
        std = np.sqrt(2.0 / (fan_in + fan_out))
        m.weight.data.normal_(0,std)

model.apply(init_xavier)

# In[449]:

# g0 = np.random.multivariate_normal([3.6,40],[[0.05,0],[0,10]],10)
# g1 = np.random.multivariate_normal([3.9,10],[[0.01,0],[0,5]],10)
# g2 = np.random.multivariate_normal([3.4,30],[[0.25,0],[0,5]],10)
# g3 = np.random.multivariate_normal([2.0,10],[[0.5,0],[0,10]],10)
# train_x = np.vstack([g0,g1,g2,g3])
# # we will do XW + B
# # that implies that the data is N x D

# # create labels
# y_idx = np.array([0 for _ in range(10)] + [1 for _ in range(10)] + [2 for _ in range(10)] + [3 for _ in range(10)])
# # turn to one_hot
# y = np.zeros((y_idx.shape[0],y_idx.max()+1))
# y[np.arange(y_idx.shape[0]),y_idx] = 1
# train_y=y

# In[450]:

train_data = torch.tensor(train_x,dtype=torch.float)
train_targets=torch.tensor(train_y, dtype=torch.long)
train_dataset = MyDataset(train_data, train_targets)

valid_data = torch.tensor(valid_x,dtype=torch.float)
valid_targets=torch.tensor(valid_y, dtype=torch.long)
valid_dataset = MyDataset(valid_data, valid_targets)

train_loader_args = dict(shuffle=True, batch_size=16, pin_memory=True)
train_loader = data.DataLoader(train_dataset, **train_loader_args)

valid_loader_args = dict(shuffle=False, batch_size=16, pin_memory=True)
valid_loader = data.DataLoader(valid_dataset, **valid_loader_args)

# In[451]:

def train_epoch(model, train_loader, criterion, optimizer):
    #model.train()
    #model.to(device)
    running_loss = 0.0
    total_acc =0.0
    print(running_loss)

    start_time = time.time()
    for batch_idx, (data, target) in enumerate(train_loader):

        #data=data.to(device)

        #print("--------------------------------",data.shape)

        optimizer.zero_grad()
        label = np.where(target == 1)[1]
        label = torch.tensor(label)
        outputs = model(data)
        #print("++++++++++++++++++++++++++++++++++",outputs.shape)
        _, predicted = torch.max(outputs.data, 1)
        #label = label.to(device)
        total_acc+= ((label==predicted).sum().item())
        loss = criterion(outputs, label)
        running_loss += loss.item()

        loss.backward()
        optimizer.step()

    end_time = time.time()

    avg_acc = total_acc / train_x.shape[0]
    return running_loss,avg_acc

#train_epoch(model, train_loader, criterion, optimizer)

# In[452]:

def test_model(model, test_loader, criterion):
    with torch.no_grad():
        model.eval()
        #model.to(device)

        running_loss = 0.0
        total_predictions = 0.0
        correct_predictions = 0.0

        for batch_idx, (data, target) in enumerate(test_loader):

            outputs = model(data)

            total_predictions += target.size(0)

            label = np.where(target == 1)[1]
            label = torch.tensor(label)

            _, predicted = torch.max(outputs.data, 1)
            correct_predictions += (predicted == label).sum().item()

            loss = criterion(outputs, label).detach()
            running_loss += loss.item()

        acc = (correct_predictions/valid_x.shape[0])

        return running_loss, acc
#test_model(model, valid_loader, criterion)

# In[453]:

n_epochs=50
TrainLoss=[]
TrainAcc=[]
ValidLoss=[]
ValidAcc=[]
TestLoss=[]
TestAcc=[]
for i in range(n_epochs):
    train_loss,avg_acc = train_epoch(model, train_loader, criterion, optimizer)
    valid_loss,valid_acc = test_model(model, valid_loader, criterion)
    TrainAcc.append(avg_acc)
    TrainLoss.append(train_loss)
    ValidAcc.append(valid_acc)
    ValidLoss.append(valid_loss)

    if(i%2==0):
        print("itr: {:02d} \t loss: {:.2f} \t acc : {:.2f}".format(i,train_loss, avg_acc))
        print(" Validation loss: {:.2f} \t Validation acc : {:.2f}".format(valid_loss, valid_acc))
    #print(train_loss)
    #break

# In[456]:

plt.figure(0)
plt.title('Training Loss and validation loss')
plt.xlabel('Epoch Number')
plt.ylabel('Loss')
plt.plot(TrainLoss , 'r')
plt.plot(ValidLoss, 'b')
plt.legend(['training loss','valid loss'])
plt.show()

# In[457]:

plt.figure(0)
plt.title('Training ACC and validation ACC')
plt.xlabel('Epoch Number')
plt.ylabel('Loss')
plt.plot(TrainAcc , 'r')
plt.plot(ValidAcc, 'b')
plt.legend(['training ACC','valid ACC'])

plt.show()

# In[ ]:


# importing all required package library
from keras.applications import ResNet50
from keras.preprocessing.image import img_to_array
from keras.applications import imagenet_utils
import numpy as np

# A class for image prediction based on ResNet pre-trained model
class ResNetPredict:
    def __init__(self):
        self.image = None
        self.model = None
        self.result_value = None

    def load_model(self):
        self.model = ResNet50(weights="imagenet")

    def prepare_image(self, image, target):
        if image.mode != "RGB":
            image = image.convert("RGB")
        image = image.resize(target)
        image = img_to_array(image)
        image = np.expand_dims(image, axis=0)
        image = imagenet_utils.preprocess_input(image)
        return image

    def predict(self, image):
        prediction = self.model.predict(image)
        results = imagenet_utils.decode_predictions(prediction)
        return results