# -*- coding: utf-8 -*-
# Traffic Data Process Automatically
# Xin Meng
# 2015.11.02
# This script is used for process the traffic txt file.
# 1. Set the configure
# 2. Run this script and get the result.
import csv
import os
import logging
import re
import utils.network
__author__ = 'xin'

# configuration variable

# OutputCSVDir = ""
# ReportDir = ""
# NormalDBDir = ""
# InputNormalDir = ""
# InputInfectDir = ""

# 这里5个重要的参数：
# 输出的csv目录
output_csv_dir = ""
# 输出的报告目录
report_dir = ""
# normal db目录
normal_db_dir = ""
# 输入的normal文件目录
input_normal_dir = ""
# 输入的infect文件目录
input_infect_dir = ""
normal_db_file = "normal_db.txt"
public_ip_range_file = "public_ip_range.txt"
output_extension = ".csv"
# remember if you want to change the global variance in function, you need use global xxx
normal_db_size = 0
# Current Path (the Script)
path = os.path.split(os.path.realpath(__file__))[0]
# The Scan Information
num_normal_input_file = 0
num_infect_input_file = 0
normal_input_file_list = []
infect_input_file_list = []
# Globle statistic information
num_normal_packet = 0
num_packet_label_infect = 0
num_packet_label_normal = 0
# remember if you want to change the global variance in function, you need use global xxx
normal_db_size = 0
# Current Path (the Script)
path = os.path.split(os.path.realpath(__file__))[0]
# The Scan Information
num_normal_input_file = 0
num_infect_input_file = 0
normal_input_file_list = []
infect_input_file_list = []
# Global statistic information
num_normal_packet = 0
num_packet_label_infect = 0
num_packet_label_normal = 0

def match_normal_db(b_infect_packet, public_ip_range, str_normal_db):
    # 打开一个DB文件,将所有DB数据读入到String中,
    flag_public_range = 0
    with open(normal_db_dir + normal_db_file, 'a+') as normal_db:
        for normal_db_line in normal_db:
            str_normal_db += normal_db_line
    for public_ip_range_line in public_ip_range:
        # compare the b_infect_packet ip source or ip destination with the IP range whether in the range
        # Because there is one of the two IP is the mobile IP address, so we just find one of [0] [1]
        # in the range, we decide the packet is normal
        if 1 == in_ip_range(public_ip_range_line, b_infect_packet[0]) or \
                        1 == in_ip_range(public_ip_range_line, b_infect_packet[1]):
            flag_public_range = 1
            break
        else:
            flag_public_range = 0
    if 1 == flag_public_range:
        # the infect packet ip in the public ip range
        return 1
    else:
        # 直接用正则表达式 将b_infect_packet 拼接成一个字符串, 在 str_normal_db 进行匹配.
        infect_ip_source_str = str(long(b_infect_packet[0]))
        infect_ip_des_str = str(long(b_infect_packet[1]))
        # infect_ip_source_str_array = infect_ip_source_str.split('.')
        # infect_ip_des_str_array = infect_ip_des_str.split('.')
        str_pattern = ".*" + infect_ip_source_str + " " + infect_ip_des_str
        pattern = re.compile(str_pattern)
        match = pattern.match(str_normal_db)
        if match:
            return 1
        else:
            return 0

####################################################
# FUNCTION: Load Configuration
# RETURN: InputNormalDir
# InputInfectDir
# OutputCSVDir
# ReportDir
# NormalDBDir
####################################################

def load_conf():
    # 主要是获取
    global output_csv_dir
    global report_dir
    global normal_db_dir
    global input_normal_dir
    global input_infect_dir
    # from __future__ import with_statement
    # DataProcess.cfg
    import ConfigParser

    config = ConfigParser.ConfigParser()
    # G:\2015-11-01-Research-experiment\0_MobileBotnetAnalysisLab\TrafficDataProcessAuto_pc.cfg
    with open(path + "\TrafficDataProcessAuto_pc.cfg", "r") as cfgfile:
        # with open(path+"\DataProcess.cfg", "r") as cfgfile:
        config.readfp(cfgfile)
        input_dir = config.get("DIR", "InputDir")
        normal_dir = config.get("DIR", "NormalDir")
        infect_dir = config.get("DIR", "InfectDir")

        output_csv_dir = os.path.normcase(config.get("DIR", "OutputDir"))
        report_dir = os.path.normcase(config.get("DIR", "ReportDir"))
        normal_db_dir = os.path.normcase(config.get("DIR", "NormalDB"))

    input_normal_dir = os.path.normcase(input_dir + normal_dir)
    input_infect_dir = os.path.normcase(input_dir + infect_dir)

    # return (InputNormalDir,InputInfectDir,OutputCSVDir,ReportDir,NormalDBDir)

####################################################
# FUNCTION: Scan the Input Data directory
####################################################

def scan_input_data():
    global num_normal_input_file
    global num_infect_input_file
    global normal_input_file_list
    global infect_input_file_list
    # Load Configuration File to get parameters
    load_conf()
    # List the normal and infect files
    logger.info(input_normal_dir)
    logger.info(input_infect_dir)

    normal_input_file_list = os.listdir(input_normal_dir)
    infect_input_file_list = os.listdir(input_infect_dir)
    num_normal_input_file = len(normal_input_file_list)
    num_infect_input_file = len(infect_input_file_list)

    logger.info(num_infect_input_file)
    logger.info(num_normal_input_file)
    # Show the result of scanning
    show_scan_result(1)

####################################################
# FUNCTION1: Show Scanning Result
# Input:     The level of show 0-Infect 1-Normal 2-Normal&Infect
####################################################

def show_scan_result(level=2):
    print("--------------------Scanning Result--------------------")
    if level > 0:
        print("The number of Normal file:" + str(num_normal_input_file))
        print("Files:")
        for i in range(0, num_normal_input_file):
            print("Input " + str(i + 1) + " to process: " + normal_input_file_list[i])
    if level < 1 or level > 1:
        print("The number of Infect file:" + str(num_infect_input_file))
        print("Files:")
        for i in range(0, num_infect_input_file):
            print("Input " + str(i + 1) + " to process: " + infect_input_file_list[i])
    print("--------------------Scanning End--------------------")

####################################################
# FUNCTION: ip2int
# INPUT:    IP
# RETURN:   int of the IP
####################################################

def ip2int(ip):
    import struct
    import socket

    return struct.unpack("!I", socket.inet_aton(ip))[0]

####################################################
# FUNCTION: int2ip
# INPUT:    int of IP
# RETURN:   IP
####################################################

def int2ip(i):
    import socket
    import struct

    return socket.inet_ntoa(struct.pack("!I", i))

####################################################
########## FUNCTION: change the IP/subset to IP range
########## Input: "216.239.32.0/19"
########## Return: IP range IP cmin cmax  216.239.32.0 216.239.63.255
####################################################
def subnet_mask_to_ip_range(iplist):
    data = iplist.split('/')
    ip = data[0]
    ti = int(data[1])
    d = int(ti / 8)
    c = 256 / (2 ** (ti % 8))
    ip_items = ip.split('.')
    if len(ip_items[d:]) == 1:
        if ti % 8 == 0:
            cmin = '%s.%s' % ('.'.join(ip_items[:d]), '0')
            cmax = '%s.%s' % ('.'.join(ip_items[:d]), '255')
        else:
            for i in range(2 ** (ti % 8)):
                mymax = (i + 1) * c - 1
                mymin = i * c
                data = int(''.join(ip_items[d:]))
                if data < mymax and data >= mymin:
                    cmin = '%s.%s' % ('.'.join(ip_items[:d]), mymin)
                    cmax = '%s.%s' % ('.'.join(ip_items[:d]), mymax)
    else:
        if ti % 8 == 0:
            cmin = '%s.%s.%s' % ('.'.join(ip_items[:d]), '0', ('0.' * (len(ip_items) - d - 1))[:-1])
            cmax = '%s.%s.%s' % ('.'.join(ip_items[:d]), '255', ('255.' * (len(ip_items) - d - 1))[:-1])
        else:
            for i in range(2 ** (ti % 8)):
                mymax = (i + 1) * c - 1
                mymin = i * c
                data = int(''.join(ip_items[d]))
                if data < mymax and data >= mymin:
                    cmin = '%s.%s.%s' % ('.'.join(ip_items[:d]), mymin, ('0.' * (len(ip_items) - d - 1))[:-1])
                    cmax = '%s.%s.%s' % ('.'.join(ip_items[:d]), mymax, ('255.' * (len(ip_items) - d - 1))[:-1])
    # print  cmin, cmax
    return (cmin, cmax)

####################################################
########## FUNCTION: decision for whether the IP in a IP range
########## Input: IP range string "216.239.32.0/19" (string) , IP (int) to decide
########## Return: 1 in the range
##########         0 not in the range
####################################################
def in_ip_range(subnet_mask_ip, ip):
    ip_range_lower, ip_range_upper = subnet_mask_to_ip_range(subnet_mask_ip)
    int_ip_lower = ip2int(ip_range_lower)
    int_ip_upper = ip2int(ip_range_upper)
    # print(int_ip_lower)
    # print(int_ip_upper)
    # print(ip)
    # print(ip_range_lower)
    # print(ip_range_upper)
    # print(int2ip(ip))
    if int_ip_upper >= ip >= int_ip_lower:
        return 1
    else:
        return 0

####################################################
# FUNCTION: compare between normal packet with the normal database
# Input: a: packet from traffic b: packet from normal database
####################################################

def compare(a, b):
    if long(a[0]) == long(b[0]) and long(a[1]) == long(b[1]) and long(a[2]) == long(b[2]):
        return 1
    else:
        return 0

####################################################
# FUNCTION: compare packet in infect traffic file with the packet in normal database,
# find whether the packet is normal or infect
# Input: a: packet from infect traffic file b: packet from normal database
# Return: 1 normal packet match with the packet in database
# 0 infect packet not match the database
####################################################

def decision_normal_infect(a_normal_packet, b_infect_packet):
    # 0 source IP
    # 1 destination IP
    # we use the fuzzy matching to decide whether the packet in the database or not
    # example:  192.168.123.***
    # Step1: compare with google public IP address
    # Step2: wildcard of the last 3 number of the IP address

    # Step 1
    # Load the public IP range address from database dir : public_ip_range.txt
    flag_public_range = 0
    with open(normal_db_dir + public_ip_range_file, 'a+') as public_ip_range:
        for public_ip_range_line in public_ip_range:
            # compare the b_infect_packet ip source or ip destination with the IP range whether in the range
            # Because there is one of the two IP is the mobile IP address, so we just find one of [0] [1]
            # in the range, we decide the packet is normal
            if 1 == in_ip_range(public_ip_range_line, b_infect_packet[0]) or \
                            1 == in_ip_range(public_ip_range_line, b_infect_packet[1]):
                flag_public_range = 1
                break
            else:
                flag_public_range = 0
    if 1 == flag_public_range:
        # the infect packet ip in the public ip range
        return 1
    else:
        # Step 2
        # compare the ip and protocol with the normal database packet, we use fuzzy matching,
        infect_ip_source_str = int2ip(b_infect_packet[0])
        infect_ip_des_str = int2ip(b_infect_packet[1])
        normal_ip_source_str = int2ip(int(a_normal_packet[0]))
        normal_ip_des_str = int2ip(int(a_normal_packet[1]))
        infect_ip_source_str_array = infect_ip_source_str.split('.')
        infect_ip_des_str_array = infect_ip_des_str.split('.')
        normal_ip_source_str_array = normal_ip_source_str.split('.')
        normal_ip_des_str_array = normal_ip_des_str.split('.')

        # logger.debug(infect_ip_source_str)
        # logger.debug(infect_ip_des_str)
        # logger.debug(normal_ip_source_str)
        # logger.debug(normal_ip_des_str)
        #
        #
        # logger.debug(infect_ip_source_str_array)
        # logger.debug(infect_ip_des_str_array)
        # logger.debug(normal_ip_source_str_array)
        # logger.debug(normal_ip_des_str_array)
        # set the level of fuzzy
        # level = 3    xxx.xxx.xxx only match the first 3 ip number
        level = 3
        index = 0
        match_flag = 1
        while index < level:
            if infect_ip_source_str_array[index] == normal_ip_source_str_array[index] and \
                            infect_ip_des_str_array[index] == normal_ip_des_str_array[index]:
                index += 1
            else:
                match_flag = 0
                break
        if match_flag == 1:
            return 1
        else:
            return 0

####################################################
# FUNCTION: Process Normal File
# Input: The number of the files in the normal list
####################################################

def normal_process(normal_file_no):
    global normal_db_size
    # Stream: initial 2 dimensions array to store the packet base on the stream index

    tcp_stream = [[[0 for col in range(0)] for row in range(0)] for high in range(50000)]
    udp_stream = [[[0 for col in range(0)] for row in range(0)] for high in range(20000)]
    num_udp_stream = 0
    num_tcp_stream = 0

    # 获取当前需要处理的文件
    process_normal_file = input_normal_dir + normal_input_file_list[normal_file_no - 1]
    print (process_normal_file)
    # Load Normal Packet database, if the file not exist, create it. And append the file every process
    # normal_db = open(normal_db_dir + normal_db_file, 'a+')
    normal_db_temp = [[0 for col in range(10)] for row in range(0)]
    with open(normal_db_dir + normal_db_file, 'a+') as normal_db:
        for normal_db_line in normal_db:
            # normal_db_array = normal_db_line.split()
            normal_db_line_array = normal_db_line.split()
            normal_db_temp += [normal_db_line_array]
            normal_db_size += 1
    # Open normal csv file and initial the head of the table
    # print (output_csv_dir + normal_input_file_list[normal_file_no - 1] + output_extension)
    # 增加三个属性
    with open(output_csv_dir + normal_input_file_list[normal_file_no - 1] + output_extension, 'wSb+') as csv_file:
        csv_writer = csv.writer(csv_file, dialect='excel')
        csv_writer.writerow(
            ['IP source', 'IP des', 'Protocol', 'Frame duration', 'UDP size', 'TCP size', 'Argument Count', 'TimeStamp',
             'T1', 'T2', 'T3', 'Lable'])

    # Stream: The output file will store the stream information
    # process every line in the normal file
    # -e ip.src -e ip.dst -e ip.proto
    # -e frame.time_delta
    # -e udp.length -e udp.stream
    # -e tcp.len -e tcp.stream
    # -e http.request.uri
    # The structure of every line: 9 if
    # IP source 0 , IP destination 1, IP protocol 2,
    # Frame duration 3,
    # UDP size 4, UDP stream index 5,
    # TCP size 6, TCP stream index 7,
    # URL 8
    # tshark -r %%f -o tcp.calculate_timestamps:true -n -T fields -e ip.src -e ip.dst -e ip.proto -e frame.time_delta
    # -e udp.length -e udp.stream -e tcp.len -e tcp.stream -e http.request.uri -e frame.time_epoch >%%f%outputFormat%
    # UPD  7
    # |--ip.src--|--ip.dst--|--ip.proto--|--frame.time_delta--|--udp.length--|--udp.stream--|--frame.time_epoch--|
    # TCP  8
    # |--ip.src--|--ip.dst--|--ip.proto--|--frame.time_delta--|--tcp.len--|--tcp.stream--|--http.request.uri--|--frame.time_epoch--|
    for txt_line in open(process_normal_file):
        txt_array = txt_line.split()
        # 重新构造txt_array
        # 如果长度小于7，直接过滤掉
        if 7 > len(txt_array):
            continue
        # 首先处理第1，2字段，ip源地址
        if len(txt_array[0]) <= 15:
            # IP地址长度大于255.255.255.255,则为IPv6
            ip_source = ip2int(txt_array[0])
            txt_array[0] = ip_source
            ip_des = ip2int(txt_array[1])
            txt_array[1] = ip_des

            # 6 UPD 将6,7,8字段配置为0
            # 17 TCP 将4,5字段配置0
            if txt_array[2] != '17':
                # TCP 6 UDP 17
                # if not UDP, SET udp size AS 0
                txt_array.insert(4, '0')
                txt_array.insert(5, '0')
            if txt_array[2] != '6':
                txt_array.insert(6, '0')
                txt_array.insert(7, '0')
                txt_array.insert(8, '0')
            else:
                # 如果是TCP并且包含有HTTP URI的话，字段为10，则计算参数个数
                if len(txt_array) == 10:
                    # include the HTTP request uri
                    arg_num = txt_array.pop(8).count('=')
                    txt_array.insert(8, str(arg_num))
                else:
                    arg_num = 0
                    txt_array.insert(8, str(arg_num))
            # 9 --> timestamp
            # txt_array.insert(9, 'normal')
            # 时间戳保留在字段9
            # 这里将lable插入到最后一个位置，这个位置不确定，有可能有带*的行，则多一个元素，所以这里，
            # txt_array.insert(10, 'normal')
            txt_array.append('normal')
            # Decided whether the packet has already existed in the normal_db
            status = 0
            # 遍历临时的db列表，比较该package的IP信息是否存在
            for i in normal_db_temp:
                if 1 == compare(i, txt_array):
                    status = 1
                    break
            # 如果不存在，则加入到 normal db中
            if 0 == status:
                normal_db_temp += [txt_array]
                normal_db_size += 1
                # append the data into to database file
                with open(normal_db_dir + normal_db_file, 'a+') as normal_db:
                    # normal_db_array = normal_db_line.split()
                    n = 0
                    s = ""
                    while n < len(txt_array):
                        s += str(txt_array[n])
                        s += " "
                        n += 1
                    s += "\n"
                    # txt_array_convert = ' '.join(str(txt_array))
                    normal_db.writelines(s)

            # 将该txt_array加入到 Stream中,最后将Stream写入CSV文件
            if '17' == txt_array[2]:
                # UDP process
                udp_stream[int(txt_array[5])].append(txt_array)

            elif '6' == txt_array[2]:
                # TCP process
                # logger.info("index of the stream:")
                # logger.info(txt_array[7])
                tcp_stream[int(txt_array[7])].append(txt_array)

    # 这是已经获得了当前文件的所有的 package，并且根据每个 package 的stream index 信息，将package存放到对应的stream二维数组中了。
    # 下面组合stream
    # 记录之前的Stream开始和结束时间。
    pre_start_timestamp = 0
    pre_end_timestamp = 0
    cur_start_timestamp = 0
    cur_end_timestamp = 0
    for packet_list in udp_stream:
        # 组合udp_stream中的每个stream的packet,将信息存储在CSV文件中.
        if len(packet_list) != 0:
            # 8['IP source', 'IP des', 'Protocol', 'Frame duration', 'UDP size', 'TCP size', 'Argument Count', 'Lable'])
            # 现在开始拼接udp_stream里面的每个packet_list到一个stream
            stream_temp = [0 for col in range(12)]
            stream_temp[0] = packet_list[0][0]
            stream_temp[1] = packet_list[0][1]
            stream_temp[2] = packet_list[0][2]
            for packet in packet_list:
                stream_temp[3] += float(packet[3])
                stream_temp[4] += int(packet[4])
                # 如果packet[9]为 *， 则取下一个做时间戳字段
                if packet[9] == "*":
                    logger.info("Encounter some line with *")
                    packet.pop(9)
                # 将stream中package的最早时间戳和最晚时间戳记录下来。
                if cur_start_timestamp == 0 and cur_end_timestamp == 0:
                    cur_start_timestamp = float(packet[9])
                    cur_end_timestamp = float(packet[9])
                if float(packet[9]) < cur_start_timestamp:
                    cur_start_timestamp = float(packet[9])
                if float(packet[9]) > cur_end_timestamp:
                    cur_end_timestamp = float(packet[9])

            stream_temp[5] = 0
            stream_temp[6] = 0
            # 将第一个package的时间戳，作为本stream的时间戳
            stream_temp[7] = packet_list[0][9]

            # 这里我们需要处理一下时间戳，加入时间戳属性。
            # 这里我们需要加入三个feature
            # 2. The experiment for the more features (time) of stream with ML
            # more features:
            # a) |Ts - Te| time between start and end of every stream
            # b) |Ts2 - Ts1| time between start time of adjacent stream
            # c) |Te1 - Ts2| time between end and start time of adjacent stream
            stream_temp[8] = abs(cur_end_timestamp - cur_start_timestamp)
            stream_temp[9] = abs(cur_start_timestamp - pre_start_timestamp)
            stream_temp[10] = abs(cur_start_timestamp - pre_end_timestamp)

            # 将第一个的label，作为本stream的label
            stream_temp[11] = packet_list[0][10]
            pre_start_timestamp = cur_start_timestamp
            pre_end_timestamp = cur_end_timestamp
            cur_start_timestamp = 0
            cur_end_timestamp = 0
            # Write the CSV file for the normal files.
            with open(output_csv_dir + normal_input_file_list[normal_file_no - 1] + output_extension, 'ab+') \
                    as csv_file:
                csv_writer = csv.writer(csv_file, dialect='excel')
                csv_writer.writerow(stream_temp)
            num_udp_stream += 1
    pre_start_timestamp = 0
    pre_end_timestamp = 0
    cur_start_timestamp = 0
    cur_end_timestamp = 0
    for packet_list in tcp_stream:
        # 组合udp_stream中的每个stream的packet,将信息存储在CSV文件中.
        if len(packet_list) != 0:
            # 8['IP source', 'IP des', 'Protocol', 'Frame duration', 'UDP size', 'TCP size', 'Argument Count', 'Lable'])
            stream_temp = [0 for col in range(12)]
            stream_temp[0] = packet_list[0][0]
            stream_temp[1] = packet_list[0][1]
            stream_temp[2] = packet_list[0][2]
            for packet in packet_list:
                stream_temp[3] += float(packet[3])
                stream_temp[5] += int(packet[6])
                stream_temp[6] += int(packet[8])
                # 将stream中package的最早时间戳和最晚时间戳记录下来。
                if cur_start_timestamp == 0 and cur_end_timestamp == 0:
                    cur_start_timestamp = float(packet[9])
                    cur_end_timestamp = float(packet[9])
                if float(packet[9]) < cur_start_timestamp:
                    cur_start_timestamp = float(packet[9])
                if float(packet[9]) > cur_end_timestamp:
                    cur_end_timestamp = float(packet[9])
            stream_temp[4] = 0
            stream_temp[7] = packet_list[0][9]
            # 这里我们需要处理一下时间戳，加入时间戳属性。
            # 这里我们需要加入三个feature
            # 2. The experiment for the more features (time) of stream with ML
            # more features:
            # a) |Ts - Te| time between start and end of every stream
            # b) |Ts2 - Ts1| time between start time of adjacent stream
            # c) |Te1 - Ts2| time between end and start time of adjacent stream
            stream_temp[8] = abs(cur_end_timestamp - cur_start_timestamp)
            stream_temp[9] = abs(cur_start_timestamp - pre_start_timestamp)
            stream_temp[10] = abs(cur_start_timestamp - pre_end_timestamp)
            # 将第一个的label，作为本stream的label
            stream_temp[11] = packet_list[0][10]
            pre_start_timestamp = cur_start_timestamp
            pre_end_timestamp = cur_end_timestamp
            cur_start_timestamp = 0
            cur_end_timestamp = 0
            # Write the CSV file for the normal files.
            with open(output_csv_dir + normal_input_file_list[normal_file_no - 1] + output_extension, 'ab+') \
                    as csv_file:
                csv_writer = csv.writer(csv_file, dialect='excel')
                csv_writer.writerow(stream_temp)
            num_tcp_stream += 1

####################################################
# FUNCTION: Process Normal File
# Input: The number of the files in the normal list
####################################################

def infect_process(infect_file_no):
    global num_packet_label_infect
    global num_packet_label_normal
    logger.info("Start to process the infect file")
    tcp_stream = [[[0 for col in range(0)] for row in range(0)] for high in range(50000)]
    udp_stream = [[[0 for col in range(0)] for row in range(0)] for high in range(20000)]
    num_udp_stream = 0
    num_tcp_stream = 0

    process_infect_file = input_infect_dir + infect_input_file_list[infect_file_no - 1]
    print (process_infect_file)
    # Load Normal Packet database, if the file not exist, create it. And append the file every process
    # normal_db = open(normal_db_dir + normal_db_file, 'a+')
    public_ip_range_temp = []
    with open(normal_db_dir + public_ip_range_file, 'a+') as public_ip_range:
        for public_ip_range_line in public_ip_range:
            public_ip_range_temp += [public_ip_range_line]

    str_normal_db = ""
    normal_db_temp = [[0 for col in range(10)] for row in range(0)]
    with open(normal_db_dir + normal_db_file, 'a+') as normal_db:
        for normal_db_line in normal_db:
            # normal_db_array = normal_db_line.split()
            normal_db_line_array = normal_db_line.split()
            normal_db_temp += [normal_db_line_array]
            str_normal_db += " "
            normal_db_line = normal_db_line.strip('\n')
            str_normal_db += normal_db_line
    # Open normal csv file and initial the head of the table
    # print (output_csv_dir + normal_input_file_list[normal_file_no - 1] + output_extension)
    with open(output_csv_dir + infect_input_file_list[infect_file_no - 1] + output_extension, 'wSb+') as csv_file:
        csv_writer = csv.writer(csv_file, dialect='excel')
        csv_writer.writerow(
            ['IP source', 'IP des', 'Protocol', 'Frame duration', 'UDP size', 'TCP size', 'Argument Count', 'TimeStamp',
             'T1', 'T2', 'T3', 'Lable'])
    # process every line in the normal
    num_txt_line = 0
    for txt_line in open(process_infect_file):
        logger.info("Process the txt line: " + str(num_txt_line))
        num_txt_line += 1
        txt_array = txt_line.split()
        if 7 > len(txt_array):
            continue
        if len(txt_array[0]) <= 15:
            # IP地址长度大于255.255.255.255,则为IPv6
            IP_source = ip2int(txt_array[0])
            txt_array[0] = IP_source
            IP_des = ip2int(txt_array[1])
            txt_array[1] = IP_des

            if txt_array[2] != '17':
                # TCP 6 UDP 17
                # if not UDP, SET udp size AS 0
                txt_array.insert(4, '0')
                txt_array.insert(5, '0')
            if txt_array[2] != '6':
                txt_array.insert(6, '0')
                txt_array.insert(7, '0')
                txt_array.insert(8, '0')
            else:
                if len(txt_array) == 10:
                    # include the HTTP request uri
                    arg_num = txt_array.pop(8).count('=')
                    txt_array.insert(8, str(arg_num))
                else:
                    arg_num = 0
                    txt_array.insert(8, str(arg_num))

            # Decisied whether the packet is Normal or Infect
            # We use some algorithms to compare with the normal database, and
            # status = 1 : normal packet match the database
            # status = 0 : infect packet not match the database
            # 将这块匹配工作进行重新设计, 使用正则表达式进行匹配, 加快速度.
            status = 0
            # 判断当前txt_array是否为infect:
            status = match_normal_db(txt_array, public_ip_range_temp, str_normal_db)
            if status:
                txt_array.append('normal')
                num_packet_label_normal += 1
            else:
                txt_array.append('infect')
                num_packet_label_infect += 1
            # for i in normal_db_temp:
            #     if 1 == decision_normal_infect(i, txt_array):
            #         status = 1
            #         txt_array.append('normal')
            #         # txt_array.insert(10, 'normal')
            #         num_packet_label_normal += 1
            #         break
            # if 0 == status:
            #     txt_array.append('infect')
            #     # txt_array.insert(10, 'infect')
            #     num_packet_label_infect += 1

            # 将该txt_array加入到 Stream中,最后将Stream写入CSV文件
            if '17' == txt_array[2]:
                # UDP process
                udp_stream[int(txt_array[5])].append(txt_array)
                # print(int(txt_array[5]))
            elif '6' == txt_array[2]:
                # TCP process
                tcp_stream[int(txt_array[7])].append(txt_array)
                # print(int(txt_array[7]))
    logger.info("Start to write the csv file")

    pre_start_timestamp = 0
    pre_end_timestamp = 0
    cur_start_timestamp = 0
    cur_end_timestamp = 0

    for packet_list in udp_stream:
        # 组合udp_stream中的每个stream的packet,将信息存储在CSV文件中.
        if len(packet_list) != 0:
            # 8 ['IP source', 'IP des', 'Protocol', 'Frame duration', 'UDP size', 'TCP size', 'Argument Count', 'Lable'])
            stream_temp = [0 for col in range(12)]
            stream_temp[0] = packet_list[0][0]
            stream_temp[1] = packet_list[0][1]
            stream_temp[2] = packet_list[0][2]
            for packet in packet_list:
                stream_temp[3] += float(packet[3])
                stream_temp[4] += int(packet[4])
                # 如果packet[9]为 *， 则取下一个做时间戳字段
                if packet[9] == "*":
                    logger.info("Encounter some line with *")
                    packet.pop(9)
                # 将stream中package的最早时间戳和最晚时间戳记录下来。
                # 如果packet[9]为 *， 则取下一个做时间戳字段
                if packet[9] == "*":
                    logger.info("Encounter some line with *")
                    packet.pop(9)
                if cur_start_timestamp == 0 and cur_end_timestamp == 0:
                    cur_start_timestamp = float(packet[9])
                    cur_end_timestamp = float(packet[9])
                if float(packet[9]) < cur_start_timestamp:
                    cur_start_timestamp = float(packet[9])
                if float(packet[9]) > cur_end_timestamp:
                    cur_end_timestamp = float(packet[9])
            stream_temp[5] = 0
            stream_temp[6] = 0
            stream_temp[7] = packet_list[0][9]
            # 这里我们需要处理一下时间戳，加入时间戳属性。
            # 这里我们需要加入三个feature
            # 2. The experiment for the more features (time) of stream with ML
            # more features:
            # a) |Ts - Te| time between start and end of every stream
            # b) |Ts2 - Ts1| time between start time of adjacent stream
            # c) |Te1 - Ts2| time between end and start time of adjacent stream
            stream_temp[8] = abs(cur_end_timestamp - cur_start_timestamp)
            stream_temp[9] = abs(cur_start_timestamp - pre_start_timestamp)
            stream_temp[10] = abs(cur_start_timestamp - pre_end_timestamp)
            # 将第一个的label，作为本stream的label
            stream_temp[11] = packet_list[0][10]
            pre_start_timestamp = cur_start_timestamp
            pre_end_timestamp = cur_end_timestamp
            cur_start_timestamp = 0
            cur_end_timestamp = 0
            # Write the CSV file for the normal files.
            with open(output_csv_dir + infect_input_file_list[infect_file_no - 1] + output_extension, 'ab+') \
                    as csv_file:
                csv_writer = csv.writer(csv_file, dialect='excel')
                csv_writer.writerow(stream_temp)

            num_udp_stream += 1
    logger.info("Finish UDP CSV write, the sum of stream: " + str(num_udp_stream))
    pre_start_timestamp = 0
    pre_end_timestamp = 0
    cur_start_timestamp = 0
    cur_end_timestamp = 0
    for packet_list in tcp_stream:
        # 组合udp_stream中的每个stream的packet,将信息存储在CSV文件中.
        if len(packet_list) != 0:
            # 8 ['IP source', 'IP des', 'Protocol', 'Frame duration', 'UDP size', 'TCP size', 'Argument Count', 'Lable'])
            stream_temp = [0 for col in range(12)]
            stream_temp[0] = packet_list[0][0]
            stream_temp[1] = packet_list[0][1]
            stream_temp[2] = packet_list[0][2]
            for packet in packet_list:
                stream_temp[3] += float(packet[3])
                stream_temp[5] += int(packet[6])
                stream_temp[6] += int(packet[8])
                # 将stream中package的最早时间戳和最晚时间戳记录下来。
                if cur_start_timestamp == 0 and cur_end_timestamp == 0:
                    cur_start_timestamp = float(packet[9])
                    cur_end_timestamp = float(packet[9])
                if float(packet[9]) < cur_start_timestamp:
                    cur_start_timestamp = float(packet[9])
                if float(packet[9]) > cur_end_timestamp:
                    cur_end_timestamp = float(packet[9])
            stream_temp[4] = 0
            stream_temp[7] = packet_list[0][9]
            # 这里我们需要处理一下时间戳，加入时间戳属性。
            # 这里我们需要加入三个feature
            # 2. The experiment for the more features (time) of stream with ML
            # more features:
            # a) |Ts - Te| time between start and end of every stream
            # b) |Ts2 - Ts1| time between start time of adjacent stream
            # c) |Te1 - Ts2| time between end and start time of adjacent stream
            stream_temp[8] = abs(cur_end_timestamp - cur_start_timestamp)
            stream_temp[9] = abs(cur_start_timestamp - pre_start_timestamp)
            stream_temp[10] = abs(cur_start_timestamp - pre_end_timestamp)

            # 将第一个的label，作为本stream的label
            stream_temp[11] = packet_list[0][10]
            pre_start_timestamp = cur_start_timestamp
            pre_end_timestamp = cur_end_timestamp
            cur_start_timestamp = 0
            cur_end_timestamp = 0
            # Write the CSV file for the normal files.
            with open(output_csv_dir + infect_input_file_list[infect_file_no - 1] + output_extension, 'ab+') \
                    as csv_file:
                csv_writer = csv.writer(csv_file, dialect='excel')
                csv_writer.writerow(stream_temp)
            num_tcp_stream += 1
    logger.info("Finish TCP CSV write, the sum of stream: " + str(num_tcp_stream))

########
#
# Main Process
#
#######

####################################################
# Logger Configuration
####################################################
# 创建一个logger
logger = logging.getLogger('mainlogger')
logger.setLevel(logging.DEBUG)

# 创建一个handler，用于写入日志文件
fh = logging.FileHandler(path + '/mainlog.log')
fh.setLevel(logging.DEBUG)

# 再创建一个handler，用于输出到控制台
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)

# 定义handler的输出格式
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)

# 给logger添加handler
logger.addHandler(fh)
logger.addHandler(ch)

# 记录一条日志
logger.info('Test mainlog')
# Set Level of the logger,
# NOTSET < DEBUG < INFO < WARNING < ERROR < CRITICAL
# logger.setLevel(logging.WARNING)    #Show Debug Information
# logger.setLevel(logging.INFO)    #Show Debug Information
logger.setLevel(logging.DEBUG)
# logger.setLevel(logging.NOTSET)  #Not show Any Information

# 加载参数
load_conf()
# 处理 normal 目录下文件，生成csv文件并创建 normal db
scan_input_data()
i = 1
for i in range(1, num_normal_input_file + 1):
    logger.info("The current process for normal:")
    logger.info(i)
    normal_process(i)
i = 1
for i in range(1, num_infect_input_file + 1):
    logger.info("The current process for infect:")
    logger.info(i)
    infect_process(i)
# 处理 malware 目录下文件，生成csv文件


import cv2
import os
import time
from time import sleep
import numpy as np
import pytesseract
import threading
from actions import doAction
import nxbt
import copy
import torch
from predictor import divide, predict
#pytesseract.pytesseract.tesseract_cmd = r'C:/Program Files/Tesseract-OCR/tesseract.exe'
resetMacro = """
L R A 1s
"""
def grayscale(img):
    original = np.array(img)
    original = cv2.cvtColor(original, cv2.COLOR_RGB2BGR)
    gray_scale = cv2.cvtColor(original, cv2.COLOR_BGR2GRAY)
    return gray_scale

def checkText(img):
    text = pytesseract.image_to_string(img)
    #print(text)
    if 'VAMPIRE' in text:
        return True
    return False
state = 0
ministate = 0
image =np.random.random((5, 5))
distance = 1
def getState():
    global state
    global ministate
    global image
    cap = cv2.VideoCapture(0)
    # The device number might be 0 or 1 depending on the device and the webcam
    while(True):
        ret, frame = cap.read()
        cv2.imshow('Switch', frame)
        frame2 =copy.deepcopy(frame)
        image = copy.deepcopy(frame)
        state = np.moveaxis(frame, -1, 0).astype(np.double) # put channels in first dimension
        state = torch.tensor(state.copy(), dtype=torch.double)

        frame2 = cv2.resize(frame2, (100, 100), interpolation=cv2.INTER_CUBIC)
        frame2 = np.moveaxis(frame2, -1, 0).astype(np.double) # put channels in first dimension
        ministate = torch.tensor(frame2.copy(), dtype=torch.double)
        #ministate = torch.unsqueeze(ministate, 0)

        cv2.waitKey(1)
def predictDistance():
    global distance
    global image
    while True:
        imgs, w, h = divide(image, 5)
        predictions = []
        enemyx = 0
        enemyy = 0
        playerx = 0
        playery = 0

        for i in range(len(imgs)):
            img, x, y = imgs[i]
            pred = predict(img)
            if pred == 1:
                enemyx= x
                enemyy= y
            elif pred== 2:
                playerx= x
                playery = y
        distance =  abs(enemyx - playerx) + abs(enemyy - playery)
t = threading.Thread(target=getState)
t.start()
sleep(1)
i = threading.Thread(target=predictDistance)
i.start()
print(image.shape)
class env():
    def __init__(self, inverted = False, controller_index = 0):
        self.draw_window = True

        self.inverted = inverted
        self.curHp = 0
        self.curEnemyHp = 0
        self.last_hp = 0
        self.last_enemy_hp = 0
        self.gotHit = 0

        self.frame = 0
        self.hitEnemy = 0
        input("Press Enter when ready to connect controller")
        self.nx = nxbt.Nxbt()
        self.controller_index = self.nx.create_controller(nxbt.PRO_CONTROLLER)
        print("Controller index: " + str(self.controller_index))
        self.nx.wait_for_connection(self.controller_index)
        print("Connecting Controller Please Wait")
        print(f"Shape of Input {state.shape}, {ministate.shape}")
        print("Connected Controller")
        input("Press Enter to start when ready")
        print("Started")
    def setHps(self):
        self.curHp = state[1, 155, 435]
        self.curEnemyHp = state[1, 295, 435]
        if self.inverted:
            self.curHp = state[1, 295, 435]
            self.curEnemyHp = state[1, 155, 435]

    def getReward(self, action):
        reward = -1
        self.setHps()
        if(self.curHp != self.last_hp): # if hp changed
            self.gotHit = 1
            #reward -=20

        hit = 0
        if(self.curEnemyHp != self.last_enemy_hp): # if enemy hp changed
            self.hitEnemy = 1

        return reward
    def step(self, action):
        last_distance = distance
        done = False
        if self.frame >= 1000:
            self.frame = 0
            done = True
        self.hitEnemy = 0
        self.gothit = 0
        reward = 0

        self.last_hp = state[1, 155, 435]
        self.last_enemy_hp = state[1, 295, 435]
        lag, actionReward = doAction(self.nx, self.controller_index, action)

        while(lag > 0 or self.gotHit!=1):
            lag -= 1
            reward += self.getReward(action)
            self.frame += 1
            time.sleep(0.01)
        if(distance < last_distance & action <= 4):
            reward +=5
        elif(distance > last_distance & action > 4):
            reward -=10
        if self.hitEnemy == 1:
            reward += actionReward
        else:
            reward -= actionReward
        if self.gotHit:
            reward -=20
        next_state = ministate

        info = []
        return next_state, reward, done, info, distance
    def reset(self):
        self.gothit = 0
        self.hitEnemy =0
        time.sleep(.5)
        self.nx.macro(self.controller_index, resetMacro)
        next_state = ministate
        return next_state


from bs4 import BeautifulSoup as BSHTML

import os
import re

html_directory = "data/patent_htmls/"
fp_citations_dir = 'data/fp_citations_from_html'
if not os.path.exists(fp_citations_dir):
    os.makedirs(fp_citations_dir)

citations_per_patent = dict() # key is patent_number; value is array of citation texts
ref_locations_per_patent = dict() # key is patent_number; value is array of reference locations (1 or 2; front page or in-text)

filecount = 0
for filename in os.listdir(html_directory):

    patent_id = re.sub(".html","",filename)

    html_patent_file = open(html_directory + filename, 'r',encoding='utf-8')
    citations_in_unlabeled_outfile = open(fp_citations_dir + "/" + patent_id + ".fpcitations.txt", 'w',encoding='utf-8')
    citations_in_unlabeled_outfile.write("\n\n"+patent_id+"\n\n")

    filecount += 1

    htmlcontent = html_patent_file.read().replace('\n', '')
    htmlstruct = BSHTML(htmlcontent,"lxml")

    patent_citations_in_html = []
    nonpatent_citations_in_html = []

    """
    Find citations in the HTML metadata (front-page citations)
    """

    for meta in htmlstruct.findAll('meta',attrs={"name":"DC.relation"}):
        #print("patent_citation in meta\t",meta['content'])
        patent_citations_in_html.append(meta['content'])
    for meta in htmlstruct.findAll('meta',attrs={"name":"citation_reference"}):
        #print("nonpatent_citation in meta\t",meta['content'])
        nonpatent_citations_in_html.append(meta['content'])

    print (patent_id,
           len(patent_citations_in_html),
           len(nonpatent_citations_in_html),
           sep="\t"
           )

    for citation_text in sorted(nonpatent_citations_in_html):
        citations_in_unlabeled_outfile.write(citation_text+"\n")

    html_patent_file.close()
    citations_in_unlabeled_outfile.close()

print(filecount,"files read")


import ipv6_utils
import log

from static import OF_TABLE_NUM
from static import WLAN_IFACE
from static import GW_IFACE

from switch import OFRule
from switch import Switch
from switch import Link
from switch import AccessPointConf
from switch import GatewayConf

from event import *
# End import from iJOIN solution files

# Start import from Ryu files
from ryu.base import app_manager
from ryu.controller.handler import set_ev_cls
from ryu.controller.handler import MAIN_DISPATCHER
from ryu.ofproto import ofproto_v1_3 as ofproto
from ryu.ofproto import ofproto_v1_3_parser as ofp_parser
from ryu.topology.api import get_switch
from ryu.topology.event import EventSwitchEnter as RyuEventSwitchEnter
from ryu.topology.event import EventSwitchLeave as RyuEventSwitchLeave
from ryu.topology.event import EventLinkAdd as RyuEventLinkAdd
from ryu.topology.event import EventLinkDelete as RyuEventLinkDelete
from ryu.topology.event import EventPortAdd as RyuEventPortAdd
from ryu.topology.event import EventPortModify as RyuEventPortModify
from ryu.topology.event import EventPortDelete as RyuEventPortDelete
# End import from Ryu files

class Nmm(app_manager.RyuApp):
	"""
	================ =========================================================
	Attribute        Description
	================ =========================================================
	OFP_VERSIONS     Declaration of supported OFP version
	_EVENTS          The list of events provided by the RyuApp
	================ =========================================================
	"""
	OFP_VERSIONS = [ofproto.OFP_VERSION]
	_EVENTS = [EventTopologyUpdate, EventSwitchEnter, EventSwitchUpdate,
				EventSwitchLeave, EventLinkAdd, EventLinkDelete]

	def __init__(self, *args, **kwargs):
		"""
		================ =========================================================
		Attribute        Description
		================ =========================================================
		switches         The dictionary storing the switches as
		                 custom Switch isntance
		================ =========================================================
		"""
		super(Nmm, self).__init__(*args, **kwargs)
		self.logger = log.get_logger(self.name)

		self.switches = {}
		self.accesspoints = {}
		self.gateways = {}

		self.of_rules = {}

	def _initialise_switch_of_tables(self, switch):
		"""
		Flush the flow table when the switch connects to the controller.
		"""
		# Flush OpenFlow entries
		match = ofp_parser.OFPMatch()
		instructions = []
		actions = []

		datapath = switch.switch.dp
		ofproto = datapath.ofproto
		parser = datapath.ofproto_parser

		for i in xrange(1, OF_TABLE_NUM):
			mod = parser.OFPFlowMod(datapath = datapath, cookie = 0,
							cookie_mask = 0, command = ofproto.OFPFC_DELETE,
							idle_timeout = 0, hard_timeout = 0, table_id = i,
							buffer_id = ofproto.OFPCML_NO_BUFFER, out_port = ofproto.OFPP_ANY,
							out_group = ofproto.OFPG_ANY, match = match, instructions = instructions)
			datapath.send_msg(mod)

		for i in xrange(0, OF_TABLE_NUM-1):
			instructions = [ofp_parser.OFPInstructionGotoTable(i+1)]
			req = EventWriteOFRule(OFRule(switch, match, actions, instructions, table_id = i, priority = 0))
			self.send_event(req.dst, req)

	def _check_if_ap_port(self, port):
		"""
		Check if a switch port is an access port.
		"""
		return port.name.startswith(WLAN_IFACE)

	def _check_if_gw_port(self, port):
		"""
		Check if a switch port is a gateway port.
		"""
		return port.name.startswith(GW_IFACE)

	def _get_ap_conf(self, switch, port):
		"""
		Get AP configuration.
		"""
		ap_conf = AccessPointConf()
		ap_conf.port = port

		return ap_conf

	def _get_gw_conf(self, switch, port):
		"""
		Get GW configuration.
		"""
		gw_conf = GatewayConf()
		gw_conf.port = port
		# FIXME	HARDCODED
		gw_conf.nw_prefix = ('2020', str(hex(switch.switch.dp.id))[-4:],'0','0','0','0','0','0')
		gw_conf.nw_prefix_len = 32

		return gw_conf

	@set_ev_cls(EventPushPacket)
	def _handler_push_packet(self, ev):
		"""
		Send the Packet pkt through the OFPort port.
		"""
		datapath = ev.switch.switch.dp
		ofproto = datapath.ofproto
		parser = datapath.ofproto_parser

		actions = [parser.OFPActionOutput(port = ev.port.port_no)]
		out = parser.OFPPacketOut(datapath = datapath,
					buffer_id = ofproto.OFP_NO_BUFFER,
					in_port = ofproto.OFPP_CONTROLLER,
					actions = actions, data = ev.pkt.data)
		datapath.send_msg(out)

	@set_ev_cls(EventProcessPacket)
	def _handler_process_packet(self, ev):
		"""
		Send the Packet pkt through the OFPP_TABLE.
		"""
		datapath = ev.switch.switch.dp
		ofproto = datapath.ofproto
		parser = datapath.ofproto_parser

		actions = [parser.OFPActionOutput(port = ofproto.OFPP_TABLE)]
		out = parser.OFPPacketOut(datapath = datapath,
					buffer_id = ofproto.OFP_NO_BUFFER,
					in_port = ofproto.OFPP_CONTROLLER,
					actions = actions, data = ev.pkt.data)
		datapath.send_msg(out)

	@set_ev_cls(EventWriteOFRule)
	def _handler_write_of_rule(self, ev):
		"""
		Write OpenFlow rule on switch.
		If the rule is already present in the switch,
		don't write it again
		"""
		try:
			if ev.of_rule.key not in self.of_rules \
				or self.of_rules[ev.of_rule.key].actions != ev.of_rule.actions:

				datapath = ev.of_rule.switch.switch.dp
				ofproto = datapath.ofproto
				parser = datapath.ofproto_parser

				mod = parser.OFPFlowMod(datapath = datapath, table_id = ev.of_rule.table_id, priority = ev.of_rule.priority, match = ev.of_rule.match, instructions = ev.of_rule.instructions, cookie = ev.of_rule.cookie)
				datapath.send_msg(mod)

				self.of_rules[ev.of_rule.key] = ev.of_rule
		except KeyError:
			pass

	@set_ev_cls(EventDelOFRule)
	def _handler_del_of_rule(self, ev):
		"""
		Delete OpenFlow rule on switch.
		If the rule has been already deleted from the switch,
		don't delete it again
		"""
		try:
			if ev.of_rule.key in self.of_rules:
				datapath = ev.of_rule.switch.switch.dp
				ofproto = datapath.ofproto
				parser = datapath.ofproto_parser

				inst = []
				mod = parser.OFPFlowMod(datapath = datapath, cookie = ev.of_rule.cookie,
									cookie_mask = ev.of_rule.cookie_mask, table_id = ev.of_rule.table_id,
									command = ofproto.OFPFC_DELETE, idle_timeout = 0, hard_timeout = 0,
									priority = ev.of_rule.priority, buffer_id = ofproto.OFPCML_NO_BUFFER,
									out_port = ofproto.OFPP_ANY, out_group = ofproto.OFPG_ANY,
									match = ev.of_rule.match, instructions = inst)
				datapath.send_msg(mod)

				del self.of_rules[ev.of_rule.key]
		except KeyError:
			pass

	@set_ev_cls(EventSwitchRequest)
	def _handler_switch_request(self, req):
		"""
		Get a switch object given the dpid.
		"""
		switches = []
		try:
			if req.dpid is None:
				# reply all list
				switches = self.switches.values()
			elif req.dpid in self.switches:
				switches = [self.switches[req.dpid]]
		except KeyError:
			pass

		rep = EventSwitchReply(req.src, switches)
		self.reply_to_request(req, rep)

	@set_ev_cls(EventLinkRequest)
	def _handler_link_request(self, req):
		"""
		Get switch's links.
		"""
		links = []
		try:
			if req.dpid is None:
				# reply all list
				links = [in_li for sw in self.switches.itervalues() for li_li in sw.links.itervalues() for in_li in li_li]
			elif req.dpid in self.switches:
				links = [in_li for li_li in self.switches[req.dpid].links.itervalues() for in_li in li_li]
		except KeyError:
			pass

		rep = EventLinkReply(req.src, links)
		self.reply_to_request(req, rep)

	@set_ev_cls(RyuEventSwitchEnter, MAIN_DISPATCHER)
	def _handler_switch_enter(self, ev):
		"""
		Handler for event.EventSwitchEnter
		Add a node to the topology.
		"""
		switch = Switch(ev.switch)
		# Check switch capabilities
		self.switches[ev.switch.dp.id] = switch
		# HARDCODED
		for p in switch.switch.ports:
			if self._check_if_ap_port(p):
				ap_conf = self._get_ap_conf(switch, p)
				switch.is_ap = True
				switch.ap_conf = ap_conf
				self.accesspoints[ev.switch.dp.id] = switch
			elif self._check_if_gw_port(p):
				gw_conf = self._get_gw_conf(switch, p)
				switch.is_gw = True
				switch.gw_conf = gw_conf
				self.gateways[ev.switch.dp.id] = switch

		self._initialise_switch_of_tables(switch)

		ev_tu = EventSwitchEnter(switch)
		self.send_event_to_observers(ev_tu)

		ev_tu = EventTopologyUpdate(self.switches)
		self.send_event_to_observers(ev_tu)
		self.logger.info("Switch <" + str(hex(switch.switch.dp.id)) + "> connected")

	@set_ev_cls(RyuEventSwitchLeave, MAIN_DISPATCHER)
	def _handler_switch_leave(self, ev):
		"""
		Handler for event.EventSwitchLeave
		Delete a node from the topology.
		"""
		try:
			ev_tu = EventSwitchLeave(self.switches[ev.switch.dp.id])
			self.send_event_to_observers(ev_tu)

			del self.switches[ev.switch.dp.id]

			ev_tu = EventTopologyUpdate(self.switches)
			self.send_event_to_observers(ev_tu)
			self.logger.info("Switch <" + str(hex(ev.switch.dp.id)) + "> disconnected")
		except KeyError:
			pass

	@set_ev_cls(RyuEventLinkAdd, MAIN_DISPATCHER)
	def _handler_link_add(self, ev):
		"""
		Handler for event.EventLinkAdd
		Add a link to the topology.
		"""
		if ev.link.dst.dpid not in self.switches[ev.link.src.dpid].links:
			self.switches[ev.link.src.dpid].links[ev.link.dst.dpid] = []
		link = Link(ev.link)
		self.switches[ev.link.src.dpid].links[ev.link.dst.dpid].append(link)

		ev_tu = EventLinkAdd(link)
		self.send_event_to_observers(ev_tu)

		ev_tu = EventTopologyUpdate(self.switches)
		self.send_event_to_observers(ev_tu)

		self.logger.info("Link <" + str(hex(ev.link.src.dpid)) + ":" + ev.link.src.name + "> => <" +
						str(hex(ev.link.dst.dpid)) + ":" + ev.link.dst.name +"> appeared")

	@set_ev_cls(RyuEventLinkDelete, MAIN_DISPATCHER)
	def _handler_link_delete(self, ev):
		"""
		Handler for event.EventLinkDelete
		Delete a link from the topology.
		"""
		try:
			tmp_links = []
			link_to_del = None
			for l in self.switches[ev.link.src.dpid].links[ev.link.dst.dpid]:
				if ((ev.link.src.dpid == l.link.src.dpid and ev.link.src.port_no == l.link.src.port_no)
						and (ev.link.dst.dpid == l.link.dst.dpid and ev.link.dst.port_no == l.link.dst.port_no)):
					link_to_del = l
				else:
					tmp_links.append(l)

			if link_to_del:
				self.switches[ev.link.src.dpid].links[ev.link.dst.dpid] = tmp_links

				ev_tu = EventLinkDelete(link_to_del)
				self.send_event_to_observers(ev_tu)

				ev_tu = EventTopologyUpdate(self.switches)
				self.send_event_to_observers(ev_tu)
				self.logger.info("Link <" + str(hex(ev.link.src.dpid)) + ":" + ev.link.src.name + "> => <" +
							str(hex(ev.link.dst.dpid)) + ":" + ev.link.dst.name +"> disappeared")
		except KeyError:
			pass

	@set_ev_cls(RyuEventPortAdd, MAIN_DISPATCHER)
	def _handler_port_add(self, ev):
		"""
		Overwrite the legacy Ryu Event.
		"""
		try:
			switch = self.switches[ev.port.dpid]
			if self._check_if_ap_port(ev.port):
				ap_conf = self._get_ap_conf(switch, ev.port)
				switch.is_ap = True
				switch.ap_conf = ap_conf
				self.accesspoints[ev.port.dpid] = switch
			elif self._check_if_gw_port(ev.port):
				gw_conf = self._get_gw_conf(switch, ev.port)
				switch.is_gw = True
				switch.gw_conf = gw_conf
				self.gateways[ev.port.dpid] = switch

			switch.switch = get_switch(self, ev.port.dpid)[0]

			ev_tu = EventSwitchUpdate(switch)
			self.send_event_to_observers(ev_tu)

			self.logger.info("Port add: " + str(ev.port))
		except KeyError:
			pass

	@set_ev_cls(RyuEventPortModify, MAIN_DISPATCHER)
	def _handler_port_modify(self, ev):
		"""
		Overwrite the legacy Ryu Event.
		"""
		try:
			switch = self.switches[ev.port.dpid]
			switch.switch = get_switch(self, ev.port.dpid)[0]

			ev_tu = EventSwitchUpdate(switch)
			self.send_event_to_observers(ev_tu)

			self.logger.info("Port modify: " + str(ev.port))
		except KeyError:
			pass

	@set_ev_cls(RyuEventPortDelete, MAIN_DISPATCHER)
	def _handler_port_delete(self, ev):
		"""
		Overwrite the legacy Ryu Event.
		"""
		try:
			switch = self.switches[ev.port.dpid]
			if self._check_if_ap_port(ev.port):
				switch.is_ap = False
				switch.ap_conf = AccessPointConf()
				del self.accesspoints[ev.port.dpid]
			elif self._check_if_gw_port(ev.port):
				switch.is_gw = False
				switch.gw_conf = GatewayConf()
				del self.gateways[ev.port.dpid]

			switch.switch = get_switch(self, ev.port.dpid)[0]

			ev_tu = EventSwitchUpdate(switch)
			self.send_event_to_observers(ev_tu)

			self.logger.info("Port delete: " + str(ev.port))
		except KeyError:
			pass


import torch.utils.data as data
import PIL.Image as Image
import os
import numpy as np
import torch
from torch.utils.data import DataLoader
from torch import autograd, optim
from torchvision.transforms import transforms
from networks.cenet import CE_Net_
import cv2
import nibabel as nib
import time

x_transforms = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize([0.5], [0.5])
])

y_transforms = transforms.ToTensor()

# 是否使用cuda
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
#device = torch.device("cpu")

#liver   'ckp_challenge_only/CE_LITS_and_challenge_model_epoch138.pth'
#tumor   'ckp_challenge_only/CE_LITS_and_challenge_model_epoch142.pth'

def end_to_end(raw_path,liver_model_path,tumor_model_path):

    volume = nib.load(raw_path).get_fdata()
    volume = np.transpose(volume, [2,0,1])

    liver_model = CE_Net_(3, 1).to(device)
    liver_model.load_state_dict(torch.load(liver_model_path))
    liver_model.to(device)
    liver_model.eval()

    liver_target = []

    tumor_model = CE_Net_(3, 1).to(device)
    tumor_model.load_state_dict(torch.load(tumor_model_path))
    tumor_model.to(device)
    tumor_model.eval()

    tumor_target = []

    for i in range(volume.shape[0]):
        a = volume[i]
        img_x = Image.fromarray(np.int16(np.array(volume[i])))
        h,w = img_x.size
        img_x = img_x.resize((512,512))
        img_x = img_x.convert('RGB')
        img_x = x_transforms(img_x)
        img_x = img_x.to(device)
        img_x = torch.unsqueeze(img_x, 0)

        img_liver = liver_model(img_x)
        img_tumor = tumor_model(img_x)

        trann = transforms.ToPILImage()

        img_liver = torch.squeeze(img_liver)
        img_liver = img_liver.detach().cpu().numpy()
        img_liver = np.transpose(img_liver,[1,2,0])
        img_liver[img_liver>=0.5] = 255
        img_liver = img_liver.astype(np.uint8)
        img_liver = trann(img_liver)
        img_liver = img_liver.convert('L')
        img_liver = img_liver.resize((h,w))
        img_liver = np.asarray(img_liver)

        img_tumor = torch.squeeze(img_tumor)
        img_tumor = img_tumor.detach().cpu().numpy()
        img_tumor = np.transpose(img_tumor,[1,2,0])
        img_tumor[img_tumor>=0.5] = 255
        img_tumor = img_tumor.astype(np.uint8)
        img_tumor = trann(img_tumor)
        img_tumor = img_tumor.convert('L')
        img_tumor = img_tumor.resize((h,w))
        img_tumor = np.asarray(img_tumor)

        #肝脏后处理
        img_liver = img_liver.copy()
        img_liver[img_tumor==255] = 255
        kernel = np.ones((7,7),np.uint8)
        img_liver[img_liver>=1] = 1
        img_liver = cv2.erode(img_liver,kernel)#腐蚀
        img_liver = cv2.dilate(img_liver,kernel,iterations = 1)#膨胀
        img_liver[img_liver>=1] = 255
        #肝脏后处理结束

        #肿瘤后处理
        img_tumor = img_tumor.copy()
        img_tumor[img_liver!=255] = 0
        #肿瘤后处理结束

        img_liver[img_liver>=1] = 1
        liver_target.append(img_liver)

        img_tumor[img_tumor>=1] = 1
        tumor_target.append(img_tumor)

    liver_target = np.asarray(liver_target)
    liver_target = np.transpose(liver_target, [1,2,0])
    liver_nib = nib.Nifti1Image(liver_target, affine=np.eye(4))

    tumor_target = np.asarray(tumor_target)
    tumor_target = np.transpose(tumor_target, [1,2,0])
    tumor_nib = nib.Nifti1Image(tumor_target, affine=np.eye(4))

    return liver_nib,tumor_nib

def run():
    liver_model_path = 'ckp_challenge_finally/liver_138.pth'
    tumor_model_path = 'ckp_challenge_finally/tumor_142.pth'

    liver_path = "D:/deep_learning/u_net_liver-test/liver"
    ls = os.listdir(liver_path)

    for i in range(len(ls)):
        nii_path = os.path.join(liver_path,ls[i],"liver.nii")
        liver_nib,tumor_nib = end_to_end(nii_path,liver_model_path,tumor_model_path)

        save_liver = os.path.join(liver_path,ls[i],"liver_seg.nii")
        save_tumor = os.path.join(liver_path,ls[i],"liver_nid.nii")

        nib.save(liver_nib,save_liver)
        nib.save(tumor_nib,save_tumor)

##处理一个病人10-15秒（GPU）
##处理一个病人169秒（CPU）

if __name__ == '__main__':

    time_start = time.time()

    liver_model_path = 'ckp_challenge_finally/liver_138.pth'
    tumor_model_path = 'ckp_challenge_finally/tumor_142.pth'

    nii_path = "liver/liver_1/liver.nii"

    liver_nib,tumor_nib = end_to_end(nii_path,liver_model_path,tumor_model_path)

    nib.save(liver_nib,"liver/liver_1/liver_seg.nii")
    nib.save(tumor_nib,"liver/liver_1/liver_nid.nii")

    time_end  = time.time()
    print(time_end-time_start)


import json

from rest_framework import status

from api.constans import AutoNotificationConstants, TaskStageConstants, \
    CopyFieldConstants
from api.models import *
from api.tests import GigaTurnipTestHelper, to_json

class CategoryTest(GigaTurnipTestHelper):

    def test_list_categories(self):
        products_category = Category.objects.create(
            name="Producs"
        )

        e_commerce_category = Category.objects.create(
            name="E-Commerce"
        )
        e_commerce_category.parents.add(self.category)

        electronics_category = Category.objects.create(
            name="Electronics"
        )
        pcs_category = Category.objects.create(
            name="Personal computers"
        )
        pcs_devices_category = Category.objects.create(
            name="Personal computers attributes."
        )
        pcs_mouses_category = Category.objects.create(
            name="Mouses"
        )

        electronics_category.out_categories.add(pcs_category)
        electronics_category.out_categories.add(pcs_devices_category)
        pcs_devices_category.out_categories.add(pcs_mouses_category)

        answer = [
            {
                'id': self.category.id,
                'name': self.category.name,
                'out_categories': [
                    e_commerce_category.id
                ]},
            {
                'id': e_commerce_category.id,
                'name': e_commerce_category.name,
                'out_categories': []},
            {
                'id': electronics_category.id,
                'name': electronics_category.name,
                'out_categories': [
                    pcs_category.id,
                    pcs_devices_category.id
                ]
            },
            {
                'id': pcs_mouses_category.id,
                'name': pcs_mouses_category.name,
                'out_categories': []},
            {
                'id': pcs_category.id,
                'name': pcs_category.name,
                'out_categories': []
            },
            {
                'id': pcs_devices_category.id,
                'name': pcs_devices_category.name,
                'out_categories': [pcs_mouses_category.id]
            },
            {
                'id': products_category.id,
                'name': products_category.name,
                'out_categories': []
            }
        ]
        response = self.get_objects("category-list")
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        content = to_json(response.content)
        self.assertEqual(content["count"], Category.objects.count())
        self.assertEqual(content["results"], answer)


from os import environ as env
from setuptools import find_packages, setup
from setuptools.command.install import install
import sys

VERSION = "0.1.0"

with open("README.md", "r", encoding="utf-8") as rdm:
    long_description = rdm.read()

class VerifyVersionCommand(install):
    """Custom command to verify that the git tag matches our version"""
    description = 'verify that the git tag matches our version'

    def run(self):
        tag = env .get('CLIPENV_TAG')
        if tag != VERSION:
            info = f"Git tag: {tag} doesn't match with this version: {VERSION}"
            sys.exit(info)

setup(
    name="clipenv",
    version=VERSION,
    description="Attach your venv variables with a clip to your project",
    long_description=long_description,
    author="Ana Valeria Calderón Briz, Daniel Omar Vergara Pérez",
    author_email="valerybriz@gmail.com, daniel.omar.vergara@gmail.com",
    url="https://github.com/dany2691/clipenv",
    license="MIT",
    install_requires=[
        "click",
        "colored"
    ],
    classifiers=[
        # Trove classifiers
        # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: Implementation :: CPython',
        'Programming Language :: Python :: Implementation :: PyPy'
    ],
    packages=find_packages(exclude=('tests',)),
    entry_points={
        "console_scripts": [
            "clipenv=clipenv.__main__:clipenv"
        ]
    },
    cmdclass={'verify': VerifyVersionCommand}
)


# -*- coding: utf-8 -*-

#http://labs.eecs.tottori-u.ac.jp/sd/Member/oyamada/OpenCV/html/py_tutorials/py_imgproc/py_morphological_ops/py_morphological_ops.html

#ROSとOpenCVの競合を避ける
import sys
try:
    py_path = sys.path
    ros_CVpath = '/opt/ros/kinetic/lib/python2.7/dist-packages'
    if py_path[3] == ros_CVpath:
        print("INFO : ROS and OpenCV are competing")
        sys.path.remove(py_path[3])
except: pass

import cv2
import numpy as np
import time
import matplotlib.pyplot as plt

try:
    import scipy.ndimage as ndimage
    from scipy.optimize import curve_fit
except: pass

#+-----[トラックバー]------+#
t_init = False

#+-----[MedianBlur]-------+#
MB = True

#+-----[fill_holes]-------+#
fill_holes = False

#+-------[opening]--------+#
opening = True

#+-------[closing]--------+#
closing = True

#+----[特定色の消去]-------+#
ColorErase = False

#動画ファイルのパス
#Windowsは、コマンドライン引数を使用するときの
#絶対パスの指定が面倒くさいのでコメントアウトしておいてください

#videofile_path = 'test.mp4'
videofile_path = sys.argv[1]

#5x5のカーネル
kernel = np.ones((8,8),np.uint8)
'''
kernel = np.array([[1, 1, 1, 1, 1],
                   [1, 1, 1, 1, 1],
                   [1, 1, 1, 1, 1],
                   [1, 1, 1, 1, 1],
                   [1, 1, 1, 1, 1]])
'''
#トラックバーの初期設定
def callback(x):
    pass

def Trackbars_init():
    cv2.namedWindow('value')
    cv2.createTrackbar('H_Hue','value',179,179,callback)
    cv2.createTrackbar('L_Hue','value',0,179,callback)

    cv2.createTrackbar('H_Saturation','value',255,255,callback)
    cv2.createTrackbar('L_Saturation','value',0,255,callback)

    cv2.createTrackbar('H_Value','value',255,255,callback)
    cv2.createTrackbar('L_Value','value',0,255,callback)

def color_detect(hsv, img):
    hsv_min = np.array([15,127,0])
    hsv_max = np.array([240,255,255])
    mask = cv2.inRange(hsv, hsv_min, hsv_max)

    return mask

#ブロブ解析
def analysis_blob(binary_img):
    label = cv2.connectedComponentsWithStats(binary_img)
    n = label[0] - 1
    data = np.delete(label[2], 0, 0)
    center = np.delete(label[3], 0, 0)
    maxblob = {}
    if len(data[:, 4]) is 0:
        max_index = None
        maxblob["center"] = [0, 0]
    else:
        max_index = np.argmax(data[:, 4])

        maxblob["upper_left"] = (data[:, 0][max_index], data[:, 1][max_index])
        maxblob["width"] = data[:, 2][max_index]
        maxblob["height"] = data[:, 3][max_index]
        maxblob["area"] = data[:, 4][max_index]
        maxblob["center"] = center[max_index]

    return data, center, maxblob

#データ出力
def data_plot(data):
    data_np = np.array(data)
    if len(data_np) <= 0:
        print("too many indices for array")
        f = 0
        x = 0
        y = 0
    else:
        f = data_np[:,0]
        x = data_np[:,1]
        y = data_np[:,2]

    print(f, x, y)

    plt.rcParams["font.family"] = "Times New Roman"
    plt.plot(f, x, "r-", label="x")
    plt.plot(f, y, "b-", label="y")
    plt.xlabel("Frame [num]", fontsize=16)
    plt.ylabel("Position[px]", fontsize=16)
    plt.grid()

    plt.legend(loc=1, fontsize=16)
    plt.show()

#トラックバーからコールバックを受け取り、値を返す
def trackbars():
    lowH = cv2.getTrackbarPos('L_Hue', 'value')
    highH = cv2.getTrackbarPos('H_Hue', 'value')

    lowS = cv2.getTrackbarPos('L_Saturation', 'value')
    highS = cv2.getTrackbarPos('H_Saturation', 'value')

    lowV = cv2.getTrackbarPos('L_Value', 'value')
    highV = cv2.getTrackbarPos('H_Value', 'value')

    return (lowH, lowS, lowV, highH, highS, highV)
#======================================================================
def median_blar(image, size):
    _image = cv2.medianBlur(image, size)
    return _image

def Fill_Holes(image):
    _image = ndimage.binary_fill_holes(image).astype(int) * 255
    return _image

def Opening(image):
    opening = cv2.morphologyEx(image, cv2.MORPH_OPEN, kernel)
    return opening

def Closing(image):
    closing = cv2.morphologyEx(image, cv2.MORPH_CLOSE, kernel)
    return closing

def color_eraser(image, RGBarray):
    pass
#======================================================================

def resize_image(img, dsize, X, Y):
    re_image = cv2.resize(img, dsize, fx=X, fy=Y)
    return re_image

#メイン
def main():
    n = 0
    data = []
    cap = cv2.VideoCapture(videofile_path)

    if t_init is True:
        Trackbars_init()

    if cap.isOpened():
        print("INFO : The Video loaded successfully.")
    else:
        print("INFO : LOAD ERROR ***Chack video path or name.***")
        print("CAP : ", cap)
        exit()

    while(cap.isOpened()):
        ret, frame = cap.read()
        if frame is None:
            print("frame is None")
            break

        #画像の認識範囲を狭めるためのトリミング操作
        """
        h, w = frame.shape[:2]
        frame = frame[80:h, 0:w]
        """

        hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

        #hsv決め打ちの時はココを編集
        if t_init is True:
            Lh, Ls, Lv = trackbars()[:3]
            Hh, Hs, Hv = trackbars()[3:]
        else:
            #青
            Lh, Ls, Lv = (40, 40, 109)
            Hh, Hs, Hv = (121, 255, 255)

        hsv_min = np.array([Lh,Ls,Lv])
        hsv_max = np.array([Hh,Hs,Hv])

        """
        print( "H:{0} - {1}\nS:{2} - {3}\nV:{4} - {5}\n-------------"
            .format(Lh, Hh, Ls, Hs, Lv, Hv))
        """

        mask = cv2.inRange(hsv, hsv_min, hsv_max)

        if MB is True: mask = median_blar(mask, 3)
        #if fill_holes is True: mask = Fill_Holes(mask)
        if opening is True: mask = Opening(mask)
        if closing is True: mask = Closing(mask)
        if ColorErase is True: mask = color_eraser(mask, None)

        _, center, maxblob = analysis_blob(mask)
        #print("target num:",len(center))
        for i in center:
            cv2.circle(frame, (int(i[0]), int(i[1])), 10, (255, 0, 0),
                    thickness=-3, lineType=cv2.LINE_AA)

        center_x = int(maxblob["center"][0])
        center_y = int(maxblob["center"][1])

        print(center_x, center_y)

        cv2.circle(frame, (center_x, center_y), 30, (0, 200, 0),
                  thickness=3, lineType=cv2.LINE_AA)

        data.append([n, center_x, center_y])

        re_frame=resize_image(frame, None, .4, .4)
        cv2.imshow("Frame", re_frame)
        mask = resize_image(mask, None, .4, .4)
        cv2.imshow("image_mask", mask)
        n += 1
        #print(n)
        print("----------")

        if cv2.waitKey(1000) & 0xFF == ord('q'):
            break

    cap.release()
    cv2.destroyAllWindows()

    data_plot(data)

if __name__ == '__main__':
    main()


import click
from screeps_loan import app
import screepsapi.screepsapi as screepsapi
from screeps_loan.models.db import get_conn
from screeps_loan.screeps_client import get_client
from screeps_loan.models import db
from screeps_loan.services.cache import cache

import screeps_loan.models.alliances as alliances
import screeps_loan.models.users as users

from random import shuffle, random
from time import sleep
import math

POWER_POW = 1.15
POWER_MULTIPLY = 1000
POWER_MAX = 750

powertotals = [{'level': 0, 'total': 0}]
powerlevels = {}
total = 0

for i in range(0, POWER_MAX):
    needed = math.pow(i, POWER_POW) * POWER_MULTIPLY
    total += needed
    powertotals.append({
        'level': i,
        'total': total
    })
    powerlevels[i] = total

powertotals = list(reversed(powertotals))

@cache.cache('getUserControlPoints')
def getUserControlPoints(username):
    user_info = getUserInfo(username)
    if not user_info:
        return 1
    if 'user' in user_info:
        if 'gcl' in user_info['user']:
            return user_info['user']['gcl']
    return 1

@cache.cache('getUserPowerPoints')
def getUserPowerPoints(username):
    user_info = getUserInfo(username)
    if not user_info:
        return 1
    if 'user' in user_info:
        if 'power' in user_info['user']:
            return user_info['user']['power']
    return 0

@cache.cache('getUserInfo')
def getUserInfo(username):
    screeps = get_client()
    sleep(1)
    return screeps.user_find(username)

class Rankings(object):
    gclcache = {}

    def run(self):

        alliance_query = alliances.AllianceQuery()
        all_alliances = alliance_query.getAll()
        alliances_names = [item["shortname"] for item in all_alliances]
        users_with_alliance = users.UserQuery().find_name_by_alliances(alliances_names)

        query = "SELECT id FROM room_imports WHERE status LIKE 'complete' ORDER BY started_at DESC"
        result = db.find_one(query)
        self.room_import_id = result[0]

        self.conn = get_conn()
        self.start()
        print(self.id)

        for alliance in all_alliances:
            users_with_alliance = self.find_name_by_alliances(alliances_names)
            members = [user['name'] for user in users_with_alliance if user['alliance'] == alliance['shortname']]
            filtered_members = [user for user in members if self.get_player_room_count(user) > 0]

            # Not enough members.
            if len(filtered_members) < 2:
                continue

            # Not enough rooms
            if self.get_room_count(alliance['shortname']) < 2:
                continue

            rcl = self.getAllianceRCL(alliance['shortname'])

            combined_gcl = sum(self.getUserGCL(user) for user in filtered_members)
            control = sum(getUserControlPoints(user) for user in filtered_members)
            alliance_gcl = self.convertGcl(control)

            combined_power = sum(self.getUserPowerLevel(user) for user in filtered_members)
            power = sum(getUserPowerPoints(user) for user in filtered_members)
            alliance_power = self.convertPowerToLevel(power)

            spawns = self.getAllianceSpawns(alliance['shortname'])
            print('%s- %s, %s, %s, %s, %s, %s, %s' % (alliance['shortname'], combined_gcl, alliance_gcl, rcl, spawns, len(filtered_members), alliance_power, combined_power))

            self.update(alliance['shortname'], alliance_gcl, combined_gcl, rcl, spawns, len(filtered_members), alliance_power, combined_power)

        self.finish()
        self.conn.commit()

    def start(self):
        query = "INSERT INTO rankings_imports(status) VALUES ('in progress') RETURNING id"
        cursor = self.conn.cursor()
        cursor.execute(query)
        self.id = cursor.fetchone()[0]

    def finish(self):
        query = "UPDATE rankings_imports SET status='complete' WHERE id=(%s)"
        cursor = self.conn.cursor()
        cursor.execute(query, (self.id, ))

    def update(self, alliance, alliance_gcl, combined_gcl, rcl, spawns, members, alliance_power, combined_power):
        # Store info in db
        cursor = self.conn.cursor()
        query = "INSERT INTO rankings(import, alliance, alliance_gcl, combined_gcl, rcl, spawns, members, alliance_power, combined_power) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s)"
        cursor.execute(query, (self.id, alliance, alliance_gcl, combined_gcl, rcl, spawns, members, alliance_power, combined_power))

    def getAllianceRCL(self, alliance):
        query = "SELECT SUM(level) FROM rooms, users WHERE rooms.owner = users.id AND users.alliance=%s AND rooms.import=%s"
        cursor = self.conn.cursor()
        cursor.execute(query, (alliance, self.room_import_id))
        result = cursor.fetchone()[0]
        if result is not None:
            return result
        return 0

    def getAllianceSpawns(self, alliance):
        count = 0
        query = "SELECT COUNT(*) FROM rooms, users WHERE rooms.owner = users.id AND users.alliance=%s AND level>=8 AND rooms.import=%s"
        cursor = self.conn.cursor()
        cursor.execute(query, (alliance, self.room_import_id))
        result = cursor.fetchone()[0]
        if result is not None:
            if result:
                count += result*3

        query = "SELECT COUNT(*) FROM rooms, users WHERE rooms.owner = users.id AND users.alliance=%s AND level=7 AND rooms.import=%s"
        cursor = self.conn.cursor()
        cursor.execute(query, (alliance, self.room_import_id))
        result = cursor.fetchone()[0]
        if result is not None:
            if result:
                count += result*2

        query = "SELECT COUNT(*) FROM rooms, users WHERE rooms.owner = users.id AND users.alliance=%s AND level>=1 AND level<7 AND rooms.import=%s"
        cursor = self.conn.cursor()
        cursor.execute(query, (alliance, self.room_import_id))
        result = cursor.fetchone()[0]
        if result is not None:
            if result:
                count += result

        return count

    def convertGcl(self, control):
        return int((control/1000000) ** (1/2.4))+1

    def getUserGCL(self, username):
        return self.convertGcl(getUserControlPoints(username))

    def convertPowerToLevel(self, power):
        if power <= 0:
            return 0
        for powerdata in powertotals:
            if powerdata['total'] < power:
                return powerdata['level']
        return 0

    def getUserPowerLevel(self, username):
        return self.convertPowerToLevel(getUserPowerPoints(username))

    def find_name_by_alliances(self, alliance):
        query = "SELECT ign, alliance FROM users where alliance = ANY(%s)"
        cursor = self.conn.cursor()
        cursor.execute(query, (alliance,))
        result = cursor.fetchall()
        return [{"name": row[0], "alliance": row[1]} for row in result]

    def get_room_count(self, alliance):
        query = '''
        SELECT COUNT(DISTINCT rooms.name)
            FROM rooms,users
            WHERE rooms.owner=users.id
                AND users.alliance=%s
                AND rooms.import = (SELECT id
                                        FROM room_imports
                                        ORDER BY id desc
                                        LIMIT 1
                                    );
        '''
        cursor = self.conn.cursor()
        cursor.execute(query, (alliance,))
        result = cursor.fetchone()
        return int(result[0])

    def get_player_room_count(self, player):
        query = '''
        SELECT COUNT(DISTINCT rooms.name)
              FROM rooms,users
              WHERE rooms.owner=users.id
                  AND users.ign=%s
                  AND rooms.import = (SELECT id
                                          FROM room_imports
                                          ORDER BY id desc
                                          LIMIT 1
                                      );
        '''
        cursor = self.conn.cursor()
        cursor.execute(query, (player,))
        result = cursor.fetchone()
        return int(result[0])

@app.cli.command()
def import_rankings():
    click.echo("Generating Rankings")
    r = Rankings()
    r.run()

@app.cli.command()
def import_user_rankings():
    click.echo("Generating User Rankings")
    dbusers = users.get_all_users_for_importing()
    for dbuser in dbusers:
        # Only retrieve information if we don't have any or the player has some active rooms.
        if not dbuser['gcl'] or users.get_player_room_count(dbuser['ign']) > 0 or random() < 0.05:
            gcl = getUserControlPoints(dbuser['ign'])
            power = getUserPowerPoints(dbuser['ign'])
            print('%s has %s gcl and %s power' % (dbuser['ign'], gcl, power))
            users.update_gcl_by_user_id(dbuser['id'], getUserControlPoints(dbuser['ign']))
            users.update_power_by_user_id(dbuser['id'], getUserPowerPoints(dbuser['ign']))
            sleep(1.5)
        else:
            print('Skipping user %s' % (dbuser['ign']))


# imports
import datetime as dt
import os
import os.path as osp
import time

import numpy as np

from .global_imports.smmpl_opcodes import *
from .quickscanpat_calc import quickscanpat_calc
from .sop import sigmampl_boot

# params
_nodoubleinit_l = [
    'suncone'
]

# main func
def main(quickscantype=None, **quickscanparams):
    '''
    quick scan pattern parameters are adjusted in their respective scripts,
    i.e. quickscanpat_calc.<quickscantype>

    But quick scanpattern type, bin resolution and shot averaging time are
    controlled in .params
    '''
    if not quickscantype:
        quickscantype = QUICKSCANTYPE

    # setting log file
    now = dt.datetime.now()
    mainlog = DIRCONFN(
        MPLDATADIR, DATEFMT.format(now), QUICKSCANLOG.format(now)
    )
    SETLOGFN(mainlog)
    today = dt.datetime.combine(dt.date.today(), dt.time())
    mainlognext_dt = today + dt.timedelta(1)  # start a new log the next day

    # determining whether to perform double initialisation
    if quickscantype in _nodoubleinit_l:
        coldstart_boo = False
    else:
        coldstart_boo = True

    # begin operation
    for i in range(QUICKSCANTIMES):
        print(f'starting {quickscantype} quickscan {i}...')

        # updating logfile
        now = dt.datetime.now()

        if now >= mainlognext_dt:
            mainlog = DIRCONFN(
                MPLDATADIR, DATEFMT.format(now), QUICKSCANLOG.format(now)
            )
            SETLOGFN(mainlog)
            mainlognext_dt += dt.timedelta(1)

        # calculating scan pattern
        scanpat_a = quickscanpat_calc(quickscantype, **quickscanparams)

        # writing scan pattern to file
        now = dt.datetime.now()
        scanpatpar_dir = DIRCONFN(
            MPLDATADIR, DATEFMT.format(now)
        )
        if not osp.isdir(scanpatpar_dir):
            os.mkdir(scanpatpar_dir)
        scanpat_dir = DIRCONFN(
            scanpatpar_dir,
            QUICKSCANFILE.format(now, quickscantype)
        )
        print(f'writing quick scanpattern to: {scanpat_dir}')
        np.savetxt(scanpat_dir, scanpat_a,
                   fmt='%.2f', delimiter=', ', newline='\n\n')

        # beginning init and measurement
        sigmampl_boot(
            coldstart_boo=coldstart_boo, scanpat_dir=scanpat_dir,
            stdoutlog=mainlog
        )
        coldstart_boo = False

        # waiting between next measurement
        print(f'waittime: {QUICKSCANWAITTIME}min, next measurement time:'
              f'{dt.datetime.now() + dt.timedelta(minutes=QUICKSCANWAITTIME)}\n')
        time.sleep(QUICKSCANWAITTIME*60)

# testing
if __name__ == '__main__':
    main()


import os
import torch
import numpy as np
import matplotlib.pyplot as plt
from sklearn.manifold import TSNE
from torchvision import transforms
from PIL import Image
from typing import List
from tqdm import tqdm
from two_tower_model.tower import TwoTowerModel
from two_tower_model.selectivesearch import selective_search, show_bbox

def adaptive_resize(batchOfImg: List[torch.Tensor]):
    img_sizes = [[image.shape[-2], image.shape[-1]] for image in batchOfImg]
    max_size = max([max(img_size[0], img_size[1]) for img_size in img_sizes])  # the maximal size
    batch_shape = (len(batchOfImg), batchOfImg[0].shape[0], max_size, max_size)  # wanted shape

    padded_images = batchOfImg[0].new_full(batch_shape, 0.0)
    for padded_img, img in zip(padded_images, batchOfImg):
        h, w = img.shape[1:]
        padded_img[..., :h, :w].copy_(img)

    return padded_images

def resized_img(in_dirs: List[str], out_dir, size):
    # resize images to size (256, 256)
    transform = transforms.Compose([transforms.Resize(size=size)])
    for in_dir in in_dirs:
        files = os.listdir(in_dir)
        bar = tqdm(files)
        bar.set_description(in_dir)
        for file in bar:
            img = Image.open(os.path.join(in_dir, file)).convert('RGB')
            img = transform(img)
            img.save(os.path.join(out_dir, file))

def random_crop(in_dirs: List[str], out_dir, repeat=1):
    transform = transforms.Compose([transforms.Resize(size=(1280, 1280)),
                                    transforms.RandomCrop(size=(256, 256))])
    for in_dir in in_dirs:
        files = os.listdir(in_dir)
        bar = tqdm(files)
        bar.set_description(in_dir)
        for file in bar:
            img = Image.open(os.path.join(in_dir, file)).convert('RGB')
            for i in range(repeat):
                img2 = transform(img)
                img2.save(os.path.join(out_dir, file.replace('.', '-{}.'.format(i + 1))))

def tSNE_visualize(gpu_accelerate=True):
    device = torch.device('cuda:0' if gpu_accelerate and torch.cuda.is_available() else 'cpu')
    model = TwoTowerModel(model_dir=None).to(device)
    transform = transforms.Compose([transforms.Resize(size=(256, 256)),
                                    transforms.ToTensor()])
    model.load_state_dict(torch.load(r'D:\Project\PyCharmProjects\WFGN\weight\tower.pth'))
    pos_dir = r'D:\Training Dataset\FurGenTMP\positive'
    neg_dir = r'D:\Training Dataset\FurGenTMP\negative'

    embeddings = []
    pos_count, neg_count = 0, 0
    with torch.no_grad():
        for img in tqdm(os.listdir(pos_dir)):
            image_tensor = transform(Image.open(os.path.join(pos_dir, img)).convert('RGB')).unsqueeze(dim=0).to(device)
            embedding = model(image_tensor).squeeze(dim=0)
            embeddings.append(embedding.cpu().numpy())
            pos_count += 1

        positive_mean = torch.tensor(np.array(embeddings))
        positive_mean = torch.mean(positive_mean, dim=0)
        print(positive_mean.shape)
        torch.save(positive_mean, r'D:\Project\PyCharmProjects\WFGN\weight\positive_mean.pth')

        for img in tqdm(os.listdir(neg_dir)):
            image_tensor = transform(Image.open(os.path.join(neg_dir, img)).convert('RGB')).unsqueeze(dim=0).to(device)
            embedding = model(image_tensor).squeeze(dim=0)
            embeddings.append(embedding.cpu().numpy())
            neg_count += 1

    tsne = TSNE(n_components=2)
    xy = tsne.fit_transform(embeddings)
    x = xy[:, 0]
    y = xy[:, 1]
    c = [0 for _ in range(pos_count)] + [1 for _ in range(neg_count)]
    plt.scatter(x, y, s=1, c=c)
    plt.show()

if __name__ == '__main__':
    '''
    resized_img(in_dirs=[r'D:\Training Dataset\FurGenTMP\新建文件夹'],
                out_dir=r'D:\Training Dataset\FurGenTMP', size=1200)
    '''
    '''
    random_crop(in_dirs=['D:\\Training Dataset\\FurGen\\2'],
                out_dir='D:\\Training Dataset\\FurGenTMP\\2333', repeat=2)
    '''
    # show_bbox(Image.open(r'D:\Training Dataset\FurGenTMP\新建文件夹\yao.jpg').convert('RGB'))


from __future__ import print_function
import argparse
import random
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable
import os
import json
from tqdm import tqdm

import models.dcgan as dcgan
import models.mlp as mlp

def generate_samples(args):

    set_seed = 42
    random.seed(set_seed)
    torch.manual_seed(set_seed)
    torch.cuda.manual_seed(set_seed)
    torch.cuda.manual_seed_all(set_seed)

    #with open(args.params_path, 'r') as gencfg:
    #    generator_config = json.loads(gencfg.read())

    generator_config = vars(args)
    imageSize = generator_config["imageSize"]
    nz = generator_config["nz"]
    nc = generator_config["nc"]
    ngf = generator_config["ngf"]
    noBN = generator_config["noBN"]
    ngpu = generator_config["ngpu"]
    mlp_G = generator_config["mlp_G"]
    n_extra_layers = generator_config["n_extra_layers"]
    invert_p = generator_config["invert_p"]

    if noBN:
        netG = dcgan.DCGAN_G_nobn(imageSize, nz, nc, ngf, ngpu, n_extra_layers)
    elif mlp_G:
        netG = mlp.MLP_G(imageSize, nz, nc, ngf, ngpu)
    else:
        netG = dcgan.DCGAN_G(imageSize, nz, nc, ngf, ngpu, n_extra_layers)

    # initialize noise
    fixed_noise = torch.FloatTensor(args.num_samples, nz, 1, 1).normal_(0, 1)

    if args.cuda:
        netG.load_state_dict(torch.load(args.weights_path)['netG_state_dict'])
        netG.cuda()
        fixed_noise = fixed_noise.cuda()
    else:
        netG.load_state_dict(torch.load(args.weights_path, map_location=torch.device('cpu'))['netG_state_dict'])

    fake = netG(fixed_noise)
    fake.data = fake.data.mul(0.5).add(0.5)

    folder_name = args.weights_path.parent.name
    save_dir = str(args.output_dir / f"wgan_{folder_name}")
    print(f"Saving on {save_dir}")

    if not os.path.exists(save_dir):
        os.makedirs(save_dir)

    for i in tqdm(range(args.num_samples)):
        vutils.save_image(fake.data[i, ...].reshape((1, nc, imageSize, imageSize)), os.path.join(save_dir, "generated_%02d.png"%i))

if __name__=="__main__":

    parser = argparse.ArgumentParser()
    #parser.add_argument('-c', '--params_path', required=True, type=str, help='path to generator config .json file')
    #parser.add_argument('-w', '--weights_path', required=True, type=str, help='path to generator weights .pth file')
    #parser.add_argument('-o', '--output_dir', required=True, type=str, help="path to to output directory")
    parser.add_argument('-n', '--num_samples', type=int, help="number of images to generate", default=1)
    parser.add_argument('--cuda', action='store_true', help='enables cuda')
    args = parser.parse_args()

    generate_samples(args)


#!/usr/bin/env python
# coding=utf-8

"""
TBW
"""

from __future__ import absolute_import
from __future__ import print_function
from collections import MutableSequence
import io
import os
import random

from ipapy.compatibility import to_str
from ipapy.compatibility import to_unicode_string

__author__ = "Alberto Pettarin"
__copyright__ = "Copyright 2016, Alberto Pettarin (www.albertopettarin.it)"
__license__ = "MIT"
__email__ = "alberto@albertopettarin.it"

class LexiconEntry(object):
    """
    TBW
    """

    def __init__(self, raw_values, lowercase=False):
        self.key = None
        self.value = None
        self.valid = False
        self._set_key(raw_values=raw_values, lowercase=lowercase)
        self._set_value(raw_values=raw_values, lowercase=lowercase)

    def _set_key(self, raw_values, lowercase):
        if len(raw_values) >= 1:
            self.key = raw_values[0]
            if lowercase:
                self.key = self.key.lower()

    def _set_value(self, raw_values, lowercase):
        if len(raw_values) >= 2:
            self.valid = True
            self.value = raw_values[1]

class Lexicon(MutableSequence):
    """
    TBW
    """

    ENTRY_TYPE = LexiconEntry

    def __init__(self, entries=None, lowercase=False):
        self.__ordered_dict = None
        self.entries = [] if entries is None else entries
        self.lowercase = lowercase
        self.train_lexicon = None
        self.test_lexicon = None
        self._update_ordered_dict()

    def _update_ordered_dict(self):
        self.__ordered_dict = dict()
        for i, e in enumerate(self.entries):
            self._add_entry_to_ordered_dict(i, e)

    def _add_entry_to_ordered_dict(self, i, e):
        k = e.key
        try:
            self.__ordered_dict[k].append(i)
        except KeyError:
            self.__ordered_dict[k] = [i]

    def _remove_entry_from_ordered_dict(self, i):
        k = self[i].key
        l = self.__ordered_dict[k]
        del l[l.index(k)]
        if len(l) == 0:
            del self.__ordered_dict[k]

    def __str__(self):
        return to_str(u"".join([e.__str__() for e in self.entries]))

    def __unicode__(self):
        return u"".join([e.__unicode__() for e in self.entries])

    def __repr__(self):
        return u"\n".join([e.__repr__() for e in self.entries])

    def __iter__(self):
        for e in self.entries:
            yield e

    def __len__(self):
        return len(self.entries)

    def __getitem__(self, i):
        return self.entries[i]

    def __delitem__(self, i):
        self._remove_entry_from_ordered_dict(i)
        del self.entries[i]

    def _check(self, value):
        if not isinstance(value, self.ENTRY_TYPE):
            raise TypeError(u"Objects stored in this lexicon must have type '%s'. (Got: '%s')" % (self.ENTRY_TYPE, type(value)))

    def __setitem__(self, i, value):
        self._check(value)
        self._remove_entry_from_ordered_dict(i)
        self.entries[i] = value
        self._add_entry_to_ordered_dict(i, value)

    def insert(self, i, value):
        self._check(value)
        self.entries.insert(i, value)
        self._update_ordered_dict()

    @property
    def keys(self):
        return [e.key for e in self]

    @property
    def unique_keys(self):
        return list(self.__ordered_dict.keys())

    @property
    def has_unique_entries(self):
        return len(self) == len(self.unique_keys)

    def entries_for_key(self, key):
        try:
            return [self[i] for i in self.__ordered_dict[key]]
        except:
            return []

    def read_file(self, lexicon_file_path, comment=u"#", delimiter=u"\t", indices=[0, 1]):
        if (lexicon_file_path is None) or (not os.path.isfile(lexicon_file_path)):
            raise ValueError("The lexicon file path must exist. (Got '%s')" % lexicon_file_path)
        comment = to_unicode_string(comment)
        delimiter = to_unicode_string(delimiter)
        with io.open(lexicon_file_path, "r", encoding="utf-8") as lexicon_file:
            for line in lexicon_file:
                line = line.strip()
                if (comment is not None) and line.startswith(comment):
                    # commented line, skip
                    pass
                else:
                    acc = line.split(delimiter)
                    if len(acc) > 0:
                        self.entries.append(
                            self.ENTRY_TYPE(raw_values=[acc[i] for i in indices], lowercase=self.lowercase)
                        )
        self._update_ordered_dict()

    def shuffle(self):
        copy = list(self.entries)
        random.shuffle(copy)
        cls = type(self)
        return cls(entry_type=self.ENTRY_TYPE, entries=copy, lowercase=self.lowercase)

    def shuffle_and_partition(self, size=0.9, store=False):
        copy = list(self.entries)
        random.shuffle(copy)
        if isinstance(size, int) and size > len(copy):
            raise ValueError(u"The given size (%d) exceeds the number of entries (%d)." % (size, len(copy)))
        if isinstance(size, float):
            if (size < 0.0) or (size > 1.0):
                raise ValueError(u"The size, when expressed as a fraction, must be in [0.0, 1.0]. (Got: '%.3f')" % size)
            size = int(len(copy) * size)
        cls = type(self)
        ret = (
            cls(entries=copy[0:size], lowercase=self.lowercase),
            cls(entries=copy[size:], lowercase=self.lowercase),
        )
        if not store:
            return ret
        self.train_lexicon, self.test_lexicon = ret