import collections
import math
import pickle
import string
import sys

# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
# Make a class of inverted index in which all the process is present to find inverted index
class InvertedIndex():
    # This is an constructor which will start fetch the document word and finally will make an inverted index
    def __init__(self):
        self.exclude = list(string.punctuation)
        self.exclude.append('।')

    def make_index(self, input_file, index_filename='index.txt'):
        # open file to write
        self.index_file = open(index_filename, 'w')
        # self.indexDict is an dictionary to store the posting list
        self.indexDict = {}
        with open(input_file, 'r') as corpus:
            sentences = corpus.readlines()
            self.N = len(sentences)
            for i in range(0, len(sentences)):
                text = sentences[i]
                # call the first operation(Remove punctuation from the document) and update the value of text variable
                text = self.remove_punctuation(text)
                # call the second operation(Remove stop word from the document) and store it as array of words
                words = self.remove_stop_word(text)
                # call the operation to apply stemming on word and store it with their document name in posting list
                self.make_PostingList(words, i)
        for i in self.indexDict.keys():
            self.index_file.write(i + "\t\t\t")
            self.index_file.write(str(self.indexDict.get(i)) + "\n")
        self.index_file.close()
        return self.indexDict

    # An operation to apply stemming and store word with their document id and frequency
    def make_PostingList(self, words, doc_id):
        # appy stemming on each word and store in lsit(word)
        words = [self.stem_word(word) for word in words]
        unique_words_frequency_dict = dict(collections.Counter(words))
        for unique_word in unique_words_frequency_dict.keys():
            # if word was present in another document then
            # append the current document at that place otherwise make another row and store it there

            if str(unique_word) in self.indexDict.keys():
                self.indexDict[str(unique_word)][doc_id] = unique_words_frequency_dict[unique_word]
            else:
                self.indexDict[str(unique_word)] = {}
                self.indexDict[str(unique_word)][doc_id] = unique_words_frequency_dict[unique_word]

# An operation to remove stop word
    def remove_stop_word(self, text):
        # print("removing stop word")
        # listOut all the stop word in stopWord list
        stopWord = open("stopWord.txt", "r").read().split("\n")
        text3 = []
        # print(stopWord)
        text = text.split()
        # make condition if text are stop word then it will remove
        for c in text:
            if c not in stopWord:
                text3.append(c)
        return text3

    # An operation to remove punctuation
    def remove_punctuation(self, text):
        text = "".join(c for c in text if c not in self.exclude)
        return text

    # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
    # An operation to find the root word by doing stemming
    # (Taken from Github)
    def stem_word(self, word):
        # suffixes have all the suffixes of hindi word which mostly comes at the end of word
        suffixes = {
            1: ["ो", "े", "ू", "ु", "ी", "ि", "ा"],
            2: ["कर", "ाओ", "िए", "ाई", "ाए", "ने", "नी", "ना", "ते", "ीं", "ती", "ता", "ाँ", "ां", "ों", "ें"],
            3: ["ाकर", "ाइए", "ाईं", "ाया", "ेगी", "ेगा", "ोगी", "ोगे", "ाने", "ाना", "ाते", "ाती", "ाता", "तीं", "ाओं",
                "ाएं", "ुओं", "ुएं", "ुआं"],
            4: ["ाएगी", "ाएगा", "ाओगी", "ाओगे", "एंगी", "ेंगी", "एंगे", "ेंगे", "ूंगी", "ूंगा", "ातीं", "नाओं", "नाएं",
                "ताओं", "ताएं", "ियाँ", "ियों", "ियां"],
            5: ["ाएंगी", "ाएंगे", "ाऊंगी", "ाऊंगा", "ाइयाँ", "ाइयों", "ाइयां"],
        }

        # make condition if word will end with these suffixes then that part will remove
        for k in 5, 4, 3, 2, 1:
            if len(word) > k:
                # print('h')
                for s in suffixes[k]:
                    if word.endswith(s):
                        # print(sf)
                        return (word[:-k])

        return word

    def search(self, query, indexDict, N):
        def getkey(item):
            return temp_calc[item]
        query = self.remove_punctuation(query)
        query = self.remove_stop_word(query)
        # query1 = list(set(query))
        temp_calc = {}
        for q_term in query:
            if q_term in indexDict:
                idf = math.log(N / len(indexDict[q_term].keys()))
                for doc_id in indexDict[q_term].keys():
                    term_frequency = indexDict[q_term][doc_id]
                    if doc_id in temp_calc:
                        temp_calc[doc_id] += term_frequency * idf
                    else:
                        temp_calc[doc_id] = term_frequency * idf
        sorted_result = [(doc, temp_calc[doc]) for doc in sorted(temp_calc, key=getkey, reverse=True)]
        return sorted_result

if __name__ == "__main__":
    # finally call all the operation by making the object of the class
    import os

    # obj = invertedIndex()
    # obj.make_index('hindi.corpus')
    # # finally dump the posting list in wordIndex.p in binary format
    # pickle.dump(obj.indexDict, open("hindi_corpus_index.pickle", "wb"))
    if not os.path.isfile("hindi_corpus_index.pickle") or len(sys.argv) == 2:
        obj = InvertedIndex()
        # finally dump the posting list in wordIndex.p in binary format
        hindi_corpus_file_name = sys.argv[1]
        pickle.dump(obj.make_index('corpus.hi'), open("hindi_corpus_index.pickle", "wb"))
    indexDict = {}
    indexDict = pickle.load(open(hindi_corpus_file_name, "rb"))
    print(len(indexDict))
    obj = InvertedIndex()
    query = input("enter Your Query\n")
    print(search(query, indexDict, N=30010.0))


import matplotlib.pyplot as plt

days=[1,2,3,4,5]

sleeping=[7,8,9,12,6]
eating=[2,3,4,3,2]
working=[7,8,7,2,2]
playing=[8,5,7,8,13]

slices = [7,2,2,13]
activities = ['Eating','Working','Playing','Sleeping']
colors= ['c','m','r','m']

plt.pie(
        slices,
        labels=activities,
        colors=colors,
        startangle=90
        ,shadow=True,
        explode=(0,0.1,0,0),
        autopct='%1.1f%%'
        )

plt.xlabel('Plot Number')
plt.ylabel('Important var')
plt.title('Interesting Graph')
#plt.legend()
plt.show()

Machine learning/sentdex course/Data Visualization/piechart.py


import matplotlib.pyplot as plt
import csv

# Read data from the CSV file
data = {"mass": [], "sigma_nominal": [], "s_n_error": [], "JER_sigma_up": [], "s_up_error": [], "JER_sigma_down": [], "s_d_error": []}

with open("JER_effect.csv") as csvfile:
    reader = csv.DictReader(csvfile)
    for row in reader:
        data["mass"].append(float(row["mass"]))
        data["sigma_nominal"].append(float(row["sigma_nominal"]))
        data["s_n_error"].append(float(row["s_n_error"]))
        data["JER_sigma_up"].append(float(row["JER_sigma_up"]))
        data["s_up_error"].append(float(row["s_up_error"]))
        data["JER_sigma_down"].append(float(row["JER_sigma_down"]))
        data["s_d_error"].append(float(row["s_d_error"]))

# Create the plot
plt.figure(figsize=(10, 6))
plt.errorbar(data["mass"], data["sigma_nominal"], yerr=data["s_n_error"], fmt="o", color="black", label="sigma_nominal")
plt.errorbar(data["mass"], data["JER_sigma_up"], yerr=data["s_up_error"], fmt="o", color="blue", label="JER_sigma_up")
plt.errorbar(data["mass"], data["JER_sigma_down"], yerr=data["s_d_error"], fmt="o", color="red", label="JER_sigma_down")

# Add labels and legend
plt.xlabel("mass")
plt.ylabel("Value")
plt.title("JER Sigma Shift")
plt.legend()

# Save the plot as PNG and PDF
plt.savefig("JER_sigma_shift.png")
plt.savefig("JER_sigma_shift.pdf")

# Show the plot (optional)
plt.show()


#!/usr/bin/env python

import numpy as np

import rospy
from geometry_msgs.msg import PoseArray, Pose, Quaternion
from tf.transformations import quaternion_from_euler

from common import *

if __name__ == '__main__':

  rospy.init_node('waypoint_publisher')

  # Generate random waypoints
  waypoints = np.random.rand(num_waypoints, 3)
  waypoints[:, 0] = (x_lim[1] - x_lim[0]) * waypoints[:, 0] + x_lim[0]
  waypoints[:, 1] = (y_lim[1] - y_lim[0]) * waypoints[:, 1] + y_lim[0]
  waypoints[:, 2] = (theta_lim[1] - theta_lim[0]) * waypoints[:, 2] + theta_lim[0]

  # Waypoint publisher
  waypoints_pub = rospy.Publisher('waypoints', PoseArray, queue_size=10)

  waypoint_array = PoseArray()
  for i in range(0, num_waypoints):
    w = Pose()
    w.position.x = waypoints[i, 0]
    w.position.y = waypoints[i, 1]
    w.orientation = Quaternion(*quaternion_from_euler(0, 0, waypoints[i, 2]))

    waypoint_array.poses.append(w)

  # Publish the waypoints
  r = rospy.Rate(1)  # 10hz
  while not rospy.is_shutdown():
    waypoints_pub.publish(waypoint_array)
    r.sleep()


from collections import defaultdict

class Solution:
    def containsNearbyDuplicate(self, nums: List[int], k: int) -> bool:
        dic = defaultdict()
        for i in range(len(nums)):
            cur = nums[i]
            if cur in dic and i - dic[cur] <= k:
                return True
            dic[cur] = i
        return False


# coding=utf-8
import math
import functools

# 场景
class Scene(object):

	def __init__(self):
		self.__infos = [] # 缓冲的行情(KLine)
		self.__ind = {} # 绑定的指标(name : BaseInd)
		self.__strategy = [] # 绑定的策略([prio(int), impl])

	def update(self, new_info):

		# 更新行情
		self.__infos.append(new_info)

		# 更新绑定的指标
		for k,v in self.__ind.iteritems():
			v.update(self)

		# 更新策略
		for i in range(len(self.__strategy)):
			strategy = self.__strategy[i]
			strategy[1](self, strategy[2])

	def addIndicator(self, name, new_ind):
		self.__ind[name] = new_ind

	def bindStrategy(self, prio=0, ctx=None):
		def wrapper_arg(func):
			@functools.wraps(func)
			def wrapper(*args, **kwargs):
				result = func(*args, **kwargs)
				return result
			new_item = [prio, wrapper, ctx]
			self.__strategy.append(new_item)
			sorted(self.__strategy, key=lambda item:item[0])
			return wrapper
		return wrapper_arg

	def indicator(self, name):
		return self.__ind[name]

	def __len__(self):
		return len(self.__infos)

	def __getitem__(self, key):
		if not isinstance(key, int):
			raise KeyError
		return self.__infos[key]

	@property
	def last(self):
		if len(self)>0:
			return self[-1]
		return None

	def __str__(self):
		ret = ''
		line = '\r\n'

		ret += '行情数据:'+line
		for i in range(len(self)):
			ret += str(self[i])+line
		ret += line

		for k,v in self.__ind.iteritems():
			ret += '指标%s数据:'%(k)+line
			ret += str(v)
			ret += line

		return ret

# 指标接口
class BaseInd(object):

	def __init__(self):
		pass

	def update(self, scene):
		pass

# MA指标(Moving Averagr)
class MA(BaseInd):
	def __init__(self, interval=10):
		super(MA, self).__init__()
		self.__interval = interval
		self.__infos = [] # [0:时间,1:ma数据]

	@property
	def interval(self):
		return self.__interval

	def __len__(self):
		return len(self.__infos)

	def __getitem__(self, key):
		if not isinstance(key, int):
			raise KeyError
		return self.__infos[key]

	@property
	def last(self):
		if len(self.__infos)<=0:
			return None
		item = self[-1]
		return (item[0], item[1])

	def update(self, scene):
		if len(scene)<self.interval:
			return
		sum = 0.0
		for i in range(self.interval):
			sum += scene[-i-1].close
		sum /= float(self.interval)
		new_item = [scene[-1].time, sum]
		self.__infos.append(new_item)

	def __str__(self):
		ret = ''
		for i in range(len(self)):
			item = self[i]
			ret += '%s %.2f\r\n' % (str(item[0]), item[1])
		return ret

# DM指标
class DM(BaseInd):
	'''
	DM : 方向性移动运动指标(Directional Move Indicator),分为"正方向移动运动指标(DM+)"与"负方向移动运动指标(DM-)".
	最高价变化 : 当前最高价-先前最高价;
	最低价变化 : 先前最低价-当前最低价.

	若"最高价/最低价变化都<0"或“最高价/最低价变化相等”,则DM+与DM-都为0;
	若"最高价变化>最低价变化",则DM+为最高价变化,DM-为0;
	若"最高价变化<最低价变化",则DM+为0,DM-为最低价变化.
	'''
	def __init__(self):
		super(DM, self).__init__()
		self.__infos = [] # [0:时间,1:DM+数据,2:DM-数据]

	def __len__(self):
		return len(self.__infos)

	def __getitem__(self, key):
		if not isinstance(key, int):
			raise KeyError
		return self.__infos[key]

	@property
	def last(self):
		if len(self.__infos)<=0:
			return None
		item = self[-1]
		return (item[0], item[1], item[2])

	def update(self, scene):
		if len(scene)<2:
			return

		actual = -1
		prev = -2
		high_var = scene[actual].high-scene[prev].high
		low_var = scene[prev].low-scene[actual].low
		if high_var==low_var or \
			(high_var<0.0 and low_var<0.0):
			dm_plus = 0.0
			dm_minus = 0.0
		elif high_var>low_var:
			dm_plus = high_var
			dm_minus = 0.0
		else:
			dm_plus = 0.0
			dm_minus = low_var
		new_item = [scene[actual].time, dm_plus, dm_minus]
		self.__infos.append(new_item)

	def __str__(self):
		ret = ''
		for i in range(len(self)):
			item = self[i]
			ret += '%s %.2f %.2f\r\n' % (str(item[0]), item[1], item[2])
		return ret

# ADM指标
class ADM(BaseInd):
	'''
	指标DM的滑动平均
	'''
	def __init__(self, interval=10):
		super(ADM, self).__init__()
		self.__internal_dm = DM()
		self.__interval = interval
		self.__infos = [] # [0:时间,1:ADM+数据,2:ADM-数据]

	def __len__(self):
		return len(self.__infos)

	def __getitem__(self, key):
		if not isinstance(key, int):
			raise KeyError
		return self.__infos[key]

	@property
	def interval(self):
		return self.__interval

	@property
	def last(self):
		if len(self.__infos)<=0:
			return None
		item = self[-1]
		return (item[0], item[1], item[2])

	def update(self, scene):
		self.__internal_dm.update(scene)

		if len(self.__internal_dm)<self.interval:
			return

		prev = -2
		actual = -1

		result_plus = 0.0
		result_minus=  0.0
		if len(self)==0:
			# ADM+/ADM-第一次计算,取先前N个DM+/DM-的算术平均
			for i in range(self.interval):
				dm = self.__internal_dm[-i-1]
				result_plus += dm[1]
				result_minus += dm[2]
		else:
			result_plus += self.last[1]*(self.interval-1)+self.__internal_dm[actual][1]
			result_minus += self.last[2]*(self.interval-1)+self.__internal_dm[actual][2]
		result_plus /= self.interval
		result_minus /= self.interval

		new_item = [scene[actual].time, result_plus, result_minus]
		self.__infos.append(new_item)

	def __str__(self):
		ret = ''
		for i in range(len(self)):
			item = self[i]
			ret += '%s %.2f %.2f\r\n' % (str(item[0]), item[1], item[2])
		return ret

# TR指标
class TR(BaseInd):

	def __init__(self):
		super(TR, self).__init__()
		self.__infos = [] # [0:时间,1:TR数据]

	def __len__(self):
		return len(self.__infos)

	def __getitem__(self, key):
		if not isinstance(key, int):
			raise KeyError
		return self.__infos[key]

	@property
	def last(self):
		if len(self.__infos)<=0:
			return None
		item = self[-1]
		return (item[0], item[1])

	def update(self, scene):
		if len(scene)<2:
			return

		actual = -1
		prev =  -2
		val = math.fabs(scene[actual].high-scene[actual].low)
		val = max(val, math.fabs(scene[actual].high-scene[prev].close))
		val = max(val, math.fabs(scene[actual].low-scene[prev].close))
		new_item = [scene[actual].time, val]
		self.__infos.append(new_item)

	def __str__(self):
		ret = ''
		for i in range(len(self)):
			item = self[i]
			ret += '%s %.2f\r\n' % (str(item[0]), item[1])
		return ret

# ATR指标
class ATR(BaseInd):
	'''
	指标TR的滑动平均
	'''
	def __init__(self, interval=10):
		super(ATR, self).__init__()
		self.__internal_tr = TR()
		self.__interval = interval
		self.__infos = [] # [0:时间,1:ATR数据]

	def __len__(self):
		return len(self.__infos)

	def __getitem__(self, key):
		if not isinstance(key, int):
			raise KeyError
		return self.__infos[key]

	@property
	def interval(self):
		return self.__interval

	@property
	def last(self):
		if len(self.__infos)<=0:
			return None
		item = self[-1]
		return (item[0], item[1])

	def update(self, scene):
		self.__internal_tr.update(scene)

		if len(self.__internal_tr)<self.interval:
			return

		prev = -2
		actual = -1
		result = 0.0
		if len(self)==0:
			# ATR第一次计算,取先前N个TR的算术平均
			for i in range(self.interval):
				tr = self.__internal_tr[-i-1]
				result += tr[1]
		else:
			result += self.last[1]*(self.interval-1)+self.__internal_tr[actual][1]
		result /= self.interval

		new_item = [scene[actual].time, result]
		self.__infos.append(new_item)

	def __str__(self):
		ret = ''
		for i in range(len(self)):
			item = self[i]
			ret += '%s %.2f\r\n' % (str(item[0]), item[1])
		return ret

# DX指标
class DX(BaseInd):

	def __init__(self, interval=10):
		super(DX, self).__init__()
		self.__interval = interval
		self.__internal_adm = ADM(self.interval)
		self.__internal_atr = ATR(self.interval)
		self.__infos = [] # [0:时间,1:DX数据]

	def __len__(self):
		return len(self.__infos)

	def __getitem__(self, key):
		if not isinstance(key, int):
			raise KeyError
		return self.__infos[key]

	@property
	def interval(self):
		return self.__interval

	@property
	def last(self):
		if len(self.__infos)<=0:
			return None
		item = self[-1]
		return (item[0], item[1])

	def update(self, scene):
		self.__internal_adm.update(scene)
		self.__internal_atr.update(scene)

		if len(self.__internal_adm)<=0 or len(self.__internal_atr)<=0:
			return

		adm_plus = self.__internal_adm.last[1]
		adm_minus = self.__internal_adm.last[2]
		atr = self.__internal_atr.last[1]

		di_plus = round(adm_plus/atr*100.0)
		di_minus = round(adm_minus/atr*100.0)
		dx = round(math.fabs(di_plus-di_minus)/(di_plus+di_minus)*100.0)

		new_item = [scene.last.time, dx]
		self.__infos.append(new_item)

	def __str__(self):
		ret = ''
		for i in range(len(self)):
			item = self[i]
			ret += '%s %.2f\r\n' % (str(item[0]), item[1])
		return ret

# ADX指标
class ADX(BaseInd):
	'''
	ADX指标 : DX指标的滑动平均.
	用于衡量当前趋势的强弱,若大于一定值(一般40),则认为处于趋势中,若小于一定值(一般20),则认为处于盘整中.
	ADX的窗口宽度多取14. ---> 14代表"两周/半月",可尝试取"10/11".
	'''
	def __init__(self, interval=10):
		super(ADX, self).__init__()
		self.__interval = interval
		self.__internal_dx = DX(self.interval)
		self.__infos = [] # [0:时间,1:adx数据]

	def __len__(self):
		return len(self.__infos)

	def __getitem__(self, key):
		if not isinstance(key, int):
			raise KeyError
		return self.__infos[key]

	@property
	def interval(self):
		return self.__interval

	@property
	def last(self):
		if len(self.__infos)<=0:
			return None
		item = self[-1]
		return (item[0], item[1])

	def update(self, scene):
		self.__internal_dx.update(scene)

		if len(self.__internal_dx)<self.interval:
			return

		result = 0.0
		if len(self)==0:
			# ADX第一次计算,取先前N个DX的算术平均
			for i in range(self.interval):
				dx = self.__internal_dx[-i-1]
				result += dx[1]
		else:
			result += self.last[1]*(self.interval-1)+self.__internal_dx.last[1]
		result /= self.interval

		new_item = [scene.last.time, result]
		self.__infos.append(new_item)

	def __str__(self):
		ret = ''
		for i in range(len(self)):
			item = self[i]
			ret += '%s %.2f\r\n' % (str(item[0]), item[1])
		return ret


import argparse
import time
import torch
from torch import nn
import apex
from optimizer.grad_scalar import *
from optimizer.optimizer import Fp16Optimizer
from modules.gpt_modules import GPTTransformerLayer

def benchmark_transformer_layer(args, device):
    module_list = []
    for _ in range(args.num_layers):
        module_list.append(GPTTransformerLayer(args.embedding_dim, args.num_heads,
                                               4 * args.embedding_dim, use_checkpoint=args.use_checkpoint))
    layers = nn.Sequential(*module_list).to(device)
    print(layers)
    # summary(one_layer, (args.seq_length, args.embedding_size), batch_dim=0)
    layers.half()
    optimizer = torch.optim.SGD(layers.parameters(), lr=args.lr)
    # optimizer = apex.optimizers.FusedSGD(layers.parameters(), lr=args.lr)
    grad_scaler = ConstantGradScaler(0.1, offload=args.offload)
    fp16_optimizer = Fp16Optimizer(optimizer, grad_scaler, device, offload=args.offload)

    batch_shape = (args.batch_size, args.seq_length, args.embedding_dim)

    forward_start_event = torch.cuda.Event(enable_timing=True)
    backward_start_event = torch.cuda.Event(enable_timing=True)
    optimizer_start_event = torch.cuda.Event(enable_timing=True)
    end_event = torch.cuda.Event(enable_timing=True)

    for i in range(11):
        fake_batch = torch.zeros(batch_shape, requires_grad=True, dtype=torch.float16).to(device)
        fake_batch.uniform_(-0.05, 0.05)
        external_gradient = torch.full(batch_shape, 0.1, dtype=torch.float16).to(device)
        external_gradient.uniform_(-0.01, 0.01)

        print("Current memory allocated: {:2.3f} MB, peak memory: {:2.3f} MB".format(
            torch.cuda.memory_allocated(device) / 1048576, torch.cuda.max_memory_allocated(device) / 1048576))

        forward_start_event.record()
        output = layers(fake_batch)

        backward_start_event.record()
        output.backward(gradient=external_gradient)

        optimizer_start_event.record()
        fp16_optimizer.step()
        end_event.record()

        torch.cuda.current_stream().synchronize()

        print("Iter ", i, "Current iter forward takes:", forward_start_event.elapsed_time(backward_start_event))
        print("Iter ", i, "Current iter backward takes:", backward_start_event.elapsed_time(optimizer_start_event))
        print("Iter ", i, "Current iter optimizer step takes:", optimizer_start_event.elapsed_time(end_event))

        print("Current memory allocated: {:2.3f} MB, peak memory: {:2.3f} MB".format(
            torch.cuda.memory_allocated(device) / 1048576, torch.cuda.max_memory_allocated(device) / 1048576))

def main():
    parser = argparse.ArgumentParser(description='Gpipe-GPT3')
    parser.add_argument('--use-checkpoint', default=True, type=lambda x: (str(x).lower() == 'true'),
                        help='if this is set to True, will use check point for activation recompute')
    parser.add_argument('--offload', default=True, type=lambda x: (str(x).lower() == 'true'),
                        help='if this is set to True, will use CPU to store model.')
    parser.add_argument('--use-cuda', default=True, type=lambda x: (str(x).lower() == 'true'),
                        help='if this is set to True, will use cuda to train')
    parser.add_argument('--cuda-id', type=int, default=0, metavar='N',
                        help='cuda index, if the instance has multiple GPUs.')
    parser.add_argument('--batch-size', type=int, default=8, metavar='N',
                        help='input batch size for training (default: 100)')
    parser.add_argument('--num-layers', type=int, default=5, metavar='N',
                        help='-')
    parser.add_argument('--seq-length', type=int, default=2048, metavar='N',
                        help='-')
    parser.add_argument('--embedding-dim', type=int, default=2048, metavar='N',
                        help='-')
    parser.add_argument('--num-heads', type=int, default=16, metavar='N',
                        help='-')
    parser.add_argument('--lr', type=float, default=0.0001, metavar='N',
                        help='-')
    args = parser.parse_args()
    assert args.use_cuda and torch.cuda.is_available()
    device = torch.device('cuda', args.cuda_id)
    benchmark_transformer_layer(args, device)

if __name__ == '__main__':
    main()


from random import choice
from typing import Tuple, List

from model.Battle import Battle
from model.Board import Board
from model.Param import Param
from model.Unit import Unit
from model.UnitFactory import UnitFactory

def generate_unit(round: int) -> Unit:
    tank_factory = UnitFactory(attack=Param(low=10, high=20, multiplier=1.1),
                               defense=Param(low=20, high=35, multiplier=1.1),
                               magicResist=Param(low=20, high=35, multiplier=1.1),
                               criticalChance=Param(low=10, high=20, multiplier=1.02, max=100),
                               hp=Param(low=100, high=140, multiplier=1.1),
                               range=Param(low=2, high=3),
                               attackSpeed=Param(low=7, high=10, multiplier=1.05),
                               type="PHYSICAL")

    sniper_factory = UnitFactory(attack=Param(low=30, high=60, multiplier=1.1),
                                 defense=Param(low=0, high=10, multiplier=1.1),
                                 magicResist=Param(low=0, high=5, multiplier=1.1),
                                 criticalChance=Param(low=40, high=50, multiplier=1.05, max=100),
                                 hp=Param(low=40, high=60, multiplier=1.1),
                                 range=Param(low=6, high=10),
                                 attackSpeed=Param(low=4, high=5, multiplier=1.05),
                                 type="PHYSICAL")

    rogue_factory = UnitFactory(attack=Param(low=25, high=50, multiplier=1.1),
                                defense=Param(low=0, high=4, multiplier=1.1),
                                magicResist=0,
                                criticalChance=Param(low=30, high=40, multiplier=1.05, max=100),
                                hp=Param(low=30, high=50, multiplier=1.1),
                                range=Param(low=2, high=4),
                                attackSpeed=Param(low=10, high=15, multiplier=1.05),
                                type="PHYSICAL")

    magician_factory = UnitFactory(attack=Param(low=20, high=30, multiplier=1.1),
                                   defense=Param(low=10, high=20, multiplier=1.1),
                                   magicResist=Param(low=15, high=25, multiplier=1.1),
                                   criticalChance=0,
                                   hp=Param(low=70, high=90, multiplier=1.1),
                                   range=Param(low=3, high=6),
                                   attackSpeed=Param(low=8, high=12, multiplier=1.05),
                                   type="MAGICAL")

    magical_tank_factory = UnitFactory(attack=Param(low=15, high=22, multiplier=1.1),
                                       defense=Param(low=25, high=40, multiplier=1.1),
                                       magicResist=Param(low=25, high=40, multiplier=1.1),
                                       criticalChance=0,
                                       hp=Param(low=90, high=120, multiplier=1.1),
                                       range=Param(low=3, high=4),
                                       attackSpeed=Param(low=7, high=10, multiplier=1.05),
                                       type="MAGICAL")

    unit_factory = choice([
        tank_factory,
        sniper_factory,
        rogue_factory,
        magician_factory,
        magical_tank_factory
    ])
    return unit_factory.create(round=round)

def resolve_battle(board1: Board, board2: Board) -> Tuple[int, int, List[dict]]:
    battle = Battle(board1, board2)
    battle.resolve()

    winner_number = 1 if battle.winner is board2 else 0
    player_hp_change = -3 * len(battle.winner.alive_tokens)
    log = battle.log

    return winner_number, player_hp_change, log


import pymysql
import time

def port_to_data(car_id,station_id):
    # 打开数据库连接
    db = pymysql.connect ("localhost", "root", "123456", "lightdata")

    # 使用cursor()方法获取操作游标
    cursor = db.cursor ()

    # SQL 插入语句
    s = time.strftime ('%Y.%m.%d.%H.%M.%S', time.localtime (time.time ()))

    sql = "INSERT INTO car" + "(date, car_id, station_id)"\
        + "VALUES" + "('" + s + "'," + car_id + "," + station_id + ");"
    try:
        # 执行sql语句
        cursor.execute (sql)
        # 提交到数据库执行
        db.commit ()
    except:
        # 如果发生错误则回滚
        db.rollback ()

    # 关闭数据库连接
    db.close ()


import datetime

import sqlbuilder.smartsql as sql

class InsertQuery:
    def __init__(self, tablename):
        self.tablename = tablename

    def __call__(self, *args):
        stagingtable = sql.T(self.tablename)

        query = sql.Q().tables(stagingtable)

        kwargs = self.generate_kwargs(*args)

        query = query.insert(kwargs,)

        return query

    def generate_kwargs(self, *args):
        return

class InsertCharInChapter(InsertQuery):
    def __init__(self):
        InsertQuery.__init__(self, "charinchapter")

    def generate_kwargs(self, chapname, charfirst, charsur):
        return {
            "charfirstname": charfirst,
            "charsurname": charsur,
            "chapname": chapname,
            'insertiontime': datetime.datetime.now()
        }

class InsertChapter(InsertQuery):
    def __init__(self):
        InsertQuery.__init__(self, "chapter")

    def generate_kwargs(self, chapname, chapnum, bname):
        return {
            "chaptername": chapname,
            "chapternumber": chapnum,
            "bookname": bname,
            'insertiontime': datetime.datetime.now()
        }

class InsertCharacter(InsertQuery):
    def __init__(self):
        InsertQuery.__init__(self, "character")

    def generate_kwargs(self, charfirst, charsur, gender, born, died):
        keydict = {
            "firstname": charfirst,
            "surname": charsur,
            'insertiontime': datetime.datetime.now()
            }

        if gender:
            keydict.update({"gender": gender})

        if born:
            keydict.update({"born": born})

        if died:
            keydict.update({"died": died})

        return keydict


import tqdm
import numpy as np
import random
from collections import deque
import pickle5 as pickle

from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import Adam

class Supervisor :
    def __init__(self, game) :
        self.game = game

    def simulate(self, p1, p2, nb_round, verbose=False) :
        for battle_count in tqdm.tqdm(range(nb_round)) :
            current_game = self.game(p1, p2, verbose)
            p1_mem = []
            p2_mem = []
            turn_p1 = []
            turn_p2 = []
            for i in range(9) :
                if current_game.p1_turn :
                    turn_p1.append(current_game.board.board.copy())
                    move = current_game.next_moves()
                    turn_p1.append(move)
                    if not current_game.is_legal_move(move) :
                        turn_p1.append(current_game.board.board.copy())
                        turn_p1.append(True)
                        turn_p1.append(-1)
                        p1_mem.append(turn_p1)
                        p1.memorize(p1_mem)
                        break
                    else :
                        current_game.apply_move(move)
                        current_game.detect_end(current_game.board)
                else :
                    current_game.board.invert()
                    turn_p2.append(current_game.board.board.copy())
                    move = current_game.next_moves()
                    turn_p2.append(move)
                    if not current_game.is_legal_move(move) :
                        turn_p2.append(current_game.board.board.copy())
                        turn_p2.append(True)
                        turn_p2.append(-1)
                        p2_mem.append(turn_p2)
                        p2.memorize(p2_mem)
                        break
                    else :
                        current_game.apply_move(move)
                        current_game.detect_end(current_game.board)
                    current_game.board.invert()

                if i != 0 :
                    if current_game.p1_turn :
                        turn_p1.append(current_game.board.board.copy())
                        turn_p1.append(current_game.done)
                        if current_game.done :
                            if current_game.final_state == 1 :
                                turn_p1.append(1)
                            elif current_game.final_state == 0 :
                                turn_p1.append(0)
                            else :
                                turn_p1.append(-1)
                        else :
                            turn_p1.append(0)
                        p1_mem.append(turn_p1)
                        turn_p1 = []
                    else :
                        turn_p2.append(current_game.board.board.copy())
                        turn_p2.append(current_game.done)
                        if current_game.done :
                            if current_game.final_state == 2 :
                                turn_p2.append(1)
                            elif current_game.final_state == 0 :
                                turn_p2.append(0)
                            else :
                                turn_p2.append(-1)
                        else :
                            turn_p2.append(0)
                        p2_mem.append(turn_p2)
                        turn_p2 = []

                    if current_game.done :
                        if not current_game.p1_turn :
                            turn_p1.append(current_game.board.board.copy())
                            turn_p1.append(current_game.done)
                            if current_game.final_state == 1 :
                                turn_p1.append(1)
                            elif current_game.final_state == 0 :
                                turn_p1.append(0)
                            else :
                                turn_p1.append(-1)
                            p1_mem.append(turn_p1)
                        else :
                            turn_p2.append(current_game.board.board.copy())
                            turn_p2.append(current_game.done)
                            if current_game.final_state == 1 :
                                turn_p2.append(-1)
                            elif current_game.final_state == 0 :
                                turn_p2.append(0)
                            else :
                                turn_p2.append(1)
                            p2_mem.append(turn_p2)

                        p1.memorize(p1_mem)
                        p2.memorize(p2_mem)
                        break
                if verbose :
                    print(current_game.board)

class Morpion :
    def __init__(self, p1, p2, verbose=False) :
        self.p1 = p1
        self.p1.jeu = self
        self.p2 = p2
        self.p2.jeu = self
        self.board = Board()
        self.p1_turn = True
        self.verbose = verbose
        self.final_state = None
        self.done = False

    def possible_moves(self, board, p1_turn) :
        possibilities = []
        for i in range(9) :
            if board.board[i] == 0 :
                possibility = np.zeros((9), dtype=np.int8)
                possibility[i] = 1 if p1_turn else 2
                possibilities.append(possibility)

        print(possibilities)
        return possibilities

    def detect_end(self, board) :
        is_win = False
        for i in range(3) : # horizontal
            if board.board[0+i*3] != 0 and board.board[1+i*3] == board.board[0+i*3] and board.board[2+i*3] == board.board[0+i*3] :
                is_win = True
        for i in range(3) : # vertical
            if board.board[0+i] != 0 and board.board[3+i] == board.board[0+i] and board.board[6+i] == board.board[0+i] :
                is_win = True
        if board.board[0] != 0 and board.board[4] == board.board[0] and board.board[8] == board.board[0] :
            is_win = True
        if board.board[6] != 0 and board.board[4] == board.board[6] and board.board[2] == board.board[6] :
            is_win = True

        if is_win :
            self.final_state = 1 if not self.p1_turn else 2
            self.done = True
            if self.verbose :
                print("Le joueur {} gagne !".format(1 if not self.p1_turn else 2))
            return

        is_draw = True
        for i in range(9) :
            if board.board[i] == 0 :
                is_draw = False
        if is_draw :
            self.final_state = 0
            self.done = True
            if self.verbose :
                print("Egalitée")
            return

    def is_legal_move(self, move) :
        return self.board.board[np.argmax(move)] == 0

    def apply_move(self, move) :
        self.board.board[np.argmax(move)] = 1

    def next_moves(self) :
        if self.p1_turn :
            if self.verbose :
                print('Joueur 1')
            move = self.p1.play(self.board.copy(), self.p1_turn)
        else :
            if self.verbose :
                print('Joueur 2')
            move = self.p2.play(self.board.copy(), self.p1_turn)

        self.p1_turn = not self.p1_turn
        return move

class Board :
    def __init__(self) :
        self.board = np.zeros((9), dtype=np.int8)

    def __repr__(self) :
        delimiteur = "\n+---+---+---+\n"
        rep = delimiteur
        for i in range(9) :
            if i % 3 == 0 and i != 0 :
                rep += "|" + delimiteur
            rep += "| {} ".format(self.board[i])
        rep += "|" + delimiteur

        return rep

    def copy(self) :
        board_copy = Board()
        board_copy.board = self.board.copy()
        return board_copy

    def invert(self) :
        for i in range(9) :
            if self.board[i] == 1 : self.board[i] = 2; continue
            if self.board[i] == 2 : self.board[i] = 1
        return self

class Agent :
    def __init__(self) :
        raise NotImplementedError

    def play(self) :
        raise NotImplementedError

    def memorize(self, game_history) :
        self.memory.append(game_history)

    def save_memory(self, file) :
        with open(file, 'wb') as f :
            pickle.dump(self.memory, f)

    def load_memory(self, file) :
        with open(file, 'rb') as f :
            self.memory = pickle.load(f)

class Singe(Agent) :
    def __init__(self, jeu=None) :
        self.jeu = jeu
        self.memory = deque(maxlen=10000)

    def play(self, board, p1_turn) :
        possibilities = []
        for i in range(9) :
            if board.board[i] == 0 :
                possibility = np.zeros((9), dtype=np.int8)
                possibility[i] = 1
                possibilities.append(possibility)
        move = random.choice(possibilities)
        return move

class Debug(Agent) :
    def __init__(self, jeu=None) :
        self.jeu = jeu
        self.memory = deque(maxlen=200)

    def play(self, board, p1_turn) :
        print(board.board)
        choice = int(input("Position à jouer \n>>"))
        move = np.zeros((9), dtype=np.int8)
        move[choice] = 1
        return move

class DQNAgent(Agent) :
    def __init__(self, state_size, action_size) :
        self.state_size = state_size
        self.action_size = action_size
        self.memory = deque(maxlen=2000)
        self.gamma = 0.95    # discount rate
        self.epsilon = 0  # exploration rate
        self.epsilon_min = 0.01
        self.epsilon_decay = 0.995
        self.learning_rate = 0.0005
        self.model = self._build_model()

    def _build_model(self):
        # Neural Net for Deep-Q learning Model
        model = Sequential()
        model.add(Dense(24, input_dim=self.state_size, activation='relu'))
        model.add(Dense(24, activation='relu'))
        model.add(Dense(self.action_size, activation='linear'))
        model.compile(loss='mse',
                      optimizer=Adam(lr=self.learning_rate))
        return model

    def play(self, board, p1_turn):
        if np.random.rand() <= self.epsilon:
            move = np.zeros((9), dtype=int)
            move[random.randrange(9)] = 1
            return move
        formated_board = np.reshape(board.board, [1, 9])
        act_values = self.model.predict(formated_board)
        move = np.zeros((9), dtype=np.int8)
        move[np.argmax(act_values[0])] = 1
        return move

    def train(self, minibatch_size, batch):
        minibatch = random.sample(batch, batch_size)
        for state, action, next_state, done, reward in minibatch:
            target = reward                                                    # A implementer correctement
            if not done:                                                       #
                target = (reward + self.gamma *                                #
                          np.amax(self.model.predict(next_state)[0]))          #
            target_f = self.model.predict(state)                               #
            target_f[0][action] = target                                       #
            self.model.fit(state, target_f, epochs=1, verbose=0)
        if self.epsilon > self.epsilon_min:
            self.epsilon *= self.epsilon_decay

    def load_model(self, name):
        self.model.load_weights(name)

    def save_model(self, name):
        self.model.save_weights(name)

class BatchMorpion :
    def __init__(self, gamma=0.8) :
        self.gamma = gamma

    def is_bad_move(self, game) :
        last = game[-1]
        return last[-1] == -1 and np.array_equal(last[0], last[2])

    def create_batch(self, file) :
        batch = []
        p1_win = 0
        p2_win = 0
        with open(file, 'rb') as f :
            games = pickle.load(f)
        for game in games :
            if self.is_bad_move(game) :
                game = [game[-1]]
            else :
                reward = game[-1][-1]
                for i, turn in enumerate(game[:-1]) :
                    turn[-1] = turn[-1] + reward * self.gamma**(len(game)-i)
            for turn in game :
                batch.append(turn)

        return batch

if __name__ == '__main__' :
    p1 = DQNAgent(9, 9)
    #p1 = Singe()
    p2 = Singe()

    mj = Supervisor(Morpion)
    mj.simulate(p1, p2, 10, verbose=False)

    p1.save_memory('p1.mem')

    batcher = BatchMorpion()
    batch = batcher.create_batch('p1.mem')

    #p1.train(32, batch)


#! /usr/bin/env python3
# coding: utf-8
from __future__ import annotations

import typing as t

import pendulum
from psycopg2.extensions import connection

from gargbot_3000.journey import common

queries = common.queries.achievements

possible = [
    {
        "query": queries.most_steps_one_day_individual,
        "emoji": ":first_place_medal:",
        "desc": "Flest skritt gått av en gargling på én dag",
        "unit": "skritt",
        "kwargs": {"less_than": None},
        "collective": False,
    },
    {
        "query": queries.most_steps_one_day_collective,
        "emoji": ":trophy:",
        "desc": "Flest skritt gått av hele gargen på én dag",
        "unit": "skritt",
        "collective": True,
    },
    {
        "query": queries.highest_share,
        "emoji": ":sports_medal:",
        "desc": "Størst andel av dagens skritt",
        "unit": "%",
        "collective": False,
    },
    {
        "query": queries.biggest_improvement_individual,
        "emoji": ":sports_medal:",
        "desc": "Størst improvement fra en dag til neste for en gargling",
        "unit": "skritt",
        "collective": False,
    },
    {
        "query": queries.biggest_improvement_collective,
        "emoji": ":trophy:",
        "desc": "Størst improvement fra en dag til neste for hele gargen",
        "unit": "skritt",
        "collective": True,
    },
    {
        "query": queries.longest_streak,
        "emoji": ":sports_medal:",
        "desc": "Lengste streak med førsteplasser",
        "unit": "dager",
        "collective": False,
    },
]

def format_new(
    desc: str,
    unit: str,
    holders: t.Optional[list[int]],
    value: int,
    prev_holders: t.Optional[t.Set[int]],
    prev_value: t.Optional[int],
    gargling_info: dict[int, dict],
):
    tangering = prev_value is None
    achv_txt = ""
    medal = ":first_place_medal:" if not tangering else ":sports_medal:"
    if holders is not None:
        names = [gargling_info[id_]["first_name"] for id_ in holders]
        achv_txt += f" {medal} og ".join(names)
    else:
        achv_txt += "Vi"
    achv_txt += f" {medal}"
    achv_txt += " har"
    if not tangering:
        achv_txt += f" satt ny rekord: {desc}, med {value} {unit}!"
    else:
        achv_txt += f" tangert rekord: {desc} ({value} {unit})!"
    if prev_holders is not None:
        names = [gargling_info[id_]["first_name"] for id_ in prev_holders]
        plural = "s" if len(prev_holders) > 1 else ""
        achv_txt += f" Forrige record holder{plural} var " + " og ".join(names)
        if not tangering:
            achv_txt += f", med {prev_value} {unit}"
    elif not tangering:
        achv_txt += f" Forrige rekord var {prev_value} {unit}"
    achv_txt += "."
    achv_txt += " Huzzah! :sonic:"
    return achv_txt

def extract(
    conn: connection,
    journey_id: int,
    date: pendulum.Date,
    query: t.Callable,
    **query_kwargs,
) -> t.Optional[
    tuple[t.Optional[list[int]], int, t.Optional[t.Set[int]], t.Optional[int]]
]:
    current = query(conn, journey_id=journey_id, taken_before=date, **query_kwargs)
    if len(current) == 0:
        return None
    if not any(rec["taken_at"] == date for rec in current):
        return None
    new = [rec for rec in current if rec["taken_at"] == date]
    value = new[0]["amount"]
    try:
        holders: t.Optional[list[int]] = [r["gargling_id"] for r in new]
    except KeyError:
        holders = None
    prev = [rec for rec in current if rec["taken_at"] < date]
    if prev:
        prev_value = None
    else:
        prev = query(
            conn,
            journey_id=journey_id,
            taken_before=date.subtract(days=1),
            **query_kwargs,
        )
        if not prev:
            return None
        prev_value = prev[0]["amount"]
    try:
        prev_holders: t.Optional[t.Set[int]] = {r["gargling_id"] for r in prev}
    except KeyError:
        prev_holders = None
    return holders, value, prev_holders, prev_value

def new(
    conn: connection,
    journey_id: int,
    date: pendulum.Date,
    gargling_info: dict[int, dict],
) -> t.Optional[str]:
    for p in possible:
        achv = extract(
            conn=conn,
            journey_id=journey_id,
            date=date,
            query=p["query"],
            **p.get("kwargs", {}),
        )
        if achv is None:
            continue
        unit = p["unit"]
        desc = p["desc"]
        holders, value, prev_holders, prev_value = achv
        formatted = format_new(
            desc=desc,
            unit=unit,
            holders=holders,
            value=value,
            prev_holders=prev_holders,
            prev_value=prev_value,
            gargling_info=gargling_info,
        )
        return formatted
    return None

def get_all_at_date(conn, journey_id, date=None):
    all_records: list[dict] = []
    p = possible[0]
    most = [
        p,
        {
            "query": queries.most_steps_one_day_individual,
            "emoji": ":second_place_medal:",
            "desc": "Nest flest skritt gått av en gargling på én dag",
            "unit": "skritt",
            "collective": False,
        },
        {
            "query": queries.most_steps_one_day_individual,
            "emoji": ":third_place_medal:",
            "desc": "Tredje flest skritt gått av en gargling på én dag",
            "unit": "skritt",
            "collective": False,
        },
    ]
    less_than = None
    for p in most:
        query = p["query"]
        rec = query(conn, journey_id=journey_id, taken_before=date, less_than=less_than)
        if not rec:  # no test coverage
            break
        all_records.append(
            {
                "records": rec,
                "emoji": p["emoji"],
                "desc": p["desc"],
                "unit": p["unit"],
                "collective": p["collective"],
            }
        )
        less_than = rec[0]["amount"]

    for p in possible[1:]:
        query = p["query"]
        rec = query(conn, journey_id=journey_id, taken_before=date)
        if not rec:
            continue
        all_records.append(
            {
                "records": rec,
                "emoji": p["emoji"],
                "desc": p["desc"],
                "unit": p["unit"],
                "collective": p["collective"],
            }
        )
    return all_records

def format_all(gargling_info: dict[int, dict], records) -> str:
    def fdate(date: pendulum.Date) -> str:
        return f"{date.day}.{date.month}.{date.year}"

    def format_col(data) -> str:
        desc = data["desc"]
        amount = data["records"][0]["amount"]
        unit = data["unit"]
        res = f"{desc}: {amount} {unit} {data['emoji']} - "
        res += " & ".join([fdate(rec["taken_at"]) for rec in data["records"]])
        return res

    def format_ind(data) -> str:
        desc = data["desc"]
        amount = data["records"][0]["amount"]
        unit = data["unit"]
        res = f"{desc}: {amount} {unit} - "
        ppl = []
        for rec in data["records"]:
            name = gargling_info[rec["gargling_id"]]["first_name"]
            ppl.append(f"{name} {data['emoji']} ({fdate(rec['taken_at'])})")
        res += " & ".join(ppl)
        return res

    result = []
    for data in records:
        if data["collective"]:
            res = format_col(data)
        else:
            res = format_ind(data)
        result.append(res)
    return "\n".join(result)

def all_at_date(conn: connection, date: pendulum.Date = None) -> str:
    ongoing_journey = common.queries.journey.get_ongoing_journey(conn)
    journey_id = ongoing_journey["id"]
    all_records = get_all_at_date(conn, journey_id, date)
    gargling_ids = set()
    for record in all_records:
        for rec in record["records"]:
            gargling_id = rec.get("gargling_id")
            if gargling_id:
                gargling_ids.add(gargling_id)
    gargling_info = common.get_colors_names(conn, ids=list(gargling_ids))
    formatted = format_all(gargling_info, all_records)
    return formatted


import cv2 as cv

'''
视频文件的读、写
'''

# 加载本地视频文件
videoInput = cv.VideoCapture('my-video.avi')
fps = videoInput.get(cv.CAP_PROP_FPS)
size = (int(videoInput.get(cv.CAP_PROP_FRAME_WIDTH)), int(videoInput.get(cv.CAP_PROP_FRAME_HEIGHT)))
# 创建到本地文件的输出
videoWriter = cv.VideoWriter('my-video-copy.avi', cv.VideoWriter_fourcc('I', '4', '2', '0'), fps, size)
# 读取输入文件，如果读取失败， success == False
success, frame = videoInput.read()
while success:
    # 写入帧到输出中
    videoWriter.write(frame)
    success, frame = videoInput.read()
print("done!")

'''
备注：cv.VideoWriter_fourcc('I', '4', '2', '0') 是一个未压缩的YUV颜色编码，是4：2：0色度子采样。这种编码有很好的兼容性，但会产生较大文件，文件扩展名为.avi
    其他常用选项：
        cv.VideoWriter_fourcc('P', 'I', 'M', '1') MPEG-1编码类型，文件扩展名为.avi
        cv.VideoWriter_fourcc('X', 'V', 'I', 'D') MPEG-4编码类型，如果希望得到的视频大小为平均值，推荐使用此选项，文件扩展名为.avi
        cv.VideoWriter_fourcc('T', 'H', 'E', 'O') Ogg Vorbis，文件扩展名为.ogv
        cv.VideoWriter_fourcc('F', 'L', 'V', '1') Flash视频，文件扩展名为.flv

'''


import argparse
import torch
import os
from datetime import datetime
import time
import torch
import random
import numpy as np
import sys

class Options(object):
    """docstring for Options"""
    def __init__(self):
        super(Options, self).__init__()

    def initialize(self):
        parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
        parser.add_argument('--mode', type=str, default='train', help='Mode of code. [train|test]')
        parser.add_argument('--lucky_seed', type=int, default=0, help='seed for random initialize, 0 to use current time.')
        parser.add_argument('--visdom_env', type=str, default="main", help='visdom env.')
        parser.add_argument('--visdom_port', type=int, default=8097, help='visdom port.')
        parser.add_argument('--visdom_display_id', type=int, default=1, help='set value larger than 0 to display with visdom.')
        parser.add_argument('--backend_pretrain', action='store_true', help='if specified, use imagenet pretrained for backend.')
        parser.add_argument('--no_data_augment', action='store_true', help='if specified, do not use data augmentation.')

        parser.add_argument('--no_test_eval', action='store_true', help='do not use eval mode during test time.')
        parser.add_argument('--aus_id', type=str, default='1,2,4,5,6,7,9,12,17,23,25', help='AUs vector index.')

        parser.add_argument('--data_root', required=True, help='paths to data set.')
        parser.add_argument('--imgs_dir', type=str, default="imgs", help='path to image')
        parser.add_argument('--train_csv', type=str, default="train_ids_0.csv", help='train images paths')
        parser.add_argument('--test_csv', type=str, default="test_ids_0.csv", help='test images paths')
        parser.add_argument('--pseudo_csv', type=str, default="pseudo_aus.csv", help='Pseudo AUs.')
        parser.add_argument('--cls_pkl', type=str, default="emotion_labels.pkl", help='facial expression labels for images.')
        parser.add_argument('--aus_pkl', type=str, default="aus_openface.pkl", help='action units ground truth for test.')

        parser.add_argument('--batch_size', type=int, default=25, help='input batch size.')
        parser.add_argument('--serial_batches', action='store_true', help='if specified, input images in order.')
        parser.add_argument('--n_threads', type=int, default=6, help='number of workers to load data.')
        parser.add_argument('--max_dataset_size', type=int, default=float("inf"), help='maximum number of samples.')

        parser.add_argument('--load_size', type=int, default=76, help='scale image to this size.')
        parser.add_argument('--final_size', type=int, default=64, help='crop image to this size.')

        parser.add_argument('--gpu_ids', type=str, default='0', help='gpu ids, eg. 0,1,2; -1 for cpu.')
        parser.add_argument('--ckpt_dir', type=str, default='./ckpts', help='directory to save check points.')
        parser.add_argument('--result_dir', type=str, default='./results', help='directory to save test results.')
        parser.add_argument('--load_epoch', type=int, default=0, help='load epoch; 0: do not load')
        parser.add_argument('--log_file', type=str, default="logs.txt", help='log loss')
        parser.add_argument('--opt_file', type=str, default="opt.txt", help='options file')

        parser.add_argument('--img_nc', type=int, default=3, help='image number of channel')
        parser.add_argument('--aus_nc', type=int, default=11, help='aus number of channel')
        parser.add_argument('--exp_nc', type=int, default=6, help='number of expression')
        parser.add_argument('--which_model_netR', type=str, default='default', help='Mode of recognition net. [default|resnet50|resnet152]')
        parser.add_argument('--hidden_nc_list', type=str, default='240,240', help='discriminator hidden layer number of channel')

        parser.add_argument('--gan_type', type=str, default='gan', help='GAN loss [wgan-gp|lsgan|gan]')
        parser.add_argument('--init_type', type=str, default='normal', help='network initialization [normal|xavier|kaiming|orthogonal]')
        parser.add_argument('--init_gain', type=float, default=0.02, help='scaling factor for normal, xavier and orthogonal.')
        parser.add_argument('--norm', type=str, default='instance', help='instance normalization or batch normalization [batch|instance|none]')
        parser.add_argument('--beta1', type=float, default=0.5, help='momentum term of adam')
        parser.add_argument('--lr', type=float, default=0.0001, help='initial learning rate for adam')
        parser.add_argument('--lr_policy', type=str, default='lambda', help='learning rate policy: lambda|step|plateau|cosine')
        parser.add_argument('--lr_decay_iters', type=int, default=50, help='multiply by a gamma every lr_decay_iters iterations')

        parser.add_argument('--epoch_count', type=int, default=1, help='the starting epoch count, we save the model by <epoch_count>, <epoch_count>+<save_latest_freq>, ...')
        parser.add_argument('--niter', type=int, default=150, help='# of iter at starting learning rate')
        parser.add_argument('--niter_decay', type=int, default=150, help='# of iter to linearly decay learning rate to zero')

        # frequency options
        parser.add_argument('--train_recog_iter', type=int, default=1, help='train G every n interations.')
        parser.add_argument('--train_dis_iter', type=int, default=5, help='train discriminator every n interations.')
        parser.add_argument('--print_losses_freq', type=int, default=100, help='print log every print_freq step.')
        parser.add_argument('--plot_losses_freq', type=int, default=500, help='plot log every plot_freq step.')
        parser.add_argument('--sample_img_freq', type=int, default=200, help='draw image every sample_img_freq step.')
        parser.add_argument('--save_epoch_freq', type=int, default=5000, help='save checkpoint every save_epoch_freq epoch.')

        return parser

    def parse(self):
        parser = self.initialize()
        parser.set_defaults(name=datetime.now().strftime("%y%m%d_%H%M%S"))
        opt = parser.parse_args()

        dataset_name = os.path.basename(opt.data_root.strip('/'))
        # update checkpoint dir
        if opt.mode == 'train' and opt.load_epoch == 0:
            tmp_list = os.path.splitext(opt.train_csv)[0].split('_')
            fold_id = "." if len(tmp_list) < 3 else ("fold_%s" % tmp_list[2])
            opt.ckpt_dir = os.path.join(opt.ckpt_dir, dataset_name, opt.which_model_netR, fold_id, opt.name)
            if not os.path.exists(opt.ckpt_dir):
                os.makedirs(opt.ckpt_dir)

        # if test, disable visdom, update results path
        if opt.mode == "test":
            opt.visdom_display_id = 0
            opt.result_dir = os.path.join(opt.result_dir, "%s_%s_%s" % (dataset_name, opt.which_model_netR, opt.load_epoch))
            if not os.path.exists(opt.result_dir):
                os.makedirs(opt.result_dir)

        # set gpu device
        str_ids = opt.gpu_ids.split(',')
        opt.gpu_ids = []
        for str_id in str_ids:
            cur_id = int(str_id)
            if cur_id >= 0:
                opt.gpu_ids.append(cur_id)
        if len(opt.gpu_ids) > 0:
            torch.cuda.set_device(opt.gpu_ids[0])

        # conver hidden_nc to list
        opt.hidden_nc_list = list(map(lambda x: int(x), opt.hidden_nc_list.split(',')))

        # set seed
        if opt.lucky_seed == 0:
            opt.lucky_seed = int(time.time())
        random.seed(a=opt.lucky_seed)
        np.random.seed(seed=opt.lucky_seed)
        torch.manual_seed(opt.lucky_seed)
        if len(opt.gpu_ids) > 0:
            torch.backends.cudnn.deterministic = True
            torch.backends.cudnn.benchmark = False
            torch.cuda.manual_seed(opt.lucky_seed)
            torch.cuda.manual_seed_all(opt.lucky_seed)

        # write command to file
        script_dir = opt.ckpt_dir
        with open(os.path.join(os.path.join(script_dir, "run_script.sh")), 'a+') as f:
            f.write("[%5s][%s]python %s\n" % (opt.mode, opt.name, ' '.join(sys.argv)))

        # print and write options file
        msg = ''
        msg += '------------------- [%5s][%s]Options --------------------\n' % (opt.mode, opt.name)
        for k, v in sorted(vars(opt).items()):
            comment = ''
            default_v = parser.get_default(k)
            if v != default_v:
                comment = '\t[default: %s]' % str(default_v)
            msg += '{:>25}: {:<30}{}\n'.format(str(k), str(v), comment)
        msg += '--------------------- [%5s][%s]End ----------------------\n' % (opt.mode, opt.name)
        print(msg)
        with open(os.path.join(os.path.join(script_dir, "opt.txt")), 'a+') as f:
            f.write(msg + '\n\n')

        return opt


import pandas as pd
import xgboost as xgb
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split

from utils.transfer_data import get_2022_stats
from utils.transfer_data import get_all_player_stats

iterations = 10

def XGBoost(X, y, X_2022):

    iterations = 10  # Number of trials
    prediction_trials = []

    for i in range(iterations):
        X_train, X_test, y_train, y_test = train_test_split(
            X, y, test_size=0.2, stratify=y,
        )
        xgb_model = xgb.XGBClassifier(
            objective='binary:logistic',
            tree_method='hist',
            eval_metric='logloss',
            use_label_encoder=False,
            n_estimators=100,
            learning_rate=0.2,
            gamma=1,
            reg_lambda=5,
            reg_alpha=0,
        )
        # Histogram-based boosting makes XGBoost much faster
        # Other speed-up options include using your CPU with CUDA (Nvidia)  xgb.XGBClassifier(tree_method = "gpu_hist")
        # and using single precision xgb.XGBClassifier(tree_method = "gpu_hist", single_precision_histogram=True)

        xgb_model.fit(X_train, y_train)
        y_pred = xgb_model.predict(X_test)

        predictions = xgb_model.predict_proba(X_2022)
        prediction_trials.append(predictions[:, 1])

    print('Confusion matrix and classification report for XGBoost model, final iteration.\n')
    print(confusion_matrix(y_test, y_pred))
    print(classification_report(y_test, y_pred))

    df = pd.DataFrame(prediction_trials).transpose()
    return df.mean(axis=1)

if __name__ == '__main__':
    X, y = get_all_player_stats()
    X_2022 = get_2022_stats()
    XGBoost(X, y, X_2022)


# -*- coding: utf-8 -*-
import onnx
import time
import numpy as np
import onnxruntime as rt
from onnx import checker
from onnx import helper
from onnx import TensorProto

def conver_rknn_tensor_type_to_onnx_type(tensor_dtype):
    datatype = tensor_dtype.split("_")[-1]
    if datatype == "INT8":
        return TensorProto.INT8
    elif datatype == "UINT8":
        return TensorProto.UINT8
    elif datatype == "INT16":
        return TensorProto.INT16
    elif datatype == "UINT16":
        return TensorProto.UINT16
    elif datatype == "INT32":
        return TensorProto.INT32
    elif datatype == "UINT32":
        return TensorProto.UINT32
    elif datatype == "FLOAT16":
        return TensorProto.FLOAT16
    elif datatype == "FLOAT32":
        return TensorProto.FLOAT
    elif datatype == "BOOL8":
        return TensorProto.BOOL
    else:
        assert False, "unspoorted datatype {}, when convert rknn tensor type to onnx type".format(datatype)

def conver_rknn_tensor_type_to_numpy_type(tensor_dtype):
    datatype = tensor_dtype.split("_")[-1]
    if datatype == "INT8":
        return np.int8
    elif datatype == "UINT8":
        return np.uint8
    elif datatype == "INT16":
        return np.int16
    elif datatype == "UINT16":
        return np.uint16
    elif datatype == "INT32":
        return np.int32
    elif datatype == "UINT32":
        return np.uint32
    elif datatype == "FLOAT16":
        return np.float16
    elif datatype == "FLOAT32":
        return np.float32
    elif datatype == "BOOL8":
        return np.bool
    else:
        assert False, "unspoorted datatype {}, when convert rknn tensor type to np type".format(datatype)

def convert_onnxruntime_type_to_numpy_dtype(onnxruntime_type):
    if onnxruntime_type == "tensor(int8)":
        return np.int8
    elif onnxruntime_type == "tensor(uint8)":
        return np.uint8
    elif onnxruntime_type == "tensor(int16)":
        return np.int16
    elif onnxruntime_type == "tensor(uint16)":
        return np.uint16
    elif onnxruntime_type == "tensor(int32)":
        return np.int32
    elif onnxruntime_type == "tensor(uint32)":
        return np.uint32
    elif onnxruntime_type == "tensor(float16)":
        return np.float16
    elif onnxruntime_type == "tensor(float)":
        return np.float32
    elif onnxruntime_type == "tensor(bool)":
        return np.bool
    else:
        assert False, "unspoorted datatype {}, when convert rknn tensor type to np type".format(onnxruntime_type)

def construct_pool_attr(op_attr):
    node_attr = {}
    for key in op_attr.keys():
        if key == "round_type":
            round_type = op_attr[key]
            if round_type == "VSI_NN_ROUND_CEIL":
                node_attr["ceil_mode"] = 1
            elif round_type == "VSI_NN_ROUND_FLOOR":
                node_attr["ceil_mode"] = 0
            else:
                assert False, "unsupported pool round type {}".format(round_type)
        elif key == "type":
            continue
        elif key == "ksize":
            node_attr["kernel_shape"] = op_attr[key]
        elif key == "stride":
            node_attr["strides"] = op_attr[key]
        elif key == "pad":
            node_attr["pads"] = op_attr[key]
        else:
            assert False, "unsupported pool attribute {}".format(key)
    return node_attr

def construct_node_attr(node_info):
    node_attr = {}
    if node_info["type"] == "POOL":
        op_attr = node_info["attribute"]["pool"]
        node_attr = construct_pool_attr(op_attr)

    return node_attr

def get_rknn_tensor_info(rknn_model_info, tensor_name):
    tensors_info = rknn_model_info["tensors"]
    for key in tensors_info.keys():
        if key == tensor_name:
            return tensors_info[key]
    assert False, "rknn model have no tensor: {}".format(tensor_name)

def insert_dequant_node(quant_input_tensor_info, onnx_model_info, log_flag):
    tensor_name = quant_input_tensor_info["name"]
    node_name = tensor_name + "_dequant"
    dequant_out_name = tensor_name + "_dequant_out"
    inserted = False
    for node_index in range(len(onnx_model_info["nodes"])):
        node = onnx_model_info["nodes"][node_index]
        if node.name == node_name:
            inserted = True
            break
    if not inserted:
        op_inputs = []
        op_inputs.append(tensor_name)
        op_inputs.append(tensor_name+"_scale")
        op_inputs.append(tensor_name+"_zero_point")
        op_outputs = []
        op_outputs.append(dequant_out_name)
        node = helper.make_node("DequantizeLinear", inputs=op_inputs, outputs=op_outputs, name=node_name)
        onnx_model_info["nodes"].append(node)
        if log_flag:
            print("insert dequant node for {}".format(tensor_name))
    return dequant_out_name

def insert_quant_node(dequant_out_tensor_info, quant_input_name, onnx_model_info, log_flag):
    tensor_name = dequant_out_tensor_info["name"]
    node_name = tensor_name + "_quant"
    op_inputs = []
    op_inputs.append(quant_input_name)
    op_inputs.append(tensor_name+"_scale")
    op_inputs.append(tensor_name+"_zero_point")
    op_outputs = []
    op_outputs.append(tensor_name)
    node = helper.make_node("QuantizeLinear", inputs=op_inputs, outputs=op_outputs, name=node_name)
    onnx_model_info["nodes"].append(node)
    if log_flag:
        print("insert dequant node for {}".format(tensor_name))

def construct_with_dequant_quant_node(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag):
    node_info = rknn_model_info["nodes"][node_index]
    input_names = node_info["inputs"]
    output_names = node_info["outputs"]
    op_inputs = []
    for index in range(len(input_names)):
        op_inputs.append(input_names[index])
    op_outputs = []
    for index in range(len(output_names)):
        op_outputs.append(output_names[index])

    for index in range(len(op_inputs)):
        input_tensor_info = get_rknn_tensor_info(rknn_model_info, op_inputs[index])
        input_dtype = conver_rknn_tensor_type_to_onnx_type(input_tensor_info["dtype"]["vx_type"])
        if input_dtype == TensorProto.UINT8 or \
            input_dtype == TensorProto.INT8:
            quant_input_tensor_info = {}
            quant_input_tensor_info["name"] = op_inputs[index]
            quant_input_tensor_info["dtype"] = input_tensor_info["dtype"]
            dequant_out_name = insert_dequant_node(quant_input_tensor_info, onnx_model_info, log_flag)
            op_inputs[index] = dequant_out_name

    need_quant = []
    quant_inputs = []
    for index in range(len(op_outputs)):
        output_tensor_info = get_rknn_tensor_info(rknn_model_info, op_outputs[index])
        output_dtype = conver_rknn_tensor_type_to_onnx_type(output_tensor_info["dtype"]["vx_type"])
        need_quant.append(False)
        quant_inputs.append(None)
        if output_dtype == TensorProto.UINT8 or \
            output_dtype == TensorProto.INT8:
            need_quant[-1] = True
            op_outputs[index] = op_outputs[index] + "_quant_input"
            quant_inputs[-1] = op_outputs[index]

    node_name = node_info["name"]
    if log_flag:
        print("node_name  : {}".format(node_name))
        print("node_input : {}".format(op_inputs))
        print("node_output: {}".format(op_outputs))

    node_attr = construct_node_attr(node_info)
    if len(node_attr.keys()) > 0:
        node = helper.make_node(onnx_op_name, inputs=op_inputs, outputs=op_outputs, name=node_name, **node_attr)
    else:
        node = helper.make_node(onnx_op_name, inputs=op_inputs, outputs=op_outputs, name=node_name)
    onnx_model_info["nodes"].append(node)

    for index in range(len(need_quant)):
        if need_quant[index]:
            quant_input_name = quant_inputs[index]
            dequant_out_tensor_info = {}
            dequant_out_tensor_info["name"] = output_names[index]
            dequant_out_tensor_info["dtype"] = output_tensor_info["dtype"]
            insert_quant_node(dequant_out_tensor_info, quant_input_name, onnx_model_info, log_flag)

def construct_sigmoid_op(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag):
    node_info = rknn_model_info["nodes"][node_index]
    input_names = node_info["inputs"]
    output_names = node_info["outputs"]
    assert len(input_names) == 1, "rknn sigmoid op should have 1 input"
    assert len(output_names) == 1, "rknn sigmoid op should have 1 output"
    construct_with_dequant_quant_node(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag)
    return True

def construct_add_op(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag):
    node_info = rknn_model_info["nodes"][node_index]
    input_names = node_info["inputs"]
    output_names = node_info["outputs"]
    assert len(input_names) == 2, "rknn add op should have 2 input"
    assert len(output_names) == 1, "rknn add op should have 1 output"
    construct_with_dequant_quant_node(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag)
    return True

def construct_relu_op(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag):
    node_info = rknn_model_info["nodes"][node_index]
    input_names = node_info["inputs"]
    output_names = node_info["outputs"]
    assert len(input_names) == 1, "rknn relu op should have 1 input"
    assert len(output_names) == 1, "rknn relu op should have 1 output"
    construct_with_dequant_quant_node(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag)
    return True

def construct_conv2d_op(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag):
    node_info = rknn_model_info["nodes"][node_index]
    input_names = node_info["inputs"]
    output_names = node_info["outputs"]
    assert len(input_names) == 2 or len(input_names) == 3, "rknn conv2d op should have 2 or 3 input"
    assert len(output_names) == 1, "rknn conv2d op should have 1 output"
    op_inputs = []
    for index in range(len(input_names)):
        if index > 1:
            break
        op_inputs.append(input_names[index])
        op_inputs.append(input_names[index]+"_scale")
        op_inputs.append(input_names[index]+"_zero_point")
    op_inputs.append(output_names[0]+"_scale")
    op_inputs.append(output_names[0]+"_zero_point")
    if len(input_names) == 3:
        op_inputs.append(input_names[2])
        tensor_info = get_rknn_tensor_info(rknn_model_info, input_names[2])
        if tensor_info["dtype"]["zero_point"] != 0:
            print(tensor_info)
    op_outputs = []
    for index in range(len(output_names)):
        op_outputs.append(output_names[index])
    op_attr = node_info["attribute"]["conv2d"]
    if "ksize" in op_attr.keys():
        kernel_shape = op_attr["ksize"]

    if "group" in op_attr.keys():
        group = op_attr["group"]

    if "dilation" in op_attr.keys():
        dilations = op_attr["dilation"]

    if "stride" in op_attr.keys():
        strides = op_attr["stride"]

    if "pad" in op_attr.keys():
        pads = op_attr["pad"]

    node_name = node_info["name"]
    if log_flag:
        print("node_name  : {}".format(node_name))
        print("node_input : {}".format(op_inputs))
        print("node_output: {}".format(op_outputs))
        print("kenel_shape: {}".format(kernel_shape))
        print("group     : {}".format(group))
        print("dilations  : {}".format(dilations))
        print("strides    : {}".format(strides))
        print("pads       : {}".format(pads))

    node = helper.make_node(onnx_op_name, inputs=op_inputs, outputs=op_outputs, name=node_name,
        kernel_shape=kernel_shape,
        group=group,
        dilations=dilations,
        strides=strides,
        pads=pads)
    onnx_model_info["nodes"].append(node)
    return True

def construct_constant_op(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag):
    node_info = rknn_model_info["nodes"][node_index]
    input_names = node_info["inputs"]
    output_names = node_info["outputs"]
    assert len(input_names) == 1, "rknn variable op should have 1 input"
    assert len(output_names) == 1, "rknn variable op should have 1 output"
    # replace node input tensor name(virtural tensor) with virable node input tensor(const tensor), to simulate constant op
    replace_nodes = []
    nodes = rknn_model_info["nodes"]
    for index in range(len(nodes)):
        node = nodes[index]
        for input_index in range(len(node["inputs"])):
            if output_names[0] == node["inputs"][input_index]:
                node["inputs"][input_index] = input_names[0]
                replace_nodes.append(node["name"])
    if log_flag:
        for node in replace_nodes:
            print("replace node {} input with const tensor".format(node))
    return True

def construct_transpose_op(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag):
    node_info = rknn_model_info["nodes"][node_index]
    input_names = node_info["inputs"]
    output_names = node_info["outputs"]
    assert len(input_names) == 1, "rknn permute op should have 1 input"
    assert len(output_names) == 1, "rknn permute op should have 1 output"
    op_inputs = []
    for index in range(len(input_names)):
        op_inputs.append(input_names[index])
    op_outputs = []
    for index in range(len(output_names)):
        op_outputs.append(output_names[index])
    op_attr = node_info["attribute"]["permute"]
    assert "perm" in op_attr.keys(), "rknn permute op should have perm attribute"
    if "perm" in op_attr.keys():
        perm = op_attr["perm"]
    node_name = node_info["name"]
    node = helper.make_node(onnx_op_name,
        inputs=op_inputs,
        outputs=op_outputs,
        name=node_name,
        perm=perm)
    onnx_model_info["nodes"].append(node)
    return True

def construct_reshape_op(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag):
    node_info = rknn_model_info["nodes"][node_index]
    input_names = node_info["inputs"]
    output_names = node_info["outputs"]
    assert len(input_names) == 1, "rknn reshape op should have 1 input"
    assert len(output_names) == 1, "rknn reshape op should have 1 output"
    op_inputs = []
    for index in range(len(input_names)):
        op_inputs.append(input_names[index])
    op_outputs = []
    for index in range(len(output_names)):
        op_outputs.append(output_names[index])
    op_attr = node_info["attribute"]["reshape"]
    assert "size" in op_attr.keys(), "rknn reshape op should have size attribute"
    assert "dim_num" in op_attr.keys(), "rknn reshape op should have dim_num attribute"
    dim_num = op_attr["dim_num"]
    size = op_attr["size"]
    assert dim_num == len(size), "rknn reshape op attribute dim_num not equal to len of attribute size"
    node_name = node_info["name"]
    reshape_shape_tesnor_name = node_name + "_shape"
    reshape_shape_tesnor_type = TensorProto.INT64
    reshape_shape_tesnor_shape = np.array(size, dtype=np.int64).shape
    reshape_shape_tesnor_data = size
    reshape_shape_tesnor = helper.make_tensor(reshape_shape_tesnor_name,
        reshape_shape_tesnor_type,
        reshape_shape_tesnor_shape,
        reshape_shape_tesnor_data)
    onnx_model_info["initializers"].append(reshape_shape_tesnor)
    op_inputs.append(reshape_shape_tesnor_name)
    node = helper.make_node(onnx_op_name,
        inputs=op_inputs,
        outputs=op_outputs,
        name=node_name)
    onnx_model_info["nodes"].append(node)
    return True

def pre_run(onnx_model_info, out_tensor_name):
    ori_outputs = onnx_model_info["outputs"]
    onnx_model_info["outputs"] = []
    out_tensor_value_info = helper.make_tensor_value_info(out_tensor_name, TensorProto.UINT8, [-1,-1,-1,-1])
    onnx_model_info["outputs"].append(out_tensor_value_info)
    time_stamp = time.localtime()
    doc_string = "created on {}".format(time.strftime('%Y-%m-%d %H-%M-%S', time_stamp))
    onnx_initializer_list = onnx_model_info["initializers"]
    onnx_input_list = onnx_model_info["inputs"]
    onnx_output_list = onnx_model_info["outputs"]
    onnx_node_list = onnx_model_info["nodes"]
    onnx_graph_def = helper.make_graph(onnx_node_list, 'rknn_graph', onnx_input_list, onnx_output_list, initializer=onnx_initializer_list, doc_string=doc_string)
    op = onnx.OperatorSetIdProto()
    op.version = 11
    onnx_mode_def = helper.make_model(onnx_graph_def, producer_name='rknn_parser', opset_imports=[op])
    onnx_mode_def.ir_version = 6
    onnx.save_model(onnx_mode_def, "shape_infer.onnx")
    sess = rt.InferenceSession("shape_infer.onnx")
    onnx_input = {}
    for input in sess.get_inputs():
        input_name = input.name
        input_shape = input.shape
        input_dtype = convert_onnxruntime_type_to_numpy_dtype(input.type)
        input_data = np.array(np.zeros(input_shape)).astype(input_dtype)
        onnx_input[input_name] = input_data
    pred_out = sess.run([out_tensor_name,], onnx_input)
    onnx_model_info["outputs"] = ori_outputs
    return list(pred_out[0].shape)

def construct_resize_op(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag):
    node_info = rknn_model_info["nodes"][node_index]
    input_names = node_info["inputs"]
    output_names = node_info["outputs"]
    assert len(input_names) == 1, "rknn resize op should have 1 input"
    assert len(output_names) == 1, "rknn resize op should have 1 output"
    op_inputs = []
    for index in range(len(input_names)):
        op_inputs.append(input_names[index])
    op_outputs = []
    for index in range(len(output_names)):
        op_outputs.append(output_names[index])
    node_name = node_info["name"]
    op_attr = node_info["attribute"]["resize"]
    if op_attr["type"] == 'VSI_NN_INTERPOLATION_NEAREST_NEIGHBOR':
        mode = "nearest"

    roi_init_tensor_name = node_name + "_roi"
    roi_init_tensor_type = TensorProto.FLOAT
    roi_init_tensor_shape = [0]
    roi_init_tensor_data = []
    roi_tesnor_def = helper.make_tensor(roi_init_tensor_name,
        roi_init_tensor_type,
        roi_init_tensor_shape,
        roi_init_tensor_data)
    onnx_model_info["initializers"].append(roi_tesnor_def)
    op_inputs.append(roi_init_tensor_name)

    scales_init_tensor_name = node_name + "_scales"
    scales_init_tensor_type = TensorProto.FLOAT
    scales_init_tensor_shape = [0]
    scales_init_tensor_data = []
    scales_tesnor_def = helper.make_tensor(scales_init_tensor_name,
        scales_init_tensor_type,
        scales_init_tensor_shape,
        scales_init_tensor_data)
    onnx_model_info["initializers"].append(scales_tesnor_def)
    op_inputs.append(scales_init_tensor_name)

    expect_shape = pre_run(onnx_model_info, op_inputs[0])
    sizes_init_tensor_name = node_name + "_sizes"
    sizes_init_tensor_type = TensorProto.INT64
    sizes_init_tensor_shape = [len(expect_shape),]
    sizes_init_tensor_data = expect_shape
    if len(op_attr["size"]) < 4:
        start = len(expect_shape) - len(op_attr["size"])
        sizes_init_tensor_data[start:] = op_attr["size"]

    sizes_tesnor_def = helper.make_tensor(sizes_init_tensor_name,
        sizes_init_tensor_type,
        sizes_init_tensor_shape,
        sizes_init_tensor_data)
    onnx_model_info["initializers"].append(sizes_tesnor_def)
    op_inputs.append(sizes_init_tensor_name)

    node = helper.make_node(onnx_op_name,
        inputs=op_inputs,
        outputs=op_outputs,
        name=node_name,
        mode=mode)
    onnx_model_info["nodes"].append(node)

    # expect_shape = pre_run(onnx_model_info, op_outputs[0])
    return True

def construct_mul_op(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag):
    node_info = rknn_model_info["nodes"][node_index]
    input_names = node_info["inputs"]
    output_names = node_info["outputs"]
    assert len(input_names) == 2, "rknn multiply op should have 2 input"
    assert len(output_names) == 1, "rknn multiply op should have 1 output"
    construct_with_dequant_quant_node(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag)
    return True

def construct_pool_op(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag):
    node_info = rknn_model_info["nodes"][node_index]
    input_names = node_info["inputs"]
    output_names = node_info["outputs"]
    assert len(input_names) == 1, "rknn pool op should have 1 input"
    assert len(output_names) == 1, "rknn pool op should have 1 output"
    construct_with_dequant_quant_node(rknn_op_name, onnx_op_name, rknn_model_info, node_index, onnx_model_info, log_flag)
    return True

def get_actual_onnx_name(onnx_op_name, rknn_model_info, node_index):
    node_info = rknn_model_info["nodes"][node_index]
    if onnx_op_name == "Pool":
        op_attr = node_info["attribute"]
        if op_attr["pool"]["type"] == "VX_CONVOLUTIONAL_NETWORK_POOLING_MAX":
            return "MaxPool"
        elif op_attr["pool"]["type"] == "VX_CONVOLUTIONAL_NETWORK_POOLING_AVG":
            return "AveragePool"
        else:
            assert False, "unspported {} pool type".format(op_attr["pool"]["type"])
    return onnx_op_name

class ConstructOnnxOp(object):
    op_construct_funcs = {}
    rknn_op_onnx_op_map = {}
    def register(self, rknn_op_name, onnx_op_name, op_func):
        if rknn_op_name not in self.rknn_op_onnx_op_map.keys():
            self.rknn_op_onnx_op_map[rknn_op_name] = onnx_op_name
            self.op_construct_funcs[onnx_op_name] = op_func
        else:
            print("already register {}".format(rknn_op_name))

    def construct_node(self, rknn_model_info, node_index, onnx_model_info, log_flag = False):
        node_info = rknn_model_info["nodes"][node_index]
        rknn_op_name = node_info['type']
        if rknn_op_name not in self.rknn_op_onnx_op_map.keys():
            print("have not register {}".format(rknn_op_name))
            return False
        else:
            onnx_op_name = self.rknn_op_onnx_op_map[rknn_op_name]
            actual_onnx_op_name = get_actual_onnx_name(onnx_op_name, rknn_model_info, node_index)
            if log_flag:
                print("{}: --------------- convert {} to {}".format(node_index, rknn_op_name, actual_onnx_op_name))
                node_input_list = node_info['input']
                node_output_list = node_info['output']
                for index in range(len(node_input_list)):
                    input_name = node_input_list[index]
                    print("input {}: {}".format(index, input_name))
                for index in range(len(node_output_list)):
                    output_name = node_output_list[index]
                    print("output {}: {}".format(index, output_name))

            convert_flag = self.op_construct_funcs[onnx_op_name](
                rknn_op_name,
                actual_onnx_op_name,
                rknn_model_info,
                node_index,
                onnx_model_info,
                log_flag)
            return convert_flag

def init_onnx_construct_obj():
    onnx_obj = ConstructOnnxOp()
    onnx_obj.register('SIGMOID', 'Sigmoid', construct_sigmoid_op)
    onnx_obj.register('ADD', 'Add', construct_add_op)
    onnx_obj.register('RELU', 'Relu', construct_relu_op)
    onnx_obj.register('CONV2D', 'QLinearConv', construct_conv2d_op)
    onnx_obj.register('VARIABLE', 'Constant', construct_constant_op)
    onnx_obj.register('PERMUTE', 'Transpose', construct_transpose_op)
    onnx_obj.register('RESHAPE', 'Reshape', construct_reshape_op)
    onnx_obj.register('RESIZE', 'Resize', construct_resize_op)
    onnx_obj.register('MULTIPLY', 'Mul', construct_mul_op)
    onnx_obj.register("POOL", "Pool", construct_pool_op)
    return onnx_obj

def constrtut_graph_initializer(rknn_model_info, layout="WHCN", log_flag=False):
    initializer_list = []
    tensor_info = rknn_model_info["tensors"]
    for tensor_name in tensor_info.keys():
        onnx_const_tensor_list = []
        tensor = tensor_info[tensor_name]
        if "norm_tensor" in tensor_name:
            continue
        onnx_dtype = conver_rknn_tensor_type_to_onnx_type(tensor["dtype"]["vx_type"])
        np_dtype = conver_rknn_tensor_type_to_numpy_type(tensor["dtype"]["vx_type"])
        if "const_tensor" in tensor_name:
            # init weight data
            shape = tensor["shape"][::-1]
            weight_data_arr = np.frombuffer(tensor["raw_data"], dtype=np_dtype).reshape(shape)
            # if layout == "WHCN":
            #     shape_len = len(tensor["shape"])
            #     transpose_axes = [shape_len - i - 1 for i in range(shape_len)]
            #     weight_data_arr = np.transpose(weight_data_arr, transpose_axes)

            onnx_const_tensor = {}
            onnx_const_tensor["name"] = tensor_name
            onnx_const_tensor["type"] = onnx_dtype
            onnx_const_tensor["shape"] = weight_data_arr.shape
            onnx_const_tensor["data"] = weight_data_arr
            onnx_const_tensor_list.append(onnx_const_tensor)

        # init scale
        if isinstance(tensor["dtype"]["scale"], list):
            scale_data_arr = np.array(tensor["dtype"]["scale"]).astype(np.float32)
        else:
            scale_data_arr = np.array([tensor["dtype"]["scale"],]).astype(np.float32)
        # init zero point
        if isinstance(tensor["dtype"]["zero_point"], list):
            zero_point_data_arr = np.array(tensor["dtype"]["zero_point"]).astype(np_dtype)
        else:
            zero_point_data_arr = np.array([tensor["dtype"]["zero_point"],]).astype(np_dtype)

        onnx_const_tensor = {}
        onnx_const_tensor["name"] = tensor_name + "_scale"
        onnx_const_tensor["type"] = TensorProto.FLOAT
        onnx_const_tensor["shape"] = scale_data_arr.shape
        onnx_const_tensor["data"] = scale_data_arr
        onnx_const_tensor_list.append(onnx_const_tensor)

        onnx_const_tensor = {}
        onnx_const_tensor["name"] = tensor_name + "_zero_point"
        onnx_const_tensor["type"] = onnx_dtype
        onnx_const_tensor["shape"] = zero_point_data_arr.shape
        onnx_const_tensor["data"] = zero_point_data_arr
        onnx_const_tensor_list.append(onnx_const_tensor)

        for index in range(len(onnx_const_tensor_list)):
            init_tensor_name = onnx_const_tensor_list[index]["name"]
            init_tensor_type = onnx_const_tensor_list[index]["type"]
            init_tensor_shape = onnx_const_tensor_list[index]["shape"]
            init_tensor_data = onnx_const_tensor_list[index]["data"].flatten().tolist()
            if log_flag:
                print("construct const tensor with:")
                print("tensor name: {}".format(init_tensor_name))
                print("tensor shape: {}".format(init_tensor_shape))
                print("tensor type: {}".format(init_tensor_type))
                print("tensor data: {}".format(init_tensor_data))
            tesnor_def = helper.make_tensor(init_tensor_name, init_tensor_type, init_tensor_shape, init_tensor_data)
            initializer_list.append(tesnor_def)
    return initializer_list

def constrtut_graph_input_output(rknn_model_info, layout="WHCN", log_flag=False):
    input_list = []
    output_list = []
    initializer_list = []
    tensor_info = rknn_model_info["tensors"]
    input_names = []
    for input in rknn_model_info["inputs"]:
        input_names.append(input["tensor_info"]["url"])
    output_names = []
    for output in rknn_model_info["outputs"]:
        output_names.append(output["tensor_info"]["url"])
    for tensor_name in tensor_info.keys():
        tensor = tensor_info[tensor_name]
        if "norm_tensor" in tensor_name:
            onnx_dtype = conver_rknn_tensor_type_to_onnx_type(tensor["dtype"]["vx_type"])
            np_dtype = conver_rknn_tensor_type_to_numpy_type(tensor["dtype"]["vx_type"])
            # init scale
            if isinstance(tensor["dtype"]["scale"], list):
                scale_data_arr = np.array(tensor["dtype"]["scale"]).astype(np.float32)
            else:
                scale_data_arr = np.array([tensor["dtype"]["scale"],]).astype(np.float32)
            # init zero point
            if isinstance(tensor["dtype"]["zero_point"], list):
                zero_point_data_arr = np.array(tensor["dtype"]["zero_point"]).astype(np_dtype)
            else:
                zero_point_data_arr = np.array([tensor["dtype"]["zero_point"],]).astype(np_dtype)

            if layout == "WHCN":
                tensor_shape = tensor["shape"][::-1]
            onnx_norm_tensor_value_info = helper.make_tensor_value_info(tensor_name, onnx_dtype, tensor_shape)
            if tensor['url'] in input_names:
                input_list.append(onnx_norm_tensor_value_info)
            elif tensor['url'] in output_names:
                output_list.append(onnx_norm_tensor_value_info)
            else:
                assert False, "unrecognize norm tensor {}".format(tensor_name)
            if log_flag:
                print("construct norm tensor with:")
                print("tensor name: {}".format(tensor_name))
                print("tensor shape: {}".format(tensor_shape))
                print("tensor type: {}".format(onnx_dtype))
            onnx_const_tensor_list = []
            onnx_const_tensor = {}
            onnx_const_tensor["name"] = tensor_name + "_scale"
            onnx_const_tensor["type"] = TensorProto.FLOAT
            onnx_const_tensor["shape"] = scale_data_arr.shape
            onnx_const_tensor["data"] = scale_data_arr
            onnx_const_tensor_list.append(onnx_const_tensor)

            onnx_const_tensor = {}
            onnx_const_tensor["name"] = tensor_name + "_zero_point"
            onnx_const_tensor["type"] = onnx_dtype
            onnx_const_tensor["shape"] = zero_point_data_arr.shape
            onnx_const_tensor["data"] = zero_point_data_arr
            onnx_const_tensor_list.append(onnx_const_tensor)
            for index in range(len(onnx_const_tensor_list)):
                init_tensor_name = onnx_const_tensor_list[index]["name"]
                init_tensor_type = onnx_const_tensor_list[index]["type"]
                init_tensor_shape = onnx_const_tensor_list[index]["shape"]
                init_tensor_data = onnx_const_tensor_list[index]["data"].flatten().tolist()
                if log_flag:
                    print("construct const tensor with:")
                    print("tensor name: {}".format(init_tensor_name))
                    print("tensor shape: {}".format(init_tensor_shape))
                    print("tensor type: {}".format(init_tensor_type))
                    print("tensor data: {}".format(init_tensor_data))
                tesnor_def = helper.make_tensor(init_tensor_name, init_tensor_type, init_tensor_shape, init_tensor_data)
                initializer_list.append(tesnor_def)
    return input_list, output_list, initializer_list

def constrtut_graph_node(rknn_model_info, onnx_model_info, onnx_construct_obj, log_flag=False):
    for index in range(len(rknn_model_info["nodes"])):
        convert_flag = onnx_construct_obj.construct_node(rknn_model_info, index, onnx_model_info, log_flag)
        assert convert_flag, "convert node {} fail!!!!".format(rknn_model_info["nodes"][index]["name"])

def construct_onnx_from_rknn_model_info(rknn_model_info, onnx_construct_obj, layout="WHCN", log_flag=False):
    graph_initializer = constrtut_graph_initializer(rknn_model_info, layout, log_flag)
    graph_input, graph_output, graph_input_output_initializer = constrtut_graph_input_output(rknn_model_info, layout, log_flag)
    graph_initializer.extend(graph_input_output_initializer)
    onnx_model_info = {}
    onnx_model_info["initializers"] = graph_initializer
    onnx_model_info["inputs"] = graph_input
    onnx_model_info["outputs"] = graph_output
    onnx_model_info["nodes"] = []
    constrtut_graph_node(rknn_model_info, onnx_model_info, onnx_construct_obj, log_flag)
    time_stamp = time.localtime()
    doc_string = "created on {}".format(time.strftime('%Y-%m-%d %H-%M-%S', time_stamp))
    onnx_initializer_list = onnx_model_info["initializers"]
    onnx_input_list = onnx_model_info["inputs"]
    onnx_output_list = onnx_model_info["outputs"]
    onnx_node_list = onnx_model_info["nodes"]
    onnx_graph_def = helper.make_graph(onnx_node_list, 'rknn_graph', onnx_input_list, onnx_output_list, initializer=onnx_initializer_list, doc_string=doc_string)
    op = onnx.OperatorSetIdProto()
    op.version = 11
    onnx_mode_def = helper.make_model(onnx_graph_def, producer_name='rknn_parser', opset_imports=[op])
    onnx_mode_def.ir_version = 6
    checker.check_model(onnx_mode_def)
    if log_flag:
        print('Model Graph:\n\n{}'.format(helper.printable_graph(onnx_mode_def.graph)))
    return onnx_mode_def

def format_onnx_nodes(rknn_model_info, layout):
    nodes_info = rknn_model_info["nodes"]
    for node in nodes_info:
        if node["type"] == "RESIZE":
            node_attr = node["attribute"]["resize"]
            if layout == "WHCN":
                node_attr["size"] = node_attr["size"][::-1]

def convert_rknn_to_onnx(rknn_model_info, save_file="convert.onnx", layout="WHCN", log_flag=False):
    construct_onnx_obj = init_onnx_construct_obj()
    format_onnx_nodes(rknn_model_info, layout)
    onnx_model = construct_onnx_from_rknn_model_info(rknn_model_info, construct_onnx_obj, layout, log_flag)
    onnx.save_model(onnx_model, save_file)


"""
Tornado handler to edit/delete a log upload entry
"""
from __future__ import print_function
import os
from html import escape
import sqlite3
import sys
import tornado.web

# this is needed for the following imports
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../plot_app'))
from config import get_db_filename, get_kml_filepath, get_overview_img_filepath
from helper import clear_ulog_cache, get_log_filename

#pylint: disable=relative-beyond-top-level
from .common import get_jinja_env

EDIT_TEMPLATE = 'edit.html'

#pylint: disable=abstract-method

class EditEntryHandler(tornado.web.RequestHandler):
    """ Edit a log entry, with confirmation (currently only delete) """

    def get(self, *args, **kwargs):
        """ GET request """
        log_id = escape(self.get_argument('log'))
        action = self.get_argument('action')
        confirmed = self.get_argument('confirm', default='0')
        token = escape(self.get_argument('token'))

        if action == 'delete':
            if confirmed == '1':
                if self.delete_log_entry(log_id, token):
                    content = """

Log File deleted

Successfully deleted the log file.

"""
                else:
                    content = """

Failed

Failed to delete the log file.

"""
            else: # request user to confirm
                # use the same url, just append 'confirm=1'
                delete_url = self.request.path+'?action=delete&log='+log_id+ \
                        '&token='+token+'&confirm=1'
                content = """

Delete Log File

Click here to confirm and delete the log {log_id}.

""".format(delete_url=delete_url, log_id=log_id)
        else:
            raise tornado.web.HTTPError(400, 'Invalid Parameter')

        template = get_jinja_env().get_template(EDIT_TEMPLATE)
        self.write(template.render(content=content))

    @staticmethod
    def delete_log_entry(log_id, token):
        """
        delete a log entry (DB & file), validate token first

        :return: True on success
        """
        con = sqlite3.connect(get_db_filename(), detect_types=sqlite3.PARSE_DECLTYPES)
        cur = con.cursor()
        cur.execute('select Token from Logs where Id = ?', (log_id,))
        db_tuple = cur.fetchone()
        if db_tuple is None:
            return False
        if token != db_tuple[0]: # validate token
            return False

        # kml file
        kml_path = get_kml_filepath()
        kml_file_name = os.path.join(kml_path, log_id.replace('/', '.')+'.kml')
        if os.path.exists(kml_file_name):
            os.unlink(kml_file_name)

        #preview image
        preview_image_filename = os.path.join(get_overview_img_filepath(), log_id+'.png')
        if os.path.exists(preview_image_filename):
            os.unlink(preview_image_filename)

        log_file_name = get_log_filename(log_id)
        print('deleting log entry {} and file {}'.format(log_id, log_file_name))
        os.unlink(log_file_name)
        cur.execute("DELETE FROM LogsGenerated WHERE Id = ?", (log_id,))
        cur.execute("DELETE FROM Logs WHERE Id = ?", (log_id,))
        con.commit()
        cur.close()
        con.close()

        # need to clear the cache as well
        clear_ulog_cache()

        return True