from module import obo2csv
from configure import obo_url,num_threads
excludeGO = "GO:0005515,GO:0005488,GO:0003674,GO:0008150,GO:0005575"
#excludeGO = ""
import sys
import os
import multiprocessing as mp
from multiprocessing import Pool
from contextlib import closing

def get_obo_dict():  # read obo_dict

    fp = open(obo_url, 'r')
    obo_txt = fp.read()
    fp.close()
    obo_dict = obo2csv.parse_obo_txt(obo_txt)
    return obo_dict

## global variables for multiprocessing
obo_dict = get_obo_dict()

def get_obsolete(obo_dict): # get obsolete terms

    return obo_dict.obsolete()

def find_parents(obo_dict, GOterm): # find parents

    obsolete_list=get_obsolete(obo_dict)

    if(GOterm in obsolete_list):
        return []
    else:
        if(obo_dict.is_a(GOterm, direct=False, name=False, number=False)==""):
            return []
        parent_term_list = obo_dict.is_a(GOterm, direct=False, name=False, number=False).split()
        return list(set(parent_term_list)-set(excludeGO.split(",")))

def find_parents_from_file(originfile, new_file):  # find parents from file

    f=open(originfile, "r")
    text=f.read()
    f.close()

    type=""
    temp_dict=dict()

    for line in text.splitlines():
        values=line.strip().split()
        term=values[0]
        type=values[1]
        pro=float(values[2])
        if (pro >= 0.01):
            temp_dict[term] = pro

    new_dict = dict()
    for term in temp_dict:
        parent_list = find_parents(obo_dict, term)
        for parent in parent_list:
            if (parent not in new_dict or temp_dict[term] > new_dict[parent]):
                new_dict[parent] = temp_dict[term]

    # sort the new_dict
    result_list = [(new_dict[term], term) for term in new_dict]
    result_list = sorted(result_list, reverse=True)

    f = open(new_file, "w")
    for value, term in result_list:
        f.write(term + " " + type + " " + str(value) + "\n")
    f.flush()
    f.close()

    # f=open(new_file, "w")
    # for term in new_dict:
    #     f.write(term+" "+type+" "+str(new_dict[term])+"\n")
    # f.flush()
    # f.close()

def sort_result(originfile):  # sort result

    f = open(originfile, "r")
    text = f.read()
    f.close()

    temp_dict = dict()

    for line in text.splitlines():
        values = line.strip().split()
        term = values[0]
        type = values[1]
        pro = float(values[2])
        if(pro>=0.01):
            temp_dict[term] = pro

    result_list=[(temp_dict[term], term) for term in temp_dict]
    result_list = sorted(result_list, reverse=True)

    f=open(originfile, "w")
    for value, term in result_list:
        f.write(term+" "+type+" "+str(value)+"\n")
    f.flush()
    f.close()

def multiprocess_find_parents_from_file(args_list):  # multiprocess
    result_file, result_file_new = args_list
    find_parents_from_file(result_file, result_file_new)

if __name__=="__main__":

    workdir = sys.argv[1]
    type = sys.argv[2]
    name_list = os.listdir(workdir)

    args_list = []
    for name in name_list:
        result_file = workdir+"/"+name+"/"+sys.argv[3]+"_"+type
        result_file_new = workdir+"/"+name+"/"+sys.argv[3]+"_"+type+"_new"
        args_list.append((result_file, result_file_new))
        #find_parents_from_file(result_file, result_file+"_new", obo_dict)

    with closing(Pool(processes=num_threads)) as pool:
        pool.map(multiprocess_find_parents_from_file, args_list)
        pool.close()
        pool.join()


from kivy.app import App
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.label import Label
from kivy.core.window import Window
from kivy.graphics import Color, Line, Ellipse, Rectangle

Window.set_system_cursor('crosshair')

class MousePosDemo(BoxLayout):

    def __init__(self, **kwargs):
        super(MousePosDemo, self).__init__(**kwargs)

        Window.bind(mouse_pos=self.mouse_pos)
        # Window.bind(on_motion=self.on_motion)

        self.bind(pos=self.draw)
        self.bind(size=self.draw)
        self.layout1 = BoxLayout(opacity=1)
        self.label = Label()
        self.add_widget(self.layout1)
        self.add_widget(self.label)

    def draw(self, *args):

        self.layout1.canvas.clear()
        with self.canvas.before:
            Color(0.6, 0.6, 0.6, 1)
            self.bg = Rectangle(pos=self.pos, size=self.size)

    def mouse_pos(self, window, pos):
        self.label.text = str(pos)

    # def on_motion(self, obj, etype, me):
    #     pass
    #     # self.label.text = str(me.pos)

    def on_touch_down(self, event):
        self.label.text = "touch DOWN: " + str(event.pos)

    def on_touch_up(self, event):
        self.label.text = "touch UP: " + str(event.pos)

class TestApp(App):
    title = "Kivy Mouse Pos Demo"

    def build(self):
        return MousePosDemo()

if __name__ == "__main__":
    TestApp().run()


import json
import matplotlib.pyplot as plt
import torch
import step7_model
import numpy as np
import sys
from sklearn.cluster import DBSCAN

np.set_printoptions(threshold=sys.maxsize)

def draw_loss_pic():  # 画出5次迭代的损失值曲线图
    loss = []
    for i in range(5):  # 遍历每一个损失文件
        path = "triplet_model_ckp/losses_Epoch{}.json".format(str(i))
        with open(path, "r", encoding="UTF-8") as rf:
            loss_part = json.load(rf)[str(i)]
            for index, e in enumerate(loss_part):
                if index % 1000 == 0:
                    loss.append(e / 16)
    x = range(len(loss))

    plt.plot(x, loss, color="blue")
    plt.show()

draw_loss_pic()

def transform_data(model, flag):  # 将数据传入model，输出的数据存储在文件里
    if flag == "train":  # 转化训练集
        with open("word2vec_result/trainpub_word2vec_result.json", "r", encoding="UTF-8") as rf:
            vec_data = json.load(rf)
    if flag == "vali":
        with open("word2vec_result/valipub_word2vec_result.json", "r", encoding="UTF-8") as rf:
            vec_data = json.load(rf)

    transformed_dict = {}  # 转化后的字典 key是pubid，value是转化后的数值
    count = 0
    l = len(vec_data)
    for pub_id in vec_data:
        pub_vec = vec_data[pub_id]  # 需要去转化的word2vec向量
        tran_result = model(torch.tensor(pub_vec)).detach().numpy().tolist()  # 转化后的结果
        transformed_dict[pub_id] = tran_result
        count = count + 1
        print("\r", "Trans {} Loading >>> {}%".format(flag, 100 * count / l), end="", flush=True)

    if flag == "train":
        with open("triplet_traned_result/triplet_train.json", "w", encoding="UTF-8") as wf:
            json.dump(transformed_dict, wf, indent=4)
    if flag == "vali":
        with open("triplet_traned_result/triplet_vali.json", "w", encoding="UTF-8") as wf:
            json.dump(transformed_dict, wf, indent=4)

def triplet_feature2pubid(flag, triplet_feature):  # 通过triplet net后的特征找到对应的pubid
    if flag == "train":  # 训练集
        with open("triplet_traned_result/triplet_train.json", "r", encoding="UTF-8") as rf:
            triplet_feature_dict = json.load(rf)
    if flag == "vali":  # 训练集
        with open("triplet_traned_result/triplet_vali.json", "r", encoding="UTF-8") as rf:
            triplet_feature_dict = json.load(rf)
    for pub_id in triplet_feature_dict:
        if triplet_feature_dict[pub_id] == triplet_feature:  # 找到了
            return pub_id

# if __name__ == '__main__':
#     # 加载模型
#     model = step7_model.LogisticRegression(100, 64)
#     torch.load("triplet_model_ckp/tripletLoss_model_Epoch4.pth")
#     model.load_state_dict(torch.load("triplet_model_ckp/tripletLoss_model_Epoch4.pth")["model"])
#     # print(torch.load("triplet_model_ckp/tripletLoss_model_Epoch4.pth")["model"]["b1"].numpy())
#     # print(torch.load("triplet_model_ckp/tripletLoss_model_Epoch4.pth")["model"]["b2"].numpy())
#     model.eval()
#     # 将vec 100维的数据通过model转换为64维的数据，写入文件中
#     transform_data(model, "train")
#     transform_data(model, "vali")
#     #进行聚类分析

论文作者冷启动同名消岐/code/方法1：参考论文复现/step9_drawloss.py


import datetime
from PyQt4.QtCore import Qt
from PyQt4.QtGui import QColor, QGraphicsDropShadowEffect, QLabel

class Log():

	def __init__(self, name="MAIN()", level="error"):
		self.level = level
		self.name = name

	def log(self, message, lvl):
		ts = "{:%x %X} ".format(datetime.datetime.now())
		nm = "{:<20}".format(self.name)

		if lvl == "debug":
			tag = "{:^16}".format("DEBUG")
		elif lvl == "info":
			tag = "{:^16}".format("INFO")
		elif lvl == "warning":
			tag = "{:^16}".format("-WARNING-")
		elif lvl == "error":
			tag = "{:*^16}".format("ERROR")

		if self.level == "debug":
			print (ts + nm + tag + message)

		elif self.level == "info":
			if lvl == "info" or lvl == "warning" or lvl == "error":
				print (ts + nm + tag + message)

		elif self.level == "warning":
			if lvl == "warning" or lvl == "error":
				print (ts + nm + tag + message)

		elif self.level == "error":
			if lvl == "error":
				print (ts + nm + tag + message)

	def debug(self, message):
		self.log(message, "debug")

	def info(self, message):
		self.log(message, "info")

	def warning(self, message):
		self.log(message, "warning")

	def error(self, message):
		self.log(message, "error")

def align(gridnum):
	if gridnum == 1:
		return (Qt.AlignLeft | Qt.AlignTop)
	elif gridnum == 2:
		return (Qt.AlignHCenter | Qt.AlignTop)
	elif gridnum == 3:
		return (Qt.AlignRight | Qt.AlignTop)
	elif gridnum == 4:
		return (Qt.AlignLeft | Qt.AlignVCenter)
	elif gridnum == 5:
		return (Qt.AlignHCenter | Qt.AlignVCenter)
	elif gridnum == 6:
		return (Qt.AlignRight | Qt.AlignVCenter)
	elif gridnum == 7:
		return (Qt.AlignLeft | Qt.AlignBottom)
	elif gridnum == 8:
		return (Qt.AlignHCenter | Qt.AlignBottom)
	elif gridnum == 9:
		return (Qt.AlignRight | Qt.AlignBottom)
	else:
		return (Qt.AlignHCenter | Qt.AlignVCenter)

def createTextLabel(page, frameName, properties):
	textLabel = QLabel(page)
	textLabel.setObjectName(frameName+properties['name'])
	textLabel.setStyleSheet("#"+frameName+properties['name']+"{ font-family:"+properties['font']+"; color: "+
		properties['color'] + "; background-color: transparent; font-size: "+
		properties['fontsize']+"px; "+
		properties['fontattr']+"}")
	textLabel.setAlignment(align(properties['alignment']))
	if properties['shadow']:
		textEffect = QGraphicsDropShadowEffect()
		textEffect.setOffset(properties['shadow']['offset'])
		textEffect.setBlurRadius(properties['shadow']['blur'])
		textEffect.setColor(QColor(properties['shadow']['color']))
		textLabel.setGraphicsEffect(textEffect)
	textLabel.setGeometry(properties['location'][0], properties['location'][1], properties['location'][2], properties['location'][3])

	return textLabel

def createImageLabel(page, frameName, properties):
	imageLabel = QLabel(page)
	imageLabel.setObjectName(frameName+properties['name'])
	imageLabel.setStyleSheet("#"+frameName+properties['name']+" { background-color: transparent; }")
	if properties['shadow']:
		textEffect = QGraphicsDropShadowEffect()
		textEffect.setOffset(properties['shadow']['offset'])
		textEffect.setBlurRadius(properties['shadow']['blur'])
		textEffect.setColor(QColor(properties['shadow']['color']))
		imageLabel.setGraphicsEffect(textEffect)
	imageLabel.setGeometry(properties['location'][0], properties['location'][1], properties['location'][2], properties['location'][3])

	return imageLabel


from .base import BaseDataset
import numpy as np
import os
import torch
import pandas as pd
import copy
from utils.tools import get_sub_luna_point,get_sub_sun_point, dprint
from utils.timefeatures import time_features
from .normlizer import PreGauessNormlizer,PreUnitNormlizer,NoneNormlizer,PostGauessNormlizer,PostUnitNormlizer,TimewiseNormlizer
from tqdm.auto import tqdm
from utils.tools import get_center_around_indexes, get_center_around_indexes_3D
import json
from .utils import load_numpy_from_url

def get_init_file_list(dates):
    """
    ---> We will require the dates via np.datetime64 format
    ---> Old code use year list below
    # file_list = []
    # for year in years:
    #     if year == 1979: # 1979年数据只有8753个，缺少第一天前7小时数据，所以这里我们从第二天开始算起
    #         for hour in range(17, 8753, 1):
    #             file_list.append([year, hour])
    #     else:
    #         if year % 4 == 0:
    #             max_item = 8784
    #         else:
    #             max_item = 8760
    #         for hour in range(0, max_item, 1):
    #             file_list.append([year, hour])
    # return file_list
    """
    years          = dates.astype('datetime64[Y]').astype(int)   ## start from 1970, thus 1979 is 9
    start_of_years = years.astype('datetime64[Y]')
    hours          = (dates - start_of_years) / np.timedelta64(1, 'h')
    hours[years == 9] -= 7 # 1979 data lake 7 hours
    hours = np.maximum(hours, 0)  # ensure no negative hour values
    file_list      = np.vstack((years+1970, hours)).T.astype(int)
    return file_list

def get_default_timestamp_pool(config):
    time_unit = config.get('time_unit', 1 )
    return {'train':np.arange(np.datetime64("1979-01-02"), np.datetime64("2017-01-01"), np.timedelta64(time_unit, "h")),
            'valid':np.arange(np.datetime64("2017-01-01"), np.datetime64("2018-01-01"), np.timedelta64(time_unit, "h")),
            'test' :np.arange(np.datetime64("2018-01-01"), np.datetime64("2019-01-01"), np.timedelta64(time_unit, "h")),
            'ftest': np.arange(np.datetime64("2018-01-01"), np.datetime64("2019-01-01"), np.timedelta64(time_unit, "h")),
            }

def read_channel_list_file(file_path):
    with open(file_path, 'r') as f:
        channel_list = json.load(f)
    return channel_list

def get_timestamp_date(split, config):
    timestamps  = config.get('timestamps_list', None)
    defaultPool = get_default_timestamp_pool(config)
    time_unit   = config.get('time_unit', 1 )
    if timestamps is None:
        timestamps = defaultPool[split]
        dprint(f"you don't assign a timestamp list for {split}, then we use the default timestamp config  per {time_unit} hour")
    else:
        dprint(f"noice you assign your own year timestamps list from {timestamps.min} to {timestamps.max()}" )
    return timestamps  # [[year,idx],[year,idx],.....]

def get_vnames_from_config(config):
    channel_name_list = config.get('channel_name_list')
    if isinstance(channel_name_list, str):
        channel_name_list = read_channel_list_file(channel_name_list)
    elif isinstance(channel_name_list, list):
        channel_name_list = channel_name_list
    else:
        raise NotImplementedError(f"channel_name_list {channel_name_list} must be filepath or list of channel name")
    return channel_name_list

def preload_full_dataset(split, config):
    '''
    This will load the whole dataset into memory, it will be very slow and consume a lot of memory
    '''

    root           = config.get('root')
    ### first check whether the train.npy/test.npy/valid.npy is exist
    fast_offline_data_name = os.path.join(root, f"{split}.npy")
    if os.path.exists(fast_offline_data_name):
        dprint(f"detect fast offline data in {fast_offline_data_name}, use this")
        dataset_tensor     = torch.Tensor(np.load(fast_offline_data_name))
        record_load_tensor = torch.ones(len(dataset_tensor))
        return dataset_tensor,record_load_tensor
    else:
        init_file_list = get_init_file_list(get_timestamp_date(split, config))
        dprint(f"dont find offline data in {fast_offline_data_name}")
        dprint("then seek split file store in format {root}/{year}/{year}-{hour:04d}.npy")
        loaded_tensor  = None
        for i, (year, hour) in tqdm(enumerate(init_file_list)):
            url = f"{root}/{year}/{year}-{hour:04d}.npy"
            odata = load_numpy_from_url(None,url)
            if loaded_tensor is None:
                assert len(init_file_list)*np.prod(odata.shape) < 2**32, "The dataset is too large, we can't load it into memory"
                loaded_tensor = torch.empty(len(init_file_list), *odata.shape)
            loaded_tensor[i] = torch.from_numpy(odata)
        dprint("concat!")
        loaded_tensor  = torch.stack(loaded_tensor)
        loaded_flag    = torch.ones(len(loaded_tensor))
        dprint("finish preload the whole dataset into memory")
        return loaded_tensor, loaded_flag

def get_the_dataset_flag(config):
    channel_picked_name = os.path.split(config.get('channel_name_list'))[-1].split('.')[0]
    normlizer_name      = config.get('normlized_flag','N')
    return channel_picked_name+normlizer_name

class WeatherBenchBase(BaseDataset):
    time_unit  = 1

    single_vnames = [ "2m_temperature", "10m_u_component_of_wind", "10m_v_component_of_wind", "total_cloud_cover",
                  "total_precipitation", "toa_incident_solar_radiation"]
    level_vnames= []
    for physics_name in ["geopotential", "temperature",
                            "specific_humidity","relative_humidity",
                            "u_component_of_wind","v_component_of_wind",
                            "vorticity","potential_vorticity"]:
        for pressure_level in [50, 100, 150, 200, 250, 300, 400, 500, 600, 700, 850, 925, 1000]:
            level_vnames.append(f"{pressure_level}hPa_{physics_name}")

    weatherbench_property_name_list = single_vnames + level_vnames
    ## the 110 channel for weatherbench dataset
    cons_vnames= ["lsm", "slt", "orography"]

    def get_channel_choice(self, property_names):
        if isinstance(property_names, (tuple,list)):
            return [self.get_channel_choice(property_name) for property_name in property_names]
        elif isinstance(property_names, str):
            return self.weatherbench_property_name_list.index(property_names)
        else:
            raise NotImplementedError(f"property_names {property_names} must be str or list of str")

class FakeWeatherBench(WeatherBenchBase):
    def __init__(self, split="train", shared_dataset_tensor_buffer=(None, None), with_idx=False, config={}, debug=False):

        self.split = split
        self.with_idx = with_idx
        self.timestamp_date = get_timestamp_date(split, config)
        self.single_data_path_list = get_init_file_list(self.timestamp_date)
        self.time_step = config.time_step
        self.time_intervel = config.time_intervel
        self.resolution_h, self.resolution_w = self.img_size = config.img_size

        self.normlized_flag = config.get('normlized_flag', 'N')
        self.use_time_feature = config.get('use_time_feature', False)
        self.time_reverse_flag = config.get('time_reverse_flag', 'only_forward')
        self.add_LunaSolarDirectly = config.get('add_LunaSolarDirectly', False)
        self.offline_data_is_already_normed = config.get('offline_data_is_already_normed', False)

        self.constant_channel_pick = config.get('constant_channel_pick', None)
        self.make_data_physical_reasonable_mode = config.get('make_data_physical_reasonable_mode', None)

        self.patch_range = patch_range = config.get('patch_range', None)
        self.cross_sample = config.get('cross_sample', True) and ((self.split == 'train') or debug) and (self.patch_range is not None)

        #self.vnames= [self.weatherbench_property_name_list[i] for i in self.channel_choice]
        self.dataset_flag = get_the_dataset_flag(config)

        self.vnames = get_vnames_from_config(config)
        self.channel_choice = np.array(self.get_channel_choice(self.vnames))

    def __len__(self):
        return len(self.single_data_path_list) - self.time_step*self.time_intervel + 1

    def __getitem__(self, idx):
        raise NotImplementedError("You must implement it")

class WeatherBench(WeatherBenchBase):
    '''
    In-memory dataset, if we activate the in memory procedure, please create shared tensor
    via Tensor.share_memory_() before call this module and pass the create tensor as args of this module
    the self.dataset_tensor should be same shape as the otensor, for example (10000, 110, 32, 64)
    the self.record_load_tensor is used to record which row is record.
    This implement will automatively offline otensor after load it once.

    no matter we use in-memory technology or not, we always initialized two tensor,
    one is the whole tensor dataset stored in (B, P, W, H) tensor
    one is the loadQ tensor that judge whether the real data is loaded.
    '''
    default_root = None
    reduce_Field_coef = np.array([1])
    clim_tensor = [0]
    img_shape = (32, 64)
    record_load_tensor = None
    dataset_tensor = None

    def __init__(self, split="train", shared_dataset_tensor_buffer=(None, None),with_idx=False, config={}):
        self.root = config.get('root', self.default_root)
        assert self.root is not None
        dprint(f"use dataset in {self.root} for {split}")
        self.split = split
        self.with_idx=with_idx
        self.timestamp_date=get_timestamp_date(split, config)
        self.single_data_path_list = get_init_file_list(self.timestamp_date)
        self.time_step     = config.time_step
        self.time_intervel = config.time_intervel
        self.resolution_h, self.resolution_w = self.img_size = config.img_size

        self.normlized_flag = config.normlized_flag
        self.use_time_feature = config.use_time_feature
        self.time_reverse_flag = config.time_reverse_flag
        self.add_LunaSolarDirectly = config.add_LunaSolarDirectly
        self.offline_data_is_already_normed = config.offline_data_is_already_normed
        self.constant_channel_pick = config.constant_channel_pick
        self.make_data_physical_reasonable_mode = config.make_data_physical_reasonable_mode

        self.data_processing_is_all_done = False

        #self.vnames= [self.weatherbench_property_name_list[i] for i in self.channel_choice]
        self.dataset_flag = get_the_dataset_flag(config)

        self.vnames = get_vnames_from_config(config)
        self.channel_choice = np.array(self.get_channel_choice(self.vnames))

        self.mean_std = self.load_original_meanstds()  # (2, P, W, H)
        self.normlizer = self.get_normlizer(
            self.mean_std[:, self.channel_choice][..., None, None], self.normlized_flag)

        #self.channel_choice, self.normlizer = self.get_config_from_preset_pool(self.dataset_flag)
        self.mean, self.std = self.normlizer.mean, self.normlizer.std
        self.unit_list      = self.std

        self.constants = torch.from_numpy(self.load_numpy_from_url(
            os.path.join(self.root, "constants.npy")))

        assert ((shared_dataset_tensor_buffer[0] is None and shared_dataset_tensor_buffer[1] is None) or
                (shared_dataset_tensor_buffer[0] is not None and shared_dataset_tensor_buffer[1] is not None))

        if shared_dataset_tensor_buffer[0] is None:
            self.dataset_tensor, self.record_load_tensor = self.create_offline_dataset_templete(
                split, create_buffer=False, config=config)
        else:
            self.dataset_tensor, self.record_load_tensor = shared_dataset_tensor_buffer

        if self.dataset_tensor is not None:
            assert len(self.dataset_tensor) == len(self.timestamp_date), f"{len(self.dataset_tensor)} == {len(self.timestamp_date)}"
            assert self.dataset_tensor.shape[-2] == self.resolution_h
            assert self.dataset_tensor.shape[-1] == self.resolution_w
        if self.single_data_path_list is not None:
            assert len(self.single_data_path_list) == len(self.timestamp_date)

        if self.time_reverse_flag: ## should add this before change vnames
            self.volicity_idx = [i for i, name in enumerate(self.vnames) if (
                ('v_component' in name) or ('u_component' in name))]

        # if self.constant_channel_pick: ### no need change vnames anymore, since we add constant at another key
        #     self.vnames = self.vnames + [self.cons_vnames[i] for i in self.constant_channel_pick]

        if self.use_time_feature:
            self.timestamp_date = time_features(pd.to_datetime(self.timestamp_date)).transpose(1, 0)

        if self.add_LunaSolarDirectly:
            self.LaLotude, self.LaLotudeVector = self.get_mesh_lon_lat(self.resolution_h, self.resolution_w)  # (32, 64 )

    def load_original_meanstds(self):
        mean_std_path = os.path.join(self.root, "mean_std.npy")
        if os.path.exists(mean_std_path):
            return self.load_numpy_from_url(mean_std_path)  # (2, P, W, H)
        else:
            dprint("""
                Warning!!!! The mean std file is not founed, we will use the mean=0 and std=1
                  """)
            return 0, 1

    @staticmethod
    def unique_name(config):
        return f"WeatherBench_{config.get('resolution_h')}x{config.get('resolution_w')}_{get_the_dataset_flag(config)}"

    @staticmethod
    def _add_constant_mean_std(mean_std, channel_num):
        mean, std = mean_std  # (70,1,1)
        mean = np.concatenate([mean, np.zeros(channel_num, *mean.shape[1:])])
        std = np.concatenate([std,  np.ones(channel_num, *std.shape[1:])])
        return np.stack([mean, std])  # (2, 71, 1, 1)

    @staticmethod
    def get_normlizer(mean_std, normlized_flag):

        if normlized_flag[-1] == 'N':
            return PreGauessNormlizer(mean_std)
        elif normlized_flag[-1] == 'U':
            return PreUnitNormlizer(mean_std)
        elif normlized_flag[-1] == 'O':
            return PostGauessNormlizer(mean_std)
        else:
            raise NotImplementedError(
                f"normlized_flag {normlized_flag} is not implemented")

    def get_config_from_preset_pool(self, dataset_flag):
        if dataset_flag == '2D110N':
            return list(range(0, 110)), PreGauessNormlizer(self.mean_std.reshape(2, 110, 1, 1))
        elif dataset_flag == '2D110U':
            return list(range(0, 110)),   PreUnitNormlizer(self.mean_std.reshape(2, 110, 1, 1))
        elif dataset_flag == '2D104N':
            return list(range(6, 110)), PreGauessNormlizer(self.mean_std[:, 6:].reshape(2, 104, 1, 1))
        elif dataset_flag == '2D104U':
            return list(range(6, 110)),   PreUnitNormlizer(self.mean_std[:, 6:].reshape(2, 104, 1, 1))
        elif dataset_flag == '3D104N':
            return self.threeD_channel, PreGauessNormlizer(self.mean_std[:, 6:].reshape(2, 8, 13, 1, 1).mean(2))
        elif dataset_flag == '3D104U':
            return self.threeD_channel,   PreUnitNormlizer(self.mean_std[:, 6:].reshape(2, 8, 13, 1, 1).mean(2))
        elif dataset_flag == '2D110O':
            return list(range(0, 110)),  PostGauessNormlizer(self.mean_std.reshape(2, 110, 1, 1))
        elif dataset_flag == '2D104O':
            return list(range(6, 110)),  PostGauessNormlizer(self.mean_std[:, 6:].reshape(2, 104, 1, 1))
        elif dataset_flag == '3D104O':
            return self.threeD_channel, PostGauessNormlizer(self.mean_std[:, 6:].reshape(2, 8, 13, 1, 1).mean(2))
        else:
            raise NotImplementedError(
                f"dataset_flag {dataset_flag} is not implemented")

    def __len__(self):
        return len(self.single_data_path_list) - self.time_step*self.time_intervel + 1

    @staticmethod
    def create_offline_dataset_templete(split='test', create_buffer=False, fully_loaded=False, config={}):
        fast_offline_data_name = os.path.join(config.get('root'), f"{split}.npy")
        if os.path.exists(fast_offline_data_name):
            create_buffer = fully_loaded = True
        if not create_buffer:return None, None
        if fully_loaded:
            return preload_full_dataset(split, config)
        else:
            ### will dynamic load during the iteration
            init_file_list = get_init_file_list(get_timestamp_date(split, config))
            vnames = get_vnames_from_config(config)
            batches = len(init_file_list)
            h  = config.get('resolution_h')
            w  = config.get('resolution_w')
            ### this create the whole buffer
            assert batches*len(vnames)*h*w < 2**32, "The dataset is too large, we can't load it into memory"
            return (torch.empty(batches, len(vnames), h , w),
                    torch.zeros(batches))

    def load_otensor(self, idx):
        '''
        This will load the whole tensor of timestamp into memory. For very large snapshot, it may be inefficient.
        '''
        if (self.record_load_tensor is None) or (not self.record_load_tensor[idx]):
            year, hour = self.single_data_path_list[idx]
            url = f"{self.root}/{year}/{year}-{hour:04d}.npy"
            odata = torch.from_numpy(self.load_numpy_from_url(url))
            if self.record_load_tensor is not None:
                '''
                <--- load data into shared buffer when dataset iteration
                '''
                self.record_load_tensor[idx] = 1
                self.dataset_tensor[idx] = odata

        if self.record_load_tensor is not None:
            # .clone() <--- be careful that do not inplace modify the source data
            odata =  self.dataset_tensor[idx]

        return odata

    def generate_runtime_data(self, idx, reversed_part=False):
        if self.data_processing_is_all_done:
            return self.dataset_tensor[idx]
        odata = self.load_otensor(idx)
        assert odata.shape[-2] == self.resolution_h
        assert odata.shape[-1] == self.resolution_w
        if isinstance(odata, torch.Tensor) and isinstance(self.channel_choice, np.ndarray):
            self.channel_choice = torch.LongTensor(self.channel_choice)
        data = odata[self.channel_choice]  # (110, W,H )->(P,W,H)
        data = self.make_data_physical_reasonable(data, odata, idx=idx)
        if not self.offline_data_is_already_normed:
            data = self.normlizer.do_pre_normlize(data)
        #assert len(data) == len(self.vnames) - len(self.constant_channel_pick)
        if isinstance(data, np.ndarray):
            data = torch.from_numpy(data)
        data = data.float()
        if reversed_part:  # the reversed_part should be
            data = data.clone() if isinstance(data, torch.Tensor) else data.copy()
            data[reversed_part] = -data[reversed_part]
        return data

    def get_item(self, idx, reversed_part=False):
        '''
        Notice for 3D case, we return (5,14,32,64) data tensor
        '''
        # if self.use_offline_data:
        #     data = self.dataset_tensor[idx]
        # else:
        data = self.generate_runtime_data(idx, reversed_part=reversed_part)

        output = {'field': data}

        if self.use_time_feature:
            output['timestamp'] = self.timestamp_date[idx]
        if self.add_LunaSolarDirectly:
            output['sun_mask'] = self.get_sum_mask(idx)
            output['luna_mask'] = self.get_moon_mask(idx)
        if self.constant_channel_pick and len(self.constant_channel_pick) > 0:
            output['constant'] = self.constants[self.constant_channel_pick]
        return output

    def get_sum_mask(self, idx):
        timenow = self.timestamp_date[idx]
        sun_lon, sun_lat = get_sub_sun_point(timenow.item())
        sun_vector = np.stack([np.cos(sun_lat/180*np.pi)*np.cos(sun_lon/180*np.pi),
                               np.cos(sun_lat/180*np.pi) *
                               np.sin(sun_lon/180*np.pi),
                               np.sin(sun_lat/180*np.pi)])

        sun_mask = (self.LaLotudeVector@sun_vector).reshape(1,
                                                            self.resolution_h, self.resolution_w)
        return sun_mask

    def get_moon_mask(self, idx):
        timenow = self.timestamp_date[idx]
        moon_lon, moon_lat = get_sub_luna_point(timenow.item())
        moon_vector = np.stack([np.cos(moon_lat/180*np.pi)*np.cos(moon_lon/180*np.pi),
                                np.cos(moon_lat/180*np.pi) *
                                np.sin(moon_lon/180*np.pi),
                                np.sin(moon_lat/180*np.pi)])
        moon_mask = (self.LaLotudeVector@moon_vector).reshape(1,
                                                              self.resolution_h, self.resolution_w)
        return moon_mask

    def make_data_physical_reasonable(self, data, odata, idx=None):
        '''
        make the data physical reasonable, for example, the total precipitation should be positive
        '''
        if self.make_data_physical_reasonable_mode is None:
            return data
        raise NotImplementedError

    @staticmethod
    def get_mesh_lon_lat(tH=32, tW=64):
        ### this is a general version
        resolution = tH
        assert tW == 2*tH
        theta_offset = (180/resolution/2)
        latitude = (np.linspace(0, 180, resolution+1) +
                    theta_offset)[:resolution]
        longitude = np.linspace(0, 360, 2*resolution+1)[:(2*resolution)]
        x, y = np.meshgrid(latitude, longitude)
        LaLotude = np.stack([y, x])/180*np.pi
        LaLotudeVector = np.stack([np.cos(LaLotude[1])*np.cos(LaLotude[0]), np.cos(
            LaLotude[1])*np.sin(LaLotude[0]), np.sin(LaLotude[1])], 2)
        return LaLotude, LaLotudeVector
        #### Below is 721x1440 origin data version
        # sH = 720
        # sW = 1440
        # steph = sH /float(tH)
        # stepw = sW /float(tW)
        # x     = np.arange(0, sW, stepw).astype('int')
        # y     = np.linspace(0, sH-1, tH, dtype='int')
        # x, y  = np.meshgrid(x, y)
        # latitude   = np.linspace(90,-90,721)
        # longitude  = np.linspace(0,360,1440)
        # LaLotude = np.stack([latitude[y],longitude[x]])/180*np.pi
        # LaLotudeVector = np.stack([np.cos(LaLotude[1])*np.cos(LaLotude[0]),np.cos(LaLotude[1])*np.sin(LaLotude[0]),np.sin(LaLotude[1])],2)
        # return LaLotude,LaLotudeVector

class WeatherBenchPhysical(WeatherBench):
    default_root='datasets/weatherbench'
    use_offline_data = False

    def make_data_physical_reasonable(self, data, odata, idx=None):
        '''
        make the data physical reasonable, for example, the total precipitation should be positive
        '''
        if self.make_data_physical_reasonable_mode is None:
            return data
        eg = torch if isinstance(data, torch.Tensor) else np
        if self.make_data_physical_reasonable_mode == 'use_constant_patch_lack_of_groud_geopotential':
            assert '2D70' in self.dataset_flag
            data = data.clone() # <-- avoid inplace modify
            data[14*4-1]    = eg.ones_like(data[14*4-1])*50 # modifiy the groud Geopotential, we use 5m height
            total_precipitaiton = odata[4]
            data[14*5-1]    = total_precipitaiton
            return data
        if self.make_data_physical_reasonable_mode == 'build_resonal_3D_tensor':
            assert '3D70' in self.dataset_flag
            raise NotImplementedError("TODO: notice the input tensor now is a 3D tensor, we need to modify the code to make it work")
            data[14*4-1] = eg.ones_like(data[14*4-1])*50 # modifiy the groud Geopotential, we use 5m height
            total_precipitaiton = odata[4]
            newdata = data[14*5-1].clone() if isinstance(data,torch.Tensor) else data[14*5-1].copy()
            newdata[total_precipitaiton>0] = 100
            newdata[data[14*5-1]>100]=data[14*5-1][data[14*5-1]>100]
            data[14*5-1] = newdata
            shape= data.shape
            data = data.reshape(5,14,*shape[-2:])
            return data

class WeatherBenchMultibranchRandom(WeatherBench):
    """
    This is a multibranch random fetcher, it will randomly select a time_intervel from multibranch_select.
    We will use outside random fetcher to call this module.
    For evaluation, we will use all time_intervel in multibranch_select via call set_time_intervel
    """
    def __init__(self, split="train", shared_dataset_tensor_buffer=(None,None),config = {}):
        super().__init__(split=split, shared_dataset_tensor_buffer=(None, None), config=config)
        use_offline_data = config.get('use_offline_data', 0)
        time_intervel    = config.get('time_intervel', 1)
        assert time_intervel == 1
        assert use_offline_data == 0
        self.multibranch_select= [int(t) for t in config['multibranch_select']]

    def set_time_intervel(self, time_intervel):
        self.time_intervel = time_intervel

    def __getitem__(self, idx):
        reversed_part = self.do_time_reverse(idx)
        time_intervel = self.time_intervel
        if self.split=='train':
            time_intervel = np.random.choice(self.multibranch_select)
            dprint(f"""
        ========> We use random time_intervel:{time_intervel}. However, this line should not appear here if your are in Train ,pde. You need call a random fetcher outside
                  """ )
        time_step_list = [idx+i*time_intervel for i in range(self.time_step)]
        if reversed_part:time_step_list = time_step_list[::-1]
        batch = [self.get_item(i, reversed_part) for i in time_step_list]
        return batch if not self.with_idx else (idx, batch)

class WeatherBench7066(WeatherBenchPhysical):
    """
    config = {
        'root':"/mnt/data/ai4earth/zhangtianning/datasets/WeatherBench/weatherbench_6hour/",
        'channel_name_list':"configs/datasets/WeatherBench/2D70.channel_list.json",
        'time_reverse_flag':'only_forward',
        'offline_data_is_already_normed':True,
        'time_unit':6
    }

    """

class WeatherBenchPatch(WeatherBench):
    """
    config = {
        'root':"/mnt/data/ai4earth/zhangtianning/datasets/WeatherBench/weatherbench_6hour/",
        'channel_name_list':"configs/datasets/WeatherBench/2D70.channel_list.json",
        'time_reverse_flag':'only_forward',
        'offline_data_is_already_normed':True,
        'time_unit':6
    }
    """
    def __init__(self, split="train", shared_dataset_tensor_buffer=(None,None),config = {}, debug=False):
        super().__init__(split=split, shared_dataset_tensor_buffer=(None, None), config=config)
        self.patch_range  = patch_range = config.data_patch_range
        self.cross_sample = ((self.split == 'train') or debug) and (self.patch_range is not None)
        self.patch_chunk  = config.patch_chunk

        if len(self.channel_choice.shape) == 2:
            patch_range = patch_range if isinstance(patch_range, (list, tuple)) else (patch_range, patch_range, patch_range)
            self.center_index, self.around_index = get_center_around_indexes_3D(patch_range, self.img_shape)
        else:
            patch_range = patch_range if isinstance(patch_range, (list, tuple)) else (patch_range, patch_range)
            self.center_index, self.around_index = get_center_around_indexes(patch_range, self.img_shape)
        self.patch_range = patch_range
        dprint(f"notice we will use around_index{self.around_index.shape} to patch data")
        #### due to the data efficiency, reshape the odata at beginning and do data preprocess
        assert self.dataset_tensor is not None
        assert (self.record_load_tensor == 1).all()
        self.dataset_tensor =  self.dataset_tensor[:,self.channel_choice] #(B, 70, 32, 64) or (B, 5, 14, 32, 64)
        self.dataset_tensor = torch.Tensor(self.dataset_tensor)
        if not self.offline_data_is_already_normed:
            self.dataset_tensor = self.normlizer.do_pre_normlize(self.dataset_tensor)
        self.dataset_tensor = self.dataset_tensor.float()
        self.data_processing_is_all_done = True
        self.img_shape  = self.dataset_tensor.shape[2:]
        self.length = len(self)

    def get_cross_sample_patch(self, idx):
        '''
        Notice for 3D case, we return (5,14,32,64) data tensor
        '''
        # if self.use_offline_data:
        #     data = self.dataset_tensor[idx]
        # else:
        if len(self.img_shape)==2:
            center_h = np.random.randint(self.img_shape[-2] - (self.patch_range[-2]//2)*2,size=(self.patch_chunk,))
            center_w = np.random.randint(self.img_shape[-1],size=(self.patch_chunk,))
            location = self.around_index[center_h, center_w] #(B,2,5,5)
            patch_idx_h = location[:,0]#(B,5,5)
            patch_idx_w = location[:,1]#(B,5,5)
            batch_idx = np.random.randint(self.length,size=(self.patch_chunk,)).reshape(self.patch_chunk,1,1) #(B,1,1)
            data = []
            for i in range(self.time_step):
                dt = i*self.time_intervel
                output = {'field':self.dataset_tensor[batch_idx+dt,:,patch_idx_h,patch_idx_w].permute(0,3,1,2)}
                if self.use_time_feature:
                    output['timestamp'] = [self.timestamp_date[idx+dt] for idx in batch_idx]
                if self.add_LunaSolarDirectly:
                    output['sun_mask']  = np.stack([self.get_sum_mask(idx)[0][a,b] for idx,a,b in zip(batch_idx,patch_idx_h,patch_idx_w)])
                    output['luna_mask'] = np.concatenate([self.get_moon_mask(idx)[0][a,b] for idx,a,b in zip(batch_idx,patch_idx_h,patch_idx_w)])
                assert not (self.constant_channel_pick and len(self.constant_channel_pick) > 0), 'do not use constant'
                output['location']  = location
                data.append(output)
            #data = [[self.data[batch_idx+i,:,patch_idx_h,patch_idx_w].permute(0,3,1,2)] for i in range(self.time_step)]
        elif len(self.img_shape)==3:
            center_z    = np.random.randint(self.img_shape[-3] - (self.patch_range[-3]//2)*2,size=(self.patch_chunk,))
            center_h    = np.random.randint(self.img_shape[-2] - (self.patch_range[-2]//2)*2,size=(self.patch_chunk,))
            center_w    = np.random.randint(self.img_shape[-1],size=(self.patch_chunk,))
            location    = self.around_index[center_z, center_h, center_w] #(B,2,5,5,5)
            patch_idx_z = location[:,0]#(B,5,5,5)
            patch_idx_h = location[:,1]#(B,5,5,5)
            patch_idx_w = location[:,2]#(B,5,5,5)
            #location = self.center_index[:,center_z, center_h, center_w].transpose(1,0)#(B,3)
            batch_idx = np.random.randint(self.length,size=(self.patch_chunk,)).reshape(self.patch_chunk,1,1,1) #(B,1,1,1)
            data = []
            for i in range(self.time_step):
                output = {'field': self.dataset_tensor[batch_idx+dt, :,patch_idx_z, patch_idx_h, patch_idx_w].permute(0, 4, 1, 2, 3)}
                if self.use_time_feature:
                    output['timestamp'] = [self.timestamp_date[idx+dt] for idx in batch_idx]
                if self.add_LunaSolarDirectly:
                    output['sun_mask']  = np.stack([self.get_sum_mask(idx)[0][a,b] for idx,a,b in zip(batch_idx[...,0],patch_idx_h,patch_idx_w)])
                    output['luna_mask'] = np.concatenate([self.get_moon_mask(idx)[0][a,b] for idx,a,b in zip(batch_idx[...,0],patch_idx_h,patch_idx_w)])
                assert not (self.constant_channel_pick and len(self.constant_channel_pick) > 0), 'do not use constant'
                output['location']  = location
                data.append(output)
            #data = [[self.data[batch_idx+i,:,patch_idx_z,patch_idx_h,patch_idx_w].permute(0,4,1,2,3)] for i in range(self.time_step)]
        else:
            raise NotImplementedError
        return data

    def get_normal_image(self, idx):
        time_step_list = [idx+i*self.time_intervel for i in range(self.time_step)]
        batch = [self.get_item(i, False) for i in time_step_list]
        dict_list =[]
        for i,data in enumerate(batch):
            _dict = {'field':data} if isinstance(data, (torch.Tensor, np.ndarray)) else data
            assert isinstance(_dict, dict)
            if self.with_idx:_dict['idx'] = idx
            dict_list.append(_dict)
        return dict_list

    def __getitem__(self, idx):
        if self.cross_sample:
            return self.get_cross_sample_patch(idx)
        else:
            return self.get_normal_image(idx)

    @staticmethod
    def collate_fn(batch):
        if isinstance(batch[0], dict):
            return {key: WeatherBenchPatch.collate_fn([d[key] for d in batch]) for key in batch[0]}
        elif isinstance(batch[0], list):
            return [WeatherBenchPatch.collate_fn(samples) for samples in zip(*batch)]
        elif isinstance(batch[0], torch.Tensor):
            return torch.cat(batch, dim=0)
        else:
            return batch

if __name__ == "__main__":
    pass
    # import sys
    # import time
    # from tqdm import tqdm
    # from multiprocessing import Pool
    # from petrel_client.client import Client
    # import petrel_client
    # dataset = WeatherBench64x128(split='test',root='weatherbench:s3://weatherbench/weatherbench64x128/npy',dataset_flag='2D68N')
    # for i in tqdm(range(0, 1000)):
    #     a = dataset[i][0]

    # def load_data_range(dataset, start, end):
    #     for i in tqdm(range(start, end)):
    #         _ = dataset[i]

    # def parallel_load_data(dataset):
    #     total_len = len(dataset)
    #     processor = 10
    #     range_list = np.linspace(0, total_len, processor+1)
    #     range_list = [int(a) for a in range_list]
    #     dprint(range_list)
    #     res = []
    #     p = Pool(processor)
    #     for i in range(processor):
    #         res.append(p.apply_async(load_data_range, args=(
    #             dataset, range_list[i], range_list[i+1])))
    #         dprint(str(i) + ' processor started !')
    #     p.close()
    #     p.join()

    # def check_range(dataset, start, end):
    #     fail_list = []
    #     for i in tqdm(range(start, end)):
    #         if i > len(dataset):
    #             continue
    #         now_time = time.time()
    #         try:
    #             _ = dataset[i]
    #         except:
    #             dprint(f"fail at {i}")
    #         fail_list.append(i)
    #         cost = time.time() - now_time
    #         #dprint(cost)
    #     return fail_list
    # dataset = ERA5CephDataset(split='train')
    # dprint(ERA5CephDataset.__name__)
    # dprint(dataset[0])

    # dprint(len(dataset[0]))
    # dprint(len(dataset[1]))

    raise
    dataset = ERA5CephDataset(split='valid')
    dprint(len(dataset))

    for idx in range(len(dataset)):
        dprint(dataset.get_url(idx))
    exit()
    processor = 10
    unit = len(dataset)//processor
    res = []
    p = Pool(processor)
    for i in range(processor):
        res.append(p.apply_async(
            check_range, args=(dataset, i*unit, (i+1)*unit)))
        #dprint(str(i) + ' processor started !')

    p.close()
    p.join()


# %% Specify which data-set should be analyzed
name_probes = 'Probes__cov-2'  # Genomic probes
#name_probes = 'Probes__cov-2--RevComp'  # Replication intermediate

# %% Imports
from pathlib import Path
import pandas as pd
from covfish import probe_designer
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
import importlib

# %% Folders for data and results
path_data = Path(Path.cwd() / '..' / '..' / 'data').resolve()
path_blast = path_data / 'probes' / f'{name_probes}' / 'blast'
path_probes = path_data / 'probes' / f'{name_probes}'
path_results = path_blast / 'results_details'

file_summary = path_probes / f'{name_probes}_ALL_summary.txt'

if not path_results.is_dir():
    path_results.mkdir()

# %%  Read blast results
importlib.reload(probe_designer)

# >>> Import web blast
blast_web_all = probe_designer.blast_summarize(path_blast=path_blast / 'web',
                                               file_identifier=path_data / 'blast_identifiers.json',
                                               sep=',',
                                               ext='csv',
                                               path_results=path_results,
                                               refseq_keep=['NM_', 'XM_', 'NR', 'XR_'])

# >>> Import lcoal blast against other viruses
importlib.reload(probe_designer)
blast_local_all = probe_designer.blast_summarize(path_blast=path_blast / 'local',
                                                 file_identifier=path_data / 'blast_identifiers.json',
                                                 sep='\t',
                                                 ext='txt',
                                                 path_results=path_results,
                                                 refseq_keep=None)

#  >>> import blast against cov-2 alignment

# Count how many genoems (identified by >)
file_cov = path_data / 'genomes' / 'cov2' / 'cov2_aligned.fasta'
genomes_cov2 = file_cov.read_text()
n_genomes = genomes_cov2.count('>')

# Open blast results
names_col = ['qseqid', 'sseqid', 'qlen', 'pident', 'length', 'mismatch', 'gapopen', 'qstart', 'qend', 'sstart', 'hsend', 'evalue', 'bitscore']
file_blast = path_blast / 'local_cov2' / 'cov2_aligned.txt'
blast_results = pd.read_csv(file_blast, sep='\t', header=None, names=names_col)
blast_results['align_diff'] = blast_results['qlen'] - blast_results['length'] + blast_results['mismatch']

# Simplify df
blast_results = blast_results.filter(items=['qseqid', 'sseqid', 'align_diff'])
# Remove potential duplicates and only heep the ones with highest alignment difference
blast_results.sort_values('align_diff', ascending=False).drop_duplicates(['qseqid','sseqid'], inplace=True)
# Keep only blast hits with 0 or 1 mismatch over the entire probe sequence
blast_covid_alignment = blast_results.query('align_diff <=1').groupby('qseqid').count()
blast_covid_alignment['align_diff'] = blast_covid_alignment['align_diff'] * 100.0 / n_genomes
blast_covid_alignment.rename(columns={'sseqid': f'cov2_align_N',
                                      'align_diff': f'cov2_align_perc'}
                             ,inplace=True)

# >>>>  Read probe summary and add other blast results

# >>>  DF with probe list
probes_summary = pd.read_csv(file_summary, sep='\t')
probes_summary['qseqid'] = probes_summary['ProbesNames'] + '--' + probes_summary['theStartPos'].astype('str') + '--' + probes_summary['theEndPos'].astype('str')

# >>> Loop over all blast results and join
probes_summary_joined = probes_summary
blast_all = [blast_covid_alignment] + blast_web_all + blast_local_all
for blast_add in blast_all:
    probes_summary_joined = pd.merge(left=probes_summary_joined, right=blast_add, how='left', left_on='qseqid', right_on='qseqid')

# >>> Save file to csv
file_save = path_probes / f'{name_probes}_ALL__with_blast_FLAPY.csv'
probes_summary_joined.to_csv(file_save, sep=',', index=False)
print(f'\n\nSUMMARY saved as {file_save}')

# %% Plot results of alignment against cov-2

# >>> Number of probe squences that match well all consensus sequences
plt.figure(figsize=(5, 4))
sns.distplot(blast_covid_alignment[f'cov2_align_perc'],
             bins = np.arange(90,100.05,0.25),
             kde=False, rug=False)
plt.title(f'Percentage of alignment\nwith {n_genomes} genomes for each oligo (n={blast_covid_alignment.shape[0]})')
plt.tight_layout()
file_save = path_results / 'cov2-alignment_hist.png'
plt.savefig(file_save, dpi=300)

# >> Cummulative histogram
plt.figure(figsize=(5, 4))
kwargs = {'cumulative': True}
sns.distplot(blast_covid_alignment[f'cov2_align_perc'], hist_kws=kwargs, kde_kws=kwargs)
plt.title(f'Percentage of alignment\nwith {n_genomes} genomes for each oligo (n={blast_covid_alignment.shape[0]})')
file_save = path_results / 'cov2-alignment_cdf.png'
plt.savefig(file_save, dpi=300)
# %%


# (c) 2017 Gregor Mitscha-Baude
"""stokesian dynamics simulation of two kinds of particles

framework: given list of particle centers xi and radii ri, for each timestep, calculate:
1.) external force on each particle (and torque?) F(xi, ri) or even F(x1,...,xn; r1,...,rn)
2.) brownian (random) force (and torque?) Frand_i
3.) velocities via ui = sum_j ( M_ij * F_j ) where M_ij(x_i, x_j) is the
    3x3 translational - translational mobility matrix.
4.) next position x_i = x_i + ui * delta t

rotational velocities can be ignored in this framework because they do not
change the particle position."""
import numpy as np
from nanopores import kT, eta, eperm, qq, rpermw, HOME
from matplotlib import pyplot as plt
#from matplotlib import patches
import matplotlib.animation as animation
from matplotlib import collections
from functools import partial
#import matplotlib
#matplotlib.use("Agg")

class Particle(object):

    def __init__(self, x, a, charge=0., color="blue"):
        self.x = np.reshape(np.array(x), (-1, 1))
        self.a = a
        self.circle = None
        self.charge = charge
        self.color = color
#
#    def add_patch(self, ax):
#        self.circle = patches.Circle(xy=self.x[::2], radius=self.a)
#        ax.add_patch(self.circle)
#
#    def patch(self):
#        self.circle = patches.Circle(xy=self.x[::2], radius=self.a)
#        return self.circle

    def move(self, dx):
        self.x = self.x + np.array(dx).reshape(-1, 1)
#        if self.circle is not None:
#            self.circle.center = self.x[::2]

class Plane(object):

    def __init__(self, p, n): # outer normal vextor and one point on plane
        self.n = np.array(n).reshape(1, 3)
        self.p = np.array(p).reshape(1, 3)

    def reflect(self, P):
        # reflect all points in P downwards that lie above plane
        # ("above" is direction of normal vector)
        x = np.array([p.x.flatten() for p in P])
        # n * (x - p) > 0
        excess = np.dot(x - self.p, self.n.T)
        above = (excess > 0).flatten()
        dx = np.zeros_like(x)
        dx[above, :] = (-2.*excess*self.n)[above, :]

        for i, p in enumerate(P):
            if above[i]:
                p.move(dx[i, :])

class Box(object):
    # reflect to be inside box
    def __init__(self, x0, Lx, Ly, Lz):
        ex = np.eye(1, 3, 0)
        ey = np.eye(1, 3, 1)
        ez = np.eye(1, 3, 2)
        x0 = np.array(x0).reshape(1, 3)
        self.planes = [Plane(x0, -ex), Plane(x0 + Lx*ex, ex),
                       Plane(x0, -ey), Plane(x0 + Ly*ey, ey),
                       Plane(x0, -ez), Plane(x0 + Lz*ez, ez)]

    def reflect(self, P):
        for plane in self.planes:
            plane.reflect(P)

def msqrt(M): # matrix square root
    #U, S, V = np.linalg.svd(M)
    #return np.dot(U, np.dot(np.diag(np.sqrt(S)), V))
    return np.linalg.cholesky(M)

# external force: constant electrical field
def f_extern(p):
    #return np.zeros_like(p.x)
    a = 0e-14
    #b = 1e-26
    f = np.zeros_like(p.x)
    f[2, :] = -a*p.a**3 #+ b*(p.x[2, :] - 1.)**(-7)
    return f

def f_brownian(p):
    rand = np.random.randn(*p.x.shape)
    rand[1] = 0.
    return rand

def RR(R):
    #R = np.reshape(R, (-1, 1))
    r = np.linalg.norm(R)
    return np.matrix(R * R.T / r**2)

def mobility_pp(p1, p2): # D/kT
    a1 = 1e-9*p1.a
    a2 = 1e-9*p2.a
    R = 1e-9*p1.x - 1e-9*p2.x
    r = np.linalg.norm(R)
    I = np.matrix(np.eye(3))

    if r <= abs(a1 - a2):
        return 1./(6.*np.pi*eta*max(a1, a2)) * I
    elif r <= a1 + a2:
        A = 0.5*(a1 + a2) - ((a1-a2)**2 + 3.*r**2)**2/(32.*r**3)
        B = 3.*((a1-a2)**2 - r**2)**2/(32.*r**3)
        return 1./(6.*np.pi*eta*a1*a2) * (A*I + B*RR(R))
    else:
        a = a1**2 + a2**2
        A = 1. + a/(3.*r**2)
        B = 1. - a/r**2
        return 1./(8.*np.pi*eta*r) * (A*I + B*RR(R))

def mobility_vectorized(P):
    # return matrix mobility(pi, pj)_kl where kl are the space coordinates
    # is, at the moment, list of particles
    # TODO: need pure numerical representation !!!!!!!!!!!!1111
    n = len(P)
    M = np.matrix(np.zeros((3*n, 3*n)))
    for i, p in enumerate(P):
        for j, q in enumerate(P):
            M[i::n, j::n] = mobility_pp(p, q)
    return M

def mobility(P):
    # nice: M is of the form Mijkl = Aij * Ikl + Bij * Rijkl
    # where Ikl, Rijkl are easily formed
    # obtain vector representations of positions, radii
    n = len(P)
    a = 1e-9*np.array([p.a for p in P])
    x = 1e-9*np.array([p.x.flatten() for p in P])
    rr = np.sum((x[:, None, :] - x[None, :, :])**2, 2) + 1e-100
    r = np.sqrt(rr)

    A = np.zeros_like(r)
    B = np.zeros_like(r)

    ama = a[:, None] - a[None, :]
    apa = a[:, None] + a[None, :]
    asq = a[:, None]**2 + a[None, :]**2

    case1 = r <= np.abs(ama)
    A[case1] = (1./(6.*np.pi*eta*np.maximum(a[:,None], a[None,:])))[case1]
    B[case1] = 0.

    case2 = ~case1 & (r <= apa)
    C = 1./(6.*np.pi*eta*a[:, None]*a[None, :])
    A[case2] = (C*(0.5*apa - (ama**2 + 3.*r**2)**2/(32.*r**3)))[case2]
    B[case2] = (C*(3.*(ama**2 - r**2)**2/(32.*r**3)))[case2]

    case3 = ~(case1 | case2) # else
    C = 1./(8.*np.pi*eta*r)
    A[case3] = (C*(1. + asq/(3.*r**2)))[case3]
    B[case3] = (C*(1. - asq/r**2))[case3]

    I = np.eye(3)
    R = (x[:, None, :] - x[None, :, :])
    RR = (R[:, :, :, None] * R[:, :, None, :]).transpose(2, 0, 3, 1)
    RR = RR / (rr[None, :, None, :])

    M = (A[None, :, None, :] * I[:, None, :, None]).reshape(3*n, 3*n) + \
        (B[None, :, None, :] * RR).reshape(3*n, 3*n)
    return np.matrix(M)

def f_vectorized(P, f):
    n = len(P)
    F = np.zeros((3*n, 1))
    for i, p in enumerate(P):
        F[i::n] = f(p)
    return F

def f_electric(P):
    n = len(P)
    a = 1e-9*np.array([p.a for p in P])
    apa = a[:, None] + a[None, :]

    x = 1e-9*np.array([p.x.flatten() for p in P])

    R = x[:, None, :] - x[None, :, :]
    r = np.sqrt(np.sum(R**2, 2) + 1e-100)
    R0 = R / (r**3)[:, :, None]

    q = np.array([float(p.charge) for p in P])
    const = qq**2 / (4.*np.pi*eperm*rpermw)

    QQ = q[:, None] * q[None, :]
    F = const * QQ[:, :, None] * R0
    #F[np.diag_indices_from(r)] = 0.
    tooclose = r <= apa
    R0i = R / (np.maximum(a[:, None], a[None, :])**3)[:, :, None]
    F[tooclose] = (const * QQ[:, :, None] * R0i)[tooclose]

    f = np.sum(F, 1).T.reshape(3*n, 1)

    return f

def f_shortrange(P):
    n = len(P)
    a = 1e-9*np.array([p.a for p in P])
    apa = a[:, None] + a[None, :]

    x = 1e-9*np.array([p.x.flatten() for p in P])
    R = x[:, None, :] - x[None, :, :]
    r = np.sqrt(np.sum(R**2, 2)) + 1e-100

    #E0 = apa*1e9*10.*kT # total energy required for r = apa*1.1 --> r = 0
    #E = E0*((r/apa/1.1 - 1.)**2)
    E0 = 1e9*1e1*kT
    cutoff = 1.05
    f = 2./cutoff*E0*np.maximum(1. - r/apa/cutoff, 0.)

    R0 = R / r[:, :, None]
    F = f[:, :, None] * R0
    ff = np.sum(F, 1).T.reshape(3*n, 1)
    return ff

def simulation(P, T=100., dt=1.):
    t = 0.
    while t < T:
        # calculate forces
        force = f_electric(P) + f_shortrange(P)
        brownian = f_vectorized(P, f_brownian)
        M = mobility(P)
        sqM = np.matrix(msqrt(M))

        # determine resulting velocities
        U = M*force + np.sqrt(2.*kT/dt*1e9)*sqM*brownian

        n = len(P)
        # move particles
        for i, p in enumerate(P):
            u = U[i::n]
            p.move(dt*u)

        yield t
        t += dt

# static particles: no forces act on them, are not affected by
#from time import time

def move(P, dt=1., boundaries=()):
    # calculate forces
    force = f_electric(P) + f_shortrange(P)
    brownian = f_vectorized(P, f_brownian)
    #t = time()
    M = mobility(P)
    #print "forming mobility", time() - t
    #t = time()
    sqM = np.matrix(msqrt(M))
    #print "matrix square root", time() - t

    # determine resulting velocities
    Udet = M*force
    Ubro = np.sqrt(2.*kT/dt*1e9)*sqM*brownian
    #U = M*force + np.sqrt(2.*kT/dt*1e9)*sqM*brownian
    U = Udet + Ubro
    #print "P0:", Udet[0], Ubro[0]
    #print "P1:", Udet[1], Ubro[1]
    #print

    n = len(P)
    # move particles
    for i, p in enumerate(P):
        u = U[i::n]
        p.move(dt*u)

    for b in boundaries:
        b.reflect(P)

class ParticleCollection(object):
    def __init__(self, particles=None):
        if particles is None:
            particles = []
        self.P = particles

    def add(self, generate, N):
        i = 0
        while i < N:
            p0 = generate()
            if all(np.linalg.norm(q.x - p0.x) > p0.a + q.a for q in self.P):
                self.P.append(p0)
                i += 1

def ellipse_collection(ax, P):
    xy = np.array([p.x[::2].flatten() for p in P])
    sizes = np.array([p.a for p in P])
    coll = collections.EllipseCollection(sizes, sizes, np.zeros_like(sizes),
               offsets=xy, units='x', facecolors=[p.color for p in P],
               transOffset=ax.transData, alpha=0.7)
    return coll

def panimate(ax, dt, pcoll, patches, boundaries, **kwargs):
    particles = pcoll.P
    coll = ellipse_collection(ax, particles)
    def init():
        return ()
        #ax.clear()
        #ax.add_patch(rect)
        #return (rect, )

    def animate(i):
        if i == 0:
            for patch in patches:
                ax.add_patch(patch)
            ax.add_collection(coll)
        else:
            move(particles, dt, boundaries)
            xy = np.array([p.x[::2].flatten() for p in particles])
            coll.set_offsets(xy)
        return tuple(patches + [coll])

    kwargs = dict(dict(frames=1800, interval=10, blit=True), **kwargs)
    ani = animation.FuncAnimation(ax.figure, animate, init_func=init, **kwargs)
    return ani

def maybe_save(do_save, ani, name):
    if do_save:
        ani.save(HOME + "/presentations/anaday17/" + name, fps=30, dpi=200,
                 writer="ffmpeg_file",
                 savefig_kwargs={"bbox_inches":0, "pad_inches":0},
                 extra_args=['-vcodec', 'libx264'])
    else:
        plt.show()

def random_particle(L, *args, **kwargs):
    x = [(2*np.random.rand() - 1.)*L, 0., (2*np.random.rand() - 1.)*L]
    return Particle(x, *args, **kwargs)

def setup_box(L):
    # ax, patches, boundaries = setup_box(L)
    fig = plt.figure()
    fig.set_size_inches(6, 6)

    ax = plt.axes([0,0,1,1], autoscale_on=False,
                     xlim=(-L, L), ylim=(-L, L))
    ax.set_axis_off()
    rect = plt.Rectangle([-L, -L], 2*L, 2*L, ec='k', lw=3, fc='none')
    box = Box([-L, -L, -L], 2*L, 2*L, 2*L)
    return ax, [rect], [box]


# -*- coding:utf-8 -*-
# Last Change : Fri 02 Nov 2018 16:40:31.
import socket
import json
import threading
import matplotlib.pyplot as plt

import pickle

# {"range":float,"angle":float,"x":float,"y":float,"radian":float}
class SLAM(object):
    '''recv map data and draw map'''
    def __init__(self, origin=[0, 0]):
        '''constractor'''
        self.origin       = origin
        self.idx          = 0
        self.data_len     = 0
        self.current_data = []
        self.max_data     = []

        self.recv_map_data_th = threading.Thread(target=self.recv_map_data)
        # self.draw_map_th      = threading.Thread(target=self.draw_map)

        # self.draw_map_th.setDaemon(True)

    def run(self):
        '''start task'''
        self.recv_map_data_th.start()
        self.recv_map_data_th.join(0.1)
        print(self.max_data)
        # with open("./map_data.json", "w") as f:
        #     json_dump_data = {"origin":self.origin, "data":self.max_data}
        #     json.dump(json_dump_data, f, indent=4)

    def recv_map_data(self):
        '''create UDP socket and recv map datas from server(Robot)'''
        with socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) as sock:
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 0)
            sock.bind(("0.0.0.0", 8888))

            # self.draw_map_th.start()

            def get_decoded_data():
                '''get data'''
                json_str, addr = sock.recvfrom(2048)
                json_str = json_str.decode("utf-8")

                if json_str == "END":
                    return json_str

                return json.loads(json_str)

            while True:
                # try:
                res = get_decoded_data()
                if res == "END":
                    break
                print("self.idx={}, self.current_data size={}, self.data_len={}".format(self.idx, len(self.current_data), self.data_len))
                if res["idx"] < self.idx:  # 1周計測し終わった時
                    self.current_data = self.current_data[:self.idx]
                    self.max_data     = self.max_data[:self.idx]
                    self.data_len     = self.idx

                # idxの更新
                self.idx = res["idx"]

                # 初回のデータを受け取っている時
                if self.data_len == 0:
                    self.current_data.append(res.copy())
                    self.max_data.append(res.copy())
                else:
                    if self.idx >= self.data_len:
                        self.current_data.append(res.copy())
                        self.max_data.append(res.copy())
                    else:
                        self.current_data[self.idx].update(res)
                        if res["range"] > self.max_data[self.idx]["range"]:
                            self.max_data[self.idx].update(res)

                # except KeyboardInterrupt:
                #     break

            with open("./map_data.json", "w") as f:
                json_dump_data = {"origin":self.origin, "data":self.max_data}
                json.dump(json_dump_data, f, indent=4)

    def draw_map(self):
        '''draw surrounding area'''
        plt.close("all")
        fig = plt.switch_backend('qt5agg')  # バックエンドをPyQt5に変更
        fig = plt.figure()
        line, = plt.plot(0, marker=".", markersize=2, linestyle="None", color="#7777FF")
        # line, = plt.plot(0, marker=".", linestyle="None", color="#7777FF")
        plt.grid(color='#CCCCCC')
        plt.xlim([-5000,5000])
        plt.ylim([-5000,5000])
        plt.show(block=False)

        while True:
            try:
                line.set_xdata(self.x)
                line.set_ydata(self.y)

                fig.canvas.draw()
                fig.canvas.flush_events()
            except KeyboardInterrupt:
                break
        plt.close(fig)

if __name__ == "__main__":
    slam = SLAM()
    slam.run()


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import tensorflow as tf
from tensorflow.keras.layers import AveragePooling2D
from tensorflow.keras.layers import Dropout
from tensorflow.keras.layers import Flatten
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Input
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam

from tensorflow.keras.utils import to_categorical

from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.preprocessing.image import img_to_array
from tensorflow.keras.preprocessing.image import load_img

from tensorflow.keras.applications import NASNetMobile
from tensorflow.keras.applications.nasnet import preprocess_input
from tensorflow.keras.applications import MobileNetV2
from tensorflow.keras.applications.mobilenet_v2 import preprocess_input
from tensorflow.keras.applications import InceptionV3
from tensorflow.keras.applications.inception_v3 import preprocess_input
from tensorflow.keras.applications import VGG16
from tensorflow.keras.applications.vgg16 import VGG16

from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import KFold

from imutils import paths
import os

LR = 1e-5
BS = 32
EP = 50
num_folds = 5

labels = []
data = []

DIR = r"/content/drive/MyDrive/dataset/"
CAT = ['without_mask', 'with_mask']

#For inceptionV3
print("Loading Images")
for cat in CAT:
    path = os.path.join(DIR,cat)
    for img in os.listdir(path):
        img_path = os.path.join(path, img)
        image = load_img(img_path, target_size=(224,224))

        image = img_to_array(image)
        image = preprocess_input(image)

        data.append(image)
        labels.append(cat)

print("Images loaded")

#One hot encoding labels
lb = LabelBinarizer()
labels = lb.fit_transform(labels)
labels = to_categorical(labels)

data = np.array(data, dtype="float32")
labels = np.array(labels)

(trainX, testX, trainY, testY) = train_test_split(data,
                                                  labels,
                                                  test_size=0.20,
                                                  stratify=labels,
                                                  random_state=42)
aug = ImageDataGenerator(
    rotation_range=20,
    zoom_range=0.15,
    width_shift_range=0.2,
    height_shift_range=0.2,
    shear_range=0.15,
    horizontal_flip=True,
    fill_mode="nearest")

import tensorflow as tf
from tensorflow import keras
callback = tf.keras.callbacks.EarlyStopping(
    monitor='val_loss', min_delta=0, patience=0, verbose=1, mode='auto',
    baseline=None, restore_best_weights=False
)

kfold = KFold(n_splits=num_folds, shuffle=True)

acc_per_fold = []
loss_per_fold = []
fold_no = 1

for train, test in kfold.split(data, labels):
  baseModel = VGG16(weights="imagenet",
                          include_top=False,
                          input_tensor=Input(shape=(224,224,3)))

  myModel = baseModel.output
  myModel = AveragePooling2D(pool_size=(7, 7))(myModel)
  myModel = Flatten(name="flatten")(myModel)
  myModel = Dense(128, activation="relu")(myModel)
  myModel = Dense(128, activation="relu")(myModel)
  myModel = Dropout(0.5)(myModel)
  myModel = Dense(2, activation="softmax")(myModel)
  model = Model(inputs=baseModel.input, outputs=myModel)

  for layer in baseModel.layers:
      layer.trainable = False

  opt = Adam(lr=LR, decay=LR/EP)
  model.compile(loss="binary_crossentropy",
                optimizer=opt,
                metrics=["accuracy"])

  print('------------------------------------------------------------------------')
  print(f'Training for fold {fold_no} ...')

  hist = model.fit(
      aug.flow(trainX, trainY, batch_size=BS),
      steps_per_epoch=len(trainX) // BS,
      validation_data=(testX, testY),
      validation_steps=len(testX) // BS,
      epochs=EP,)

  scores = model.evaluate(data[test], labels[test], verbose=0)
  print(f'Score for fold {fold_no}: {model.metrics_names[0]} of {scores[0]}; {model.metrics_names[1]} of {scores[1]*100}%')
  acc_per_fold.append(scores[1] * 100)
  loss_per_fold.append(scores[0])

  # Increase fold number
  fold_no = fold_no + 1

print('------------------------------------------------------------------------')
print('Score per fold')
for i in range(0, len(acc_per_fold)):
  print('------------------------------------------------------------------------')
  print(f'> Fold {i+1} - Loss: {loss_per_fold[i]} - Accuracy: {acc_per_fold[i]}%')
print('------------------------------------------------------------------------')
print('Average scores for all folds:')
print(f'> Accuracy: {np.mean(acc_per_fold)} (+- {np.std(acc_per_fold)})')
print(f'> Loss: {np.mean(loss_per_fold)}')
print('------------------------------------------------------------------------')

#Eval
predIdxs = model.predict(testX, batch_size=BS)

# for each image in the testing set we need to find the index of the
# label with corresponding largest predicted probability
predIdxs = np.argmax(predIdxs, axis=1)

# show a nicely formatted classification report
print(classification_report(testY.argmax(axis=1), predIdxs,target_names=lb.classes_))

# serialize the model to disk
# model.save(DIR+"mask_detector_mbv2test0ii.model", save_format="h5")

# plot the training loss and accuracy
N = EP
plt.style.use("ggplot")
plt.figure()
plt.plot(np.arange(0, N), hist.history["loss"], label="train_loss")
plt.plot(np.arange(0, N), hist.history["val_loss"], label="val_loss")
plt.plot(np.arange(0, N), hist.history["accuracy"], label="train_acc")
plt.plot(np.arange(0, N), hist.history["val_accuracy"], label="val_acc")

tn, fp, fn, tp = confusion_matrix(testY.argmax(axis=1), predIdxs).ravel()
print("True Negatives: ",tn)
print("False Positives: ",fp)
print("False Negatives: ",fn)
print("True Positives: ",tp)

plt.title("Training Loss and Accuracy")
plt.xlabel("Epoch #")
plt.ylabel("Loss/Accuracy")
plt.legend(loc="lower left")
# plt.savefig("vgg16e100-plot.png")


import csv
from datetime import datetime

# Year,  Avg Price, Min Price, Max Price, Avg Volume, Min Volume, Max Volume

def get_year(date: str) -> int:
    date = datetime.strptime(date, "%d-%m-%Y").date()
    year = date.year
    return year

def create_new_csv_file(file_name: str, fieldnames: list[str]) -> None:
    with open(file_name, "w") as new_file:
        writer = csv.DictWriter(new_file, fieldnames=fieldnames, delimiter=",", quoting=csv.QUOTE_NONE)
        writer.writeheader()

def write_to_file(file_name: str, fieldnames: list[str], row_dict: dict[str, int | float]) -> None:
    with open(file_name, "a") as new_file:
        writer = csv.DictWriter(new_file, fieldnames=fieldnames, delimiter=",", quoting=csv.QUOTE_NONE)
        writer.writerow(row_dict)

def create_row_dict(year: int, avg_price: float, min_price: float, max_price: float, avg_volume: float,
                    min_volume: float, max_volume: float) -> dict[str, int | float]:
    return {
        "Year": year,
        "Avg Price": avg_price,
        "Min Price": min_price,
        "Max Price": max_price,
        "Avg Volume": avg_volume,
        "Min Volume": min_volume,
        "Max Volume": max_volume,
    }

def update_file(file_name: str, from_file: str, fieldnames: list[str]) -> None:
    avg_price, min_price, max_price = 0.0, float('inf'), -float('inf')
    avg_volume, min_volume, max_volume = 0.0, float('inf'), -float('inf')
    total_volume: float = 0
    total_price: float = 0
    prev_year: int = 0
    row_cntr: int = 0

    with open(from_file, 'r') as csv_file:
        reader = csv.DictReader(csv_file, delimiter=",", quoting=csv.QUOTE_NONE)

        for row in reader:
            date: str = row["Date"]
            curr_year: int = get_year(date)

            if 0 != prev_year != curr_year:
                row_dict = create_row_dict(prev_year,
                                           total_price / row_cntr,
                                           min_price, max_price,
                                           total_volume / row_cntr,
                                           min_volume,
                                           max_volume)

                avg_price, min_price, max_price = 0.0, float('inf'), -float('inf')
                avg_volume, min_volume, max_volume = 0.0, float('inf'), -float('inf')
                row_cntr = 0
                total_price = 0
                total_volume = 0

                write_to_file(file_name, fieldnames, row_dict)

            # calculate values
            min_price = min(min_price, float(row["Low"]))
            max_price = max(max_price, float(row["High"]))
            min_volume = min(min_volume, float(row["Volume"]))
            max_volume = max(max_volume, float(row["Volume"]))
            total_volume += float(row["Volume"])
            total_price += ((float(row["Low"]) + float(row["High"])) / 2)
            row_cntr += 1

            prev_year = curr_year

    # to get the missing final year
    row_dict = create_row_dict(prev_year,
                               total_price / row_cntr,
                               min_price,
                               max_price,
                               total_volume / row_cntr,
                               min_volume,
                               max_volume)

    write_to_file(file_name, fieldnames, row_dict)

def apple_stock_csv(fpath: str, new_file_name: str):
    fieldnames = ['Year', 'Avg Price', 'Min Price', "Max Price", 'Avg Volume', "Min Volume", 'Max Volume']
    create_new_csv_file(new_file_name, fieldnames)
    update_file(new_file_name, fpath, fieldnames)

if __name__ == "__main__":
    # create a new csv of AAPL stock avgs
    apple_stock_csv("/Users/danielraz/EduLabs-FullStack-Course/Lessons/Lesson_9/src/Lesson_Code/files/data/AAPL.csv",
                    "avg_aapl_prices.csv")

    # test out the file we created
    with open("avg_aapl_prices.csv", 'r') as f:
        csvreader = csv.reader(f, delimiter=",", quoting=csv.QUOTE_NONE)
        for _row in csvreader:
            print(_row)


import torch
from torch import nn
from torch.nn import init
from torchvision.datasets import FashionMNIST
import numpy as np
import sys
from torch.utils.data import DataLoader
from torchvision import transforms

'''
1、数据加载
2、权重初始化、数据预处理 分割 裁剪
3、loss函数定义
4、准确度估计

torchvision.datasets: 一些加载数据的函数及常用的数据集接口；

torchvision.models: 包含常用的模型结构(含预训练模型)，例如AlexNet、VGG、ResNet等；

torchvision.transforms: 常用的图片变换，例如裁剪、旋转等；

torchvision.utils: 其他的一些有用的方法

解释方法含义：

1、dataset：提前定义的dataset的实例；
2、batch_size：传入数据的batch大小，常常是32、64、128、256’
3、shuffle：bool类型，表示是否在每次获取数据的时候提前打乱数据；
4、num_workers：加载数据的线程数。
5、drop_last：bool类型，为真，表示最后的数据不足一个batch，就删掉

'''

root_train =r'D:\4_data\1_DataSet\Minist\FashionMNIST\FashionMNIST\raw\train-images-idx3-ubyte.gz'
root_test =r'D:\4_data\1_DataSet\Minist\FashionMNIST\FashionMNIST\raw\t10k-images-idx3-ubyte.gz'

batch_size =256
mnist_train =FashionMNIST(root=r'D:\4_data\1_DataSet\Minist\FashionMNIST',train=True, download=False, transform=transforms.ToTensor())
mnist_test = FashionMNIST(root=r'D:\4_data\1_DataSet\Minist\FashionMNIST',train=False, download=False, transform=transforms.ToTensor())
train_iter =DataLoader(mnist_train,batch_size=256,shuffle=True)
test_iter =DataLoader(mnist_test,batch_size=256,shuffle=False)
# print(train_iter)
# print(test_iter)

#3.6.2 初始化模型参数
num_inputs = 784
num_outputs = 10#10个类

W = torch.tensor(np.random.normal(0, 0.01, (num_inputs, num_outputs)), dtype=torch.float)
b = torch.zeros(num_outputs, dtype=torch.float)#相当于没有偏置

#同之前一样，我们需要模型参数梯度。
W.requires_grad_(requires_grad=True)
b.requires_grad_(requires_grad=True)

X =torch.tensor([[1,2,3],[4,5,6]])
print(X.sum(dim =0,keepdim=True))#dim=0是列  dim=1是行
print(X.sum(dim =1,keepdim=True))

#开始定义softmax运算
'''
我们先描述一下如何对多维的tensor按维度操作
给定一个tensor矩阵x。

我们可以只对其中同一列（dim=0）或者同一行（dim=1）的元素求和，

并在结果中保留行和列这两个维度（keepdim=True）
'''
def softmax(m):
    '''
    输入一个tensor 对每一行的数据做softmax归一化处理
    :param m:
    :return:
    '''
    X_exp =m.exp()# 这是一个矩阵     输入的每一个值都做指数处理
    partition =X_exp.sum(dim=1,keepdim =True)#对行做softmax
    return X_exp/partition#对每一个值做了归一化处理

# X = torch.rand((2, 5))
# X_prob = softmax(X)
# print(X_prob, X_prob.sum(dim=1))

#定义模型（调用softmax算损失）
def net(X):
    return softmax(torch.mm(X.view(-1,num_inputs),W)+b)

#定义损失函数
'''为了得到标签的预测概率 我们使用gather函数

y_hat是在2个样本 在三个类别的预测概率
y是两个样本的实际标签

在代码中，标签类别的离散值是从0开始逐一递增的。
'''
y_hat = torch.tensor([[0.1, 0.3, 0.6], [0.3, 0.2, 0.5]])
y = torch.LongTensor([0, 2])#向量 或者列表
print('\n查看一下功能')
print(y)
y_hat.gather(1, y.view(-1, 1))

#计算分类准确度

'''
给定一个类别的-----预测概率分布y-hat ------我们把预测概率-----------最大的类别-----------作为输出类别
如果预测的值和真实值一致 则说明预测正确 否则预测错误

----
其中y_hat.argmax(dim=1)返回矩阵y_hat每行中最大元素的索引，且返回结果与变量y形状相同。argmax返回最大值index
----
相等条件判断式(y_hat.argmax(dim=1) == y)是一个类型为ByteTensor的Tensor，我们用float()将其转换为值为0（相等为假）或1（相等为真）的浮点型Tensor
'''
def accuracy(y_hat, y):
    return (y_hat.argmax(dim=1) == y).float().mean().item()
print(accuracy(y_hat, y))

# 本函数已保存在d2lzh_pytorch包中方便以后使用。该函数将被逐步改进：它的完整实现将在“图像增广”一节中描述
def evaluate_accuracy(data_iter, net):
    acc_sum, n = 0.0, 0
    for X, y in data_iter:
        acc_sum += (net(X).argmax(dim=1) == y).float().sum().item()
        n += y.shape[0]
    return acc_sum / n

print(evaluate_accuracy(test_iter, net))

# num_epochs, lr = 5, 0.1
#
# # 本函数已保存在d2lzh包中方便以后使用
# def train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size,
#               params=None, lr=None, optimizer=None):
#     for epoch in range(num_epochs):
#         train_l_sum, train_acc_sum, n = 0.0, 0.0, 0
#         for X, y in train_iter:
#             y_hat = net(X)
#             l = loss(y_hat, y).sum()
#
#             # 梯度清零
#             if optimizer is not None:
#                 optimizer.zero_grad()
#             elif params is not None and params[0].grad is not None:
#                 for param in params:
#                     param.grad.data.zero_()
#
#             l.backward()
#             if optimizer is None:
#                 d2l.sgd(params, lr, batch_size)
#             else:
#                 optimizer.step()  # “softmax回归的简洁实现”一节将用到
#
#
#             train_l_sum += l.item()
#             train_acc_sum += (y_hat.argmax(dim=1) == y).sum().item()
#             n += y.shape[0]
#         test_acc = evaluate_accuracy(test_iter, net)
#         print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f'
#               % (epoch + 1, train_l_sum / n, train_acc_sum / n, test_acc))
#
# train_ch3(net, train_iter, test_iter, cross_entropy, num_epochs, batch_size, [W, b], lr)


# @author: Gautam Patel
# Problem Description URL: https://www.hackerrank.com/challenges/two-characters/problem
#!/bin/python3

import math
import os
import random
import re
import sys
from itertools import groupby
from collections import Counter

# Complete the alternate function below.
def isStringAlternate(s):
    if len(s) < 2: return False
    e_ch = s[0]
    o_ch = s[1]
    if e_ch == o_ch:
        return False
    for i in range(2,len(s)):
        if i%2 == 0 and e_ch != s[i]:
            return False
        if i%2 == 1 and o_ch != s[i]:
            return False
    return True

def alternate(s):
    matcher = re.compile(r'(.)\1+')
    for m in matcher.finditer(s):
        ch = m.group()[0]
        s = s.replace(ch, '')
    if len(s) == 0 or isStringAlternate(s):
        return len(s)
    c = Counter(s)
    cnt_max = 0
    for k in c.items():
        cnt = alternate(s.replace(k[0], ''))
        if cnt > cnt_max:
            cnt_max = cnt
    return cnt_max

if __name__ == '__main__':
    fptr = open(os.environ['OUTPUT_PATH'], 'w')

    l = int(input().strip())

    s = input()

    result = alternate(s)

    fptr.write(str(result) + '\n')

    fptr.close()


#
# @lc app=leetcode id=162 lang=python3
#
# [162] Find Peak Element
#

# @lc code=start
from typing import List

class Solution:
    def findPeakElement(self, nums: List[int]) -> int:
        length = len(nums)
        left = 0
        right = length - 1

        while left < right:
            mid = (left + right) // 2
            if mid == 0 or mid == length - 1:
                break

            if nums[mid] > nums[mid-1]:
                if nums[mid] > nums[mid+1]:
                    return mid
                left = mid + 1

            elif nums[mid] > nums[mid+1]:
                if nums[mid] > nums[mid-1]:
                    return mid
                right = mid - 1

            else:
                if nums[mid+1] > nums[mid-1]:
                    left = mid + 1
                else:
                    right = mid - 1

        if nums[left] < nums[right]:
            return right
        return left

if __name__ == "__main__":
    s = Solution()
    print(s.findPeakElement([3, 1, 2]))
# @lc code=end


import serial
from serial.tools.list_ports import comports
import time
from statistics import mean

class Connect:
    connection_status = False

    def __init__(self):
        self.xData = []
        self.yData = []
        self.ocpData = []
        # Get a list of all connected devices through serial port
        com_devices = comports()

        # Connect to the AWT kit. Index might need to change.
        ser = serial.Serial(com_devices[1][0],timeout=10)
        ## Test connection
        print(ser.name)
        ser.write('*'.encode('utf-8'))
        time.sleep(.1)
        device_response = ser.read(size=1).decode('utf-8')

        # should return a '&'
        if(device_response == '&'):
            print("Connection established.")
            # connection_status = True
            self.ser = ser
        else:
            print("The device says: " + device_response)

    # setup_connection()

    def importParam(self, param_list):
        '''
        mode     = 0      vOffset      = 5
        vInit    = 1      initialDelay = 6
        vFinal   = 2      nRuns        = 7
        scanRate = 3      logIvl       = 8
        cGain    = 4      del2         = 9
        '''
        for p in param_list:
            self.ser.write(p.encode('utf-8'))
            self.ser.write(','.encode('utf-8'))
            time.sleep(.1)

    def readData(self):
        # read headers
        header_line = self.ser.read_until().decode('utf-8')
        print(header_line)

        while(header_line != '$\r\n'):
            header_line = self.ser.read_until().decode('utf-8')
            print(header_line)
        print('-------------------------')

        ## record actual data
        data_list = [""]
        while True:
            data_line = self.ser.read_until().decode('utf-8')
            data_list = data_line.split(",")
            ## check if the run ends
            if(data_list[0] == '99999'):
                break

            ## check if the first output is empty
            if(len(data_list) != 1):
                xRead = float(data_list[0])
                yRead1 = float(data_list[1])/1000
                yRead2 = float(data_list[2])/1000
                yRead = yRead1 - yRead2

                mVmin = float(data_list[3])
                mVmax = float(data_list[4])

                # record data
                self.xData.append(xRead)
                self.yData.append(yRead)

    def readOCP(self):
        # read headers
        header_line = self.ser.read_until().decode('utf-8')
        print(header_line)

        while(header_line != '$\r\n'):
            header_line = self.ser.read_until().decode('utf-8')
            print(header_line)
        print('-------------------------')

        ## record actual data
        data_list = [""]
        while True:
            data_line = self.ser.read_until().decode('utf-8')
            data_list = data_line.split(",")
            ## check if the run ends
            if(data_list[0] == '99999'):
                break

            ocp_read = float(data_list[0])
            self.ocpData.append(ocp_read)
            self.ocp_average = mean(self.ocpData)


from sklearn import metrics

def CD_map_to_int(x):
    # Returns D:0 C:1
    return 68 - ord(x)

# Itertuples is a list of (index,col1,col2,..) for each row. We sort them by score first
df_vectors = pd.DataFrame()
df_vectors_strings = pd.DataFrame()
df_generation_max = df_generation_max.sort_values('opponent_name')
df_generation_max = df_generation_max.sort_values('best_score')
for tup in df_generation_max.itertuples():
    # index vals mean the indexing starts at 1 so +1 to these:
    #(0, 'generation'), (1, 'score_mean'), (2, 'score_median'), (3, 'score_pop_var'), (4, 'score_range'), (5, 'best_score'), (6, 'best_sequence'), (7, 'opponent_name'), (8, 'seed'), (9, 'analysis_name'), (10, 'base_player'), (11, 'stochastic'), (12, 'memory_depth'), (13, 'makes_use_of'), (14, 'score_bin'), (15, 'start_move'), (16, 'num_blocks'), (17, 'mean_block_len'), (18, 'median_block_len')
    seq_str = tup[7]
    opponent_name = str(tup[8])
    best_score = tup[6]
    bins = tup[15]
    # Mapping to integers 0 &1
    df_vectors[opponent_name] = list(
        map(CD_map_to_int, seq_str)) + [best_score] + [seq_str] + [bins]

    # No Mapping Cs & Ds
    df_vectors_strings[opponent_name] = list(
        seq_str) + [best_score] + [seq_str] + [bins]

move_cols = ['move ' + str(x+1) for x in range(200)]
# Transposing and labeling moves as were forming it in a rotated way.
# (Its easier to just make 2 dfs than map one to the other)
df_vectors = df_vectors.transpose()
df_vectors.columns = move_cols + ["best_score", "best_sequence", "score_bins"]

df_vectors_strings = df_vectors_strings.transpose()
df_vectors_strings.columns = move_cols + \
    ["best_score", "best_sequence", "score_bins"]

# Manhatten Distance == Hamming Distance in this example
# Intresting examples: cosine(??), hamming, jaccard sim
dist_array_ham = metrics.pairwise.pairwise_distances(
    df_vectors[move_cols], metric='hamming')
# add labels
dist_array_ham = pd.DataFrame(
    dist_array_ham, index=df_vectors.index, columns=df_vectors.index)

dist_array_cos = metrics.pairwise.pairwise_distances(
    df_vectors[move_cols], metric='cosine')
dist_array_cos = pd.DataFrame(
    dist_array_cos, index=df_vectors.index, columns=df_vectors.index)

dist_array_jac = metrics.pairwise.pairwise_distances(
    df_vectors[move_cols], metric='jaccard')
dist_array_cos = pd.DataFrame(
    dist_array_cos, index=df_vectors.index, columns=df_vectors.index)

dist_array_other = metrics.pairwise.pairwise_distances(
    df_vectors[move_cols], metric='correlation')
dist_array_cos = pd.DataFrame(
    dist_array_cos, index=df_vectors.index, columns=df_vectors.index)


#!/usr/bin/python3

from typing import List, Tuple, Dict, Set, Union
from re import split
from operator import mul, add

def compute_equal_precedence(enumerator: enumerate) -> int:
    op = lambda a, b: b
    result = 0
    for i, t in enumerator:
        if t == "*":
            op = mul
        elif t == "+":
            op = add
        elif t == "(":
            result = op(result, compute_equal_precedence(enumerator))
        elif t == ")":
            return result
        else:
            result = op(result, int(t))
    return result

def find_matching_brace(tokens: List[Union[str,int]], idx: int) -> int:
    depth = 0
    for i, t in enumerate(tokens[idx:]):
        if t == '(':
            depth += 1
        elif t == ')':
            depth -= 1
            if depth == 0:
                return i
    return idx

def compute(tokens: List[Union[str,int]]) -> int:
    result = 0
    # print(f'Starting from {tokens}')
    while '(' in tokens:
        a = tokens.index('(')
        b = find_matching_brace(tokens, a) + a
        val = compute(tokens[a+1:b])
        tokens = tokens[:a] + [val] + tokens[b+1:]
    while '+' in tokens:
        m = tokens.index('+')
        v: int = int(tokens[m-1]) + int(tokens[m+1])
        tokens = tokens[:m-1] + [int(v)] + tokens[m+2:]
    while '*' in tokens:
        m = tokens.index('*')
        v = int(tokens[m-1]) * int(tokens[m+1])
        tokens = tokens[:m-1] + [int(v)] + tokens[m+2:]
    return int(tokens[0])

def read_file(day: str) -> List[str]:
    with open(f"input/day_{day}.txt") as file:
        lines = [line.strip() for line in file if line.strip() != ""]
        return lines

def phase_1(input: List[str]):
    xsum: int = 0
    for expression in input:
        # print(expression)
        tokens = [t.strip() for t in split(r"( |\(|\))", expression) if t.strip() != ""]
        res = compute_equal_precedence(enumerate(tokens))
        # print(f' --> {res}')
        xsum += res
    return xsum

def phase_2(input: List[str]):
    xsum: int = 0
    for expression in input:
        # print(expression)
        tokens: List[Union[str, int]] = [t.strip() for t in split(r"( |\(|\))", expression) if t.strip() != ""]
        res = compute(tokens)
        # print(f'  --> {res}')
        xsum += res
    return xsum

def execute(input: List[str]):
    p1 = phase_1(input)
    print(f"Phase 1: {p1}")

    p2 = phase_2(input)
    print(f"Phase 2: {p2}")

if __name__ == "__main__":
    for day_input in ["18_s", "18"]:
        print(f"For {day_input}:")
        input = read_file(day_input)
        execute(input)
        print("..............")

# Answers
# 1: 800602729153
# 2: 92173009047076


from tkinter import *
import math, random
import sqlite3
import sys
import django
from time import sleep
django.setup()
# Conductor est un daemon
from threading import Thread
from components.mission import Mission
import signal

polling_interval = 1

def bddCreation():
    """
    CREATE TYPE droneStatus AS ENUM('busy','free','maintenance');

    CREATE TYPE stationStatus AS ENUM('working','empty');

    CREATE TYPE deliveryStatus AS ENUM('not assigned','in progress','cancelled'); """

    conn.commit()

    cursor.execute("""
    CREATE TABLE IF NOT EXISTS drone(
         id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE,
         status TEXT
    )
    """)

    cursor.execute("""
       CREATE TABLE IF NOT EXISTS station(
            id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE,
            status TEXT,
            xCoord FLOAT,
            yCoord FLOAT,
            batterieNumber INTEGER ,
            anticipatedBatterieNumber INTEGER
       )
       """)
    cursor.execute("""
        CREATE TABLE IF NOT EXISTS mission(
             id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE,
             status TEXT
             xStockCoord FLOAT,
             yStockCoord FLOAT,
             xFinalCoord FLOAT,
             yFinalCoord FLOAT,
             creationDate DATETIME
        )
        """)
    cursor.execute("""
        CREATE TABLE IF NOT EXISTS mission_drone(
             drone_id INTEGER,
             mission_id INTEGER,
             FOREIGN KEY(drone_id) REFERENCES drone(id),
             FOREIGN KEY(mission_id) REFERENCES mission(id)
        )
        """)
    conn.commit()

def random2DCoords():
    return [random.uniform(RAYON/2, LARGEUR-RAYON/2), random.uniform(RAYON/2, HAUTEUR-RAYON/2)]

def createDrones(number):
    i = 1
    while i < number:
        cursor.execute("""
        INSERT INTO drone(status) VALUES("free")""" )
        i = i+1
    conn.commit()

def deplacement():
    """ Dplacement de la balle """
    global X, Y, DX, DY, RAYON, LARGEUR, HAUTEUR

    # rebond  droite
    if X + RAYON + DX > LARGEUR:
        X = 2 * (LARGEUR - RAYON) - X
        DX = -DX

    # rebond  gauche
    if X - RAYON + DX < 0:
        X = 2 * RAYON - X
        DX = -DX

    # rebond en bas
    if Y + RAYON + DY > HAUTEUR:
        Y = 2 * (HAUTEUR - RAYON) - Y
        DY = -DY

    # rebond en haut
    if Y - RAYON + DY < 0:
        Y = 2 * RAYON - Y
        DY = -DY

    X = X + DX
    Y = Y + DY

    # affichage
    Canevas.coords(Balle, X - RAYON, Y - RAYON, X + RAYON, Y + RAYON)

    # mise a jour toutes les 50 ms
    Mafenetre.after(50, deplacement)

"""
    # Fenetre
    LARGEUR = 480
    HAUTEUR = 320

    # Drone
    RAYON = 5  # rayon des drones
    NUMBER_DRONE = 10

    # Station
    COTE = 5
    NUMBER_STATION = 5

    # direction initiale alatoire
    vitesse = random.uniform(1.8, 2) * 5
    angle = random.uniform(0, 2 * math.pi)
    DX = vitesse * math.cos(angle)
    DY = vitesse * math.sin(angle)

    conn = sqlite3.connect(':memory:')
    cursor = conn.cursor()
    # Cration de la fentre principale
    Mafenetre = Tk()
    Mafenetre.title("Animation Balle")

    # Cration d'un widget Canvas
    Canevas = Canvas(Mafenetre, height=HAUTEUR, width=LARGEUR, bg='white')
    Canevas.pack(padx=15, pady=15)
    bddCreation()
    createDrones(5)
    # Cration d'un objet graphique
    Balle = Canevas.create_oval(X - RAYON, Y - RAYON, X + RAYON, Y + RAYON, width=1, fill='green')

    deplacement()
    Mafenetre.mainloop()
    conn.close()
"""

# Fonction quand le thread du daemon est lance
def run():
    print("Processing... press q key to exit")
    #traitement ici
    while True:
        if mission_mgr.flag_has_new_mission:
            missions = mission_mgr.get_missions
            print(missions)
        sleep(polling_interval)

mission_mgr = Mission()
processing_th = Thread(target=run)
processing_th.start()
signal.signal(signal.SIGINT, signal_sigint_handler)
processing_th.join()