'''
Created on 2021-12-27

@author: wf
'''
from OSMPythonTools.cachingStrategy import CachingStrategy, JSON
from OSMPythonTools.nominatim import Nominatim
from geopy.geocoders import Nominatim as GeoNominatim

from pathlib import Path
import os
import logging

class NominatimWrapper(object):
    '''
    Nominatim Wrapper to hide technical details of Nominatim interface
    '''

    def __init__(self, cacheDir:str=None,user_agent:str="ConferenceCorpus"):
        '''
        Constructor

            create a nominatim instance for the given cacheDir - if cacheDir is None use ~/.nominatim as cachedir

        Args:
            cacheDir(str): the path to the cache directory to be use by Noninatims JSON caching  Strategy
            user_agent(str): the user_agent to use for the geolocator

        '''
        if cacheDir is None:
            home = str(Path.home())
            cacheDir=f"{home}/.nominatim"
        self.cacheDir=cacheDir
        if not os.path.exists(self.cacheDir):
            os.makedirs(cacheDir)
        logging.getLogger('OSMPythonTools').setLevel(logging.ERROR)
        CachingStrategy.use(JSON, cacheDir=cacheDir)
        self.nominatim = Nominatim()
        self.geolocator = GeoNominatim(user_agent=user_agent)

    def lookupWikiDataId(self,locationText:str):
        '''
        lookup the Wikidata Identifier for the given locationText (if any)

        Args:
            locationText(str): the location text to search for

        Return:
            the wikidata Q identifier most fitting the given location text

        '''
        wikidataId=None
        nresult=self.nominatim.query(locationText,params={"extratags":"1"})
        nlod=nresult._json
        if len(nlod)>0:
            nrecord=nlod[0]
            if "extratags" in nrecord:
                extratags=nrecord["extratags"]
                if "wikidata" in extratags:
                    wikidataId=extratags["wikidata"]
        return wikidataId


from urllib.request import urlopen
from bs4 import BeautifulSoup
import requests
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import numpy as np
import pandas as pd
import time
import json
from pandas.io.json import json_normalize
#We will have to use selenium

def fis():
	ids = []
	sex = []
	count = 0
	#start with the men
	startlist_list = ['https://www.biathlonworld.com/competitions/world-cup/events/bmw-ibu-world-cup-biathlon-oberhof-2021/bt2021swrlcp05/race/women-75km-sprint/bt2021swrlcp05swsp/',
'https://www.biathlonworld.com/competitions/world-cup/events/bmw-ibu-world-cup-biathlon-oberhof-2021/bt2021swrlcp05/race/men-10km-sprint/bt2021swrlcp05smsp/']
	for a in startlist_list:
		startlist = BeautifulSoup(urlopen(a), 'html.parser')
	#print(startlist)
		body = startlist.find_all('div', {'class':'pr-1 g-lg-2 g-md-2 g-sm-2 hidden-xs justify-right gray'})
		print(a)

		for b in range(len(body)):
			#print(body[a].text.strip())
			ids.append(int(body[b].text.strip()))
			#if(count==0):
			#	sex.append('M')
			#else:
			#	sex.append('L')
		count+=1

	#now for the ladies

	return ids
	#return {'id':ids, 'sex':sex}

def fantasy(startlist):
	name = []
	ski_id = []
	price =[]
	sex = []
	#sex = startlist['sex']
	#startlist = startlist['id']
	#print(sex)
	#print(startlist)
	fantasy = 'https://www.fantasyxc.se/api/athletes'
	#soup = BeautifulSoup(urlopen(fantasy), 'html5lib')
	#print(soup)
	with requests.Session() as s:
		r=s.get(fantasy)
		soup = BeautifulSoup(r.content, 'html5lib')
	API_json = json.loads(soup.get_text())
	API_df = pd.DataFrame.from_dict(pd.json_normalize(API_json), orient='columns')

	##Change to locate for increased speed
	for a in range(len(startlist)):
		athlete = API_df.loc[API_df['athlete_id']==startlist[a]]

		first_name = []
		last_name = []
		try:
			test_name = (athlete['name'].iloc[0])
		except:
			print(startlist[a])
			continue
		test_name = test_name.split(" ")
		for word in test_name:
			if word.isupper():
				last_name.append(word)
			else:
				first_name.append(word)
		first_name = ' '.join(first_name)
		last_name = ' '.join(last_name)
		test_name = first_name + " " + last_name

		name.append(test_name)
		try:
			ski_id.append(athlete['athlete_id'].iloc[0])
			price.append(athlete['price'].iloc[0])
			sex.append(athlete['gender'].iloc[0])
		except:
			print(test_name)

	d = {'name':name, 'id':ski_id, 'price':price, 'sex':sex}
	fantasy_df = pd.DataFrame(data=d)
	return fantasy_df

def pursuit(fantasydf):
	stage = [50, 46, 43, 40, 37, 34, 32, 30, 28, 26, 24, 22, 20, 18, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1]
	wc = [100, 80, 60, 50, 45, 40, 36, 32, 29, 26, 24, 2, 20, 18, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1]
	mendf = fantasydf.loc[fantasydf['sex']=='m']
	mendf = mendf.sort_values(by='elo', ascending=False)
	mendf['pursuit'] = np.arange(1, len(mendf['name'])+1, 1)
	mendf['pursuit'] = .3*mendf['pursuit'] + .7*mendf['place']
	mendf = mendf.sort_values(by='pursuit', ascending=True)
	mendf['pursuit'] = np.arange(1, len(mendf['name'])+1, 1)
	mendf = mendf[:30]
	mendf['points'] = stage

	ladiesdf = fantasydf.loc[fantasydf['sex']=='f']
	ladiesdf = ladiesdf.sort_values(by='elo', ascending=False)
	ladiesdf['pursuit'] = np.arange(1,len(ladiesdf['name'])+1,1)
	ladiesdf['pursuit'] = .3*ladiesdf['pursuit'] + .7*ladiesdf['place']
	ladiesdf = ladiesdf.sort_values(by='pursuit', ascending=True)
	ladiesdf['pursuit'] = np.arange(1,len(ladiesdf['name'])+1,1)
	ladiesdf = ladiesdf[:30]
	ladiesdf['points'] = stage

	fantasydf = mendf
	fantasydf = fantasydf.append(ladiesdf)
	return fantasydf

def elo(fantasydf):
	stage = [50, 46, 43, 40, 37, 34, 32, 30, 28, 26, 24, 22, 20, 18, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1]
	wc = [100, 80, 60, 50, 45, 40, 36, 32, 29, 26, 24, 2, 20, 18, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1]
	skier_elo = []
	df = pd.read_pickle("~/ski/elo/python/ski/men/varmen.pkl")
	ladiesdf = pd.read_pickle("~/ski/elo/python/ski/ladies/varladies.pkl")
	df = df.append(ladiesdf, ignore_index = True)
	df['name'] = df['name'].str.replace('ø', 'oe')

	df['name'] = df['name'].str.replace('ä', 'ae')
	df['name'] = df['name'].str.replace('æ', 'ae')
	df['name']= df['name'].str.replace('ö', 'oe')
	df['name']= df['name'].str.replace('ü', 'ue')
	df['name'] = df['name'].str.replace('Aleksandr Terentev', 'alexander terentev')
	df['name'] = df['name'].str.replace('Irineu Esteve Altimiras', 'ireneu esteve altimiras')
	df['name'] = df['name'].str.replace('Thomas Hjalmar Westgård', 'thomas maloney westgaard')
	df['name'] = df['name'].str.replace('Aleksandr Terentev', 'alexander terentev')
	df['name'] = df['name'].str.replace('Lauri Lepistoe', 'lauri lepisto')
	df['name'] = df['name'].str.replace('Philip Bellingham', 'phillip bellingham')
	df['name'] = df['name'].str.replace('Snorri Einarsson', 'snorri eythor einarsson')
	df['name'] = df['name'].str.replace('Krista Paermaekoski', 'krista parmakoski')
	df['name'] = df['name'].str.replace('Jessica Diggins', 'jessie diggins')
	df['name'] = df['name'].str.replace('Patricijia Eiduka', 'patricija eiduka')
	df['name'] = df['name'].str.replace('Katri Lylynperae', 'katri lylynpera')
	df['name'] = df['name'].str.replace('Julia Belger', 'julia preussger')

	fantasy_names = fantasydf['name']
	fantasy_names = fantasy_names.str.lower()
	fantasy_names  = fantasy_names.tolist()
	#print(fantasy_names)
	for a in range(len(fantasy_names)):
		skier = df.loc[df['name'].str.lower() == fantasy_names[a]]
		if(len(skier['name'])==0):
			print(fantasy_names[a])
		#print(skier)
		try:
			elo = skier['elo'].iloc[-1]
			skier_elo.append(elo)
		#elo = (skier.loc[skier['date']==20210500]['elo'])
		except:
			print(fantasy_names[a])
			skier_elo.append(1300)

	fantasydf['elo'] = skier_elo
	mendf = fantasydf.loc[fantasydf['sex']=='m']
	#Edit out these next three and the ladies three for pursuit.  One for actual

	mendf = mendf.sort_values(by='elo', ascending=False)
	mendf = mendf[:30]
	mendf['points'] = stage
	ladiesdf = fantasydf.loc[fantasydf['sex']=='f']
	ladiesdf = ladiesdf.sort_values(by='elo', ascending=False)
	ladiesdf = ladiesdf[:30]
	ladiesdf['points'] = stage
	mendf['place'] = np.arange(1, len(mendf['name'])+1, 1)
	ladiesdf['place'] = np.arange(1,len(ladiesdf['name'])+1,1)
	fantasydf = mendf
	fantasydf = fantasydf.append(ladiesdf)

	return fantasydf

	#WebDriverWait(driver, 30).until(EC.invisibility_of_element_located((By.XPATH,
	#	"//div[@class='js-off-canvas-overlay is-overlay-fixed']")))

startlist = fis()
#print(startlist)
#print(startlist)
fantasydf = (fantasy(startlist))
fantasydf = elo(fantasydf)
#fantasydf = pursuit(fantasydf)

fantasydf.to_pickle("~/ski/elo/knapsack/fantasydf.pkl")
fantasydf.to_excel("~/ski/elo/knapsack/fantasydf.xlsx")


from django.db import models
import uuid

class Character(models.Model):
    id = models.UUIDField(default=uuid.uuid4, primary_key=True, editable=False)
    name = models.CharField(max_length=20, unique=True)
    level = models.PositiveIntegerField()
    silver = models.PositiveIntegerField()
    gold = models.PositiveIntegerField()

    char_class = models.ForeignKey(
        "classes.Class",
        on_delete=models.CASCADE,
        related_name="char_class",
    )

    equipments = models.ManyToManyField(
        "equipments.Equipment",
        related_name="equipments"
    )

    skills = models.ManyToManyField(
        "skills.Skill",
        related_name="skills"
    )

    user = models.OneToOneField(
        "users.User",
        on_delete=models.CASCADE,
        related_name="user_char"
    )


"""
# @Time : 2022/12/5 14:35
# @Author : ruetrash
# @File : AlexNet_server.py
"""

import torch
import TCP.tcp as tcp
from ProtocolOnRing.secret_sharing_fixpoint import ShareFloat
from ModelAndLayers.AlexNetAndMnist import AlexNet
import ProtocolOnRing.secret_sharing_fixpoint as ssf
from ModelAndLayers.onnx_converter import onnx_converter

# 初始化参数
server = tcp.TCPServer("127.0.0.1", 9999, 40960)
server.run()
p = 0

device = "cpu"

net = AlexNet.AlexNet()
net.load_state_dict(torch.load('ModelAndLayers/AlexNetAndMnist/MNIST_bak.pkl'))
# secure_model = buildSecureModelProcessOnServer(model=net, server=server)
dummy_input = torch.ones(100, 1, 28, 28)
secure_model = onnx_converter.from_pytorch(net, dummy_input, party=server, device=device)

while 1:
    img_0 = server.receive_torch_array(device)
    input_image = ShareFloat(img_0, p, server, device)

    secure_model.set_input(input_image)
    secure_model.predict()

    output = secure_model.output['output']
    res_ = ssf.restore_float(output)
    # res_ = ssf.restore_float(secure_model.input)
    res = ssf.decode(res_)


# Standard library imports
import os

# Third party imports
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import numpy as np

# Local application imports
from modules.arrhenius import mass2conv
from modules.file_handlers import read_filtrated_datafile, read_units, get_data
from modules.reaction_models import Model
from modules.reaction_rate_numerics import data2Polrate, data2Exprate
from modules.euclidean_distance import get_best_model
from modules.regressors import activation_enthalpy
from modules.arrhenius import csv2Arrhenius, rateConstant

# basic plot settings
graph_format = 'png'
graph_dpi    = 300
font_size    = 13
lwidth       = 3

# simple scatter plot the experimental data
def graph_experimental_data(DATA_DIR,OUTPUT_DIR):

    # make output directory
    DIR = os.path.join(OUTPUT_DIR,'experimental')
    if not os.path.exists(DIR):
        os.makedirs(DIR)

    # concentration range
    low  = 0.0
    high = 1.0

    # get the csv data in a list
    Csvs = get_data(DATA_DIR)
    # filnames
    # filnames
    fnames = [f.split('.csv')[0] for f in os.listdir(DATA_DIR)]

    fig  = plt.figure()
    # export a graph for the fitting of the integral reaction rate
    Plot = os.path.join(DIR,'experimental_conversion.'+graph_format)

    for indx, Csv in enumerate(Csvs):
        df = pd.read_csv(Csv)
        # read data file
        conversion, time, temperature   = read_filtrated_datafile(df,low,high)
        # read variable units
        timeUnits, tempUnits = read_units(df)
        plt.scatter(time,conversion,s=10,label=str(temperature)+tempUnits)
        # export experimental data
        data = {
            'time'        : time,
            'conversion'  : conversion,
            'temperature' : temperature,
            'temperature_units': tempUnits
        }
        df = pd.DataFrame(data)
        csv_name = fnames[indx] + '_experimental_conversion.csv'
        df.to_csv(os.path.join(DIR,csv_name),index=False)

    plt.xlim(0,)
    plt.ylim(0,1.0)
    plt.legend()
    plt.ylabel('conversion')
    plt.xlabel('time [' + timeUnits +']')
    plt.tight_layout()
    plt.savefig(Plot, format=graph_format, dpi=graph_dpi)
    plt.close() # to avoid memory warnings

# heatmap the accuracy measures
# function for one dataframe and fname
def df2heatmap(WDIR,df,fname):

    # rank for each measure
    def df2rank(dframe,measure):
        # dframe  : the accuracy data dataframe
        # measure : particular measure (column name)
        # returns a ranking list

        # sort dataframe by the specific measure
        if measure == 'rSquared':
            df = dframe.sort_values(by=[measure],ascending=False)
        else:
            df = dframe.sort_values(by=[measure])
        # get only the model and specific measure columns into a new frame
        selDF = df[['model',measure]]
        # reset the index of this frame so the best one starts from 0
        selDF.reset_index(inplace=True)
        # sort the dataframe by the initial index, e.g. by the initial models column
        # the real index of the dataframe (not the column named index) is the ranking
        # of the models
        selDF = selDF.sort_values(by=['index'])
        rank  = [i+1 for i in selDF.index.to_list()]

        return rank

    # get only the numerical columns regarding accuracy measures
    criteria = df.columns.to_list()
    criteria.remove('model')
    criteria.remove('temperature')
    criteria.remove('temperature_units')
    if 'k_arrhenius' in criteria:
        criteria.remove('k_arrhenius')

    # get the models in a list
    models = df['model'].tolist()

    # initialise ranking dictionary
    ranking = {'models': models}

    # loop over all criteria
    for measure in criteria:
        # get the ranking for the particular measure
        rank = df2rank(df, measure)
        # parse it to dictionary
        ranking[measure] = rank

    # sns.set_theme()

    # create ranking dataframe
    rankDF = pd.DataFrame(ranking)
    rankDF = rankDF.set_index('models')
    # heatmap plot
    Plot = os.path.join(WDIR,fname+'.'+graph_format)
    fig    = plt.figure()
    sns.heatmap(rankDF, vmin=1, vmax=len(models), cmap='RdBu', linecolor='black', linewidths=0.5, annot=True)
    plt.tight_layout()
    plt.savefig(Plot, format=graph_format, dpi=graph_dpi)
    plt.close()

# export heatmaps for all csvs for all directories
def measures2heatmaps(OUTPUT_DIR):
    ext = ['integral_regression', 'differential_regression', 'conversion_regression']
    # loop over directories
    for DIR in ext:
        # set working directory
        WDIR = os.path.join(OUTPUT_DIR, DIR)
        # get all csvs in the directory
        Csvs = get_data(WDIR)
        # filnames
        fnames = [f.split('.csv')[0] for f in os.listdir(WDIR) if 'csv' in f]
        # loop over the csv files in this DIR
        for indx, Csv in enumerate(Csvs):
            df    = pd.read_csv(Csv)
            fname = fnames[indx]
            df2heatmap(WDIR, df, fname)

def sortOnData(bnames,l):
    # bnames : list of names on which data will be sorted
    # l      : list to sort
    # sl     : sorted list
    sl = []
    for bn in bnames:
        for csv in l:
            if bn in csv:
                sl.append(csv)
    return sl

def integralRegressionGraphs(DATA_DIR,OUTPUT_DIR,low,high,npoints):

    # get names (without format suffix) of the data csv files
    # bnames : base names
    bnames = [f.split('.csv')[0] for f in os.listdir(DATA_DIR)]
    # paths of the experimental data csv files
    data_Csvs = get_data(DATA_DIR)
    # metrics directory
    METRICS_DIR = os.path.join(OUTPUT_DIR, 'integral_regression')
    # paths of the metrics from the integral regression
    metrics_Csvs = get_data(METRICS_DIR)

    # zip data files and metrics
    data    = sortOnData(bnames,data_Csvs)
    metrics = sortOnData(bnames,metrics_Csvs)
    data_and_metrics = list(zip(data,metrics))

    # loop over all data
    for i_csv, csv in enumerate(data_and_metrics):

        # make directory for the graphs
        DIR       = os.path.join(METRICS_DIR,'png')
        GRAPH_DIR = os.path.join(DIR, bnames[i_csv])
        if not os.path.exists(GRAPH_DIR):
            os.makedirs(GRAPH_DIR)

        # data dataframe
        data_df    = pd.read_csv(csv[0])
        # metrics dataframe
        metrics_df = pd.read_csv(csv[1])
        # data
        conversion, time, temperature = read_filtrated_datafile(data_df,low,high)
        # read variable units
        timeUnits, tempUnits = read_units(data_df)

        modelNames = metrics_df['model'].tolist()
        ks     = metrics_df['k_arrhenius'].to_numpy()

        # loop over models
        for i_model, modelName in enumerate(modelNames):
            # pick up a model
            model = Model(modelName)
            # choose the corresponding arrhenius rate constant
            k = ks[i_model]
            # calculate experimental integral reaction rate
            y = np.array( [ model.g(a) for a in conversion ] )
            x = time
            # fit
            tfit = np.linspace(time[0], time[-1], num=npoints)
            yfit = k*tfit
            xfit = tfit

            # export a graph for the fitting of the integral reaction rate
            fig  = plt.figure()
            fname = modelName + '_' + bnames[i_csv] + '_integral_regression.'+graph_format
            Plot = os.path.join(GRAPH_DIR,fname)
            plt.scatter(x, y, s=10, label='experimental')
            plt.plot(xfit, yfit, lw=lwidth, label=r'kt')
            plt.legend()
            plt.ylabel(r'g (a)')
            plt.xlabel('time [' + timeUnits +']')
            plt.tight_layout()
            plt.savefig(Plot, format=graph_format, dpi=graph_dpi)
            plt.close() # to avoid memory warnings

def conversionRegressionGraphs(DATA_DIR,OUTPUT_DIR,low,high,npoints):

    # get names (without format suffix) of the data csv files
    # bnames : base names
    bnames = [f.split('.csv')[0] for f in os.listdir(DATA_DIR)]
    # paths of the experimental data csv files
    data_Csvs = get_data(DATA_DIR)
    # metrics directory
    METRICS_DIR = os.path.join(OUTPUT_DIR, 'conversion_regression')
    # paths of the metrics from the conversion regression
    metrics_Csvs = get_data(METRICS_DIR)
    # filter proper csvs
    metrics_Csvs = [f for f in metrics_Csvs if 'conversion_regression_accuracy' in f]

    # zip data files and metrics
    data    = sortOnData(bnames,data_Csvs)
    metrics = sortOnData(bnames,metrics_Csvs)
    data_and_metrics = list(zip(data,metrics))

    # loop over all data
    for i_csv, csv in enumerate(data_and_metrics):

        # make directory for the graphs
        DIR       = os.path.join(METRICS_DIR,'png')
        GRAPH_DIR = os.path.join(DIR, bnames[i_csv])
        if not os.path.exists(GRAPH_DIR):
            os.makedirs(GRAPH_DIR)

        # data dataframe
        data_df    = pd.read_csv(csv[0])
        # metrics dataframe
        metrics_df = pd.read_csv(csv[1])
        # data
        conversion, time, temperature = read_filtrated_datafile(data_df,low,high)
        # read variable units
        timeUnits, tempUnits = read_units(data_df)

        modelNames = metrics_df['model'].tolist()
        ks     = metrics_df['k_arrhenius'].to_numpy()

        # loop over models
        for i_model, modelName in enumerate(modelNames):
            # pick up a model
            model = Model(modelName)
            # choose the corresponding arrhenius rate constant
            k = ks[i_model]
            # calculate experimental conversion
            y = conversion
            x = time
            if modelName not in ['D2','D4']:
                # fit
                tfit = np.linspace(time[0], time[-1], num=npoints)
                yfit = np.array( [ model.alpha(t, k) for t in tfit ] )
                xfit = tfit

                # export a graph for the fitting of the integral reaction rate
                fig  = plt.figure()
                fname = modelName + '_' + bnames[i_csv] + '_conversion_regression.'+graph_format
                Plot = os.path.join(GRAPH_DIR,fname)
                plt.scatter(x, y, s=10, label='experimental')
                plt.plot(xfit, yfit, lw=lwidth, label=modelName)
                plt.legend()
                plt.ylabel(r'conversion')
                plt.xlabel('time [' + timeUnits +']')
                plt.ylim(0.0, 1.0)
                plt.tight_layout()
                plt.savefig(Plot, format=graph_format, dpi=graph_dpi)
                plt.close() # to avoid memory warnings

def differentialRegressionGraphs(DATA_DIR,OUTPUT_DIR,low,high,npoints):

    # get names (without format suffix) of the data csv files
    # bnames : base names
    bnames = [f.split('.csv')[0] for f in os.listdir(DATA_DIR)]
    # paths of the experimental data csv files
    data_Csvs = get_data(DATA_DIR)
    # metrics directory
    METRICS_DIR = os.path.join(OUTPUT_DIR, 'differential_regression')
    # paths of the metrics from the conversion regression
    metrics_Csvs = get_data(METRICS_DIR)

    # zip data files and metrics
    data    = sortOnData(bnames,data_Csvs)
    metrics = sortOnData(bnames,metrics_Csvs)
    data_and_metrics = list(zip(data,metrics))

    # loop over all data
    for i_csv, csv in enumerate(data_and_metrics):

        # make directory for the graphs
        DIR       = os.path.join(METRICS_DIR,'png')
        GRAPH_DIR = os.path.join(DIR, bnames[i_csv])
        if not os.path.exists(GRAPH_DIR):
            os.makedirs(GRAPH_DIR)

        # data dataframe
        data_df    = pd.read_csv(csv[0])
        # metrics dataframe
        metrics_df = pd.read_csv(csv[1])
        # data
        conversion, time, temperature = read_filtrated_datafile(data_df,low,high)
        # read variable units
        timeUnits, tempUnits = read_units(data_df)

        modelNames = metrics_df['model'].tolist()
        ks     = metrics_df['k_arrhenius'].to_numpy()

        # loop over models
        for i_model, modelName in enumerate(modelNames):
            # pick up a model
            model = Model(modelName)
            # choose the corresponding arrhenius rate constant
            k = ks[i_model]
            # calculate experimental conversion
            y = conversion
            x = time
            if modelName not in ['D2','D4']:
                # fit
                tfit = np.linspace(time[0], time[-1], num=npoints)
                yfit = np.array( [ model.alpha(t, k) for t in tfit ] )
                xfit = tfit

                # export a graph for the fitting of the integral reaction rate
                fig  = plt.figure()
                fname = modelName + '_' + bnames[i_csv] + '_differential_regression.'+graph_format
                Plot = os.path.join(GRAPH_DIR,fname)
                plt.scatter(x, y, s=10, label='experimental')
                plt.plot(xfit, yfit, lw=lwidth, label=modelName)
                plt.legend()
                plt.ylabel(r'conversion')
                plt.xlabel('time [' + timeUnits +']')
                plt.ylim(0.0, 1.0)
                plt.tight_layout()
                plt.savefig(Plot, format=graph_format, dpi=graph_dpi)
                plt.close() # to avoid memory warnings

def rateFitGraphs(DATA_DIR,OUTPUT_DIR,low,high,pdeg,npoints,fitExp):

    # get names (without format suffix) of the data csv files
    # bnames : base names
    bnames = [f.split('.csv')[0] for f in os.listdir(DATA_DIR)]
    # paths of the experimental data csv files
    data_Csvs = get_data(DATA_DIR)
    # metrics directory
    METRICS_DIR = os.path.join(OUTPUT_DIR, 'conversion_regression')
    # paths of the metrics from the conversion regression
    metrics_Csvs = get_data(METRICS_DIR)
    # filter proper csvs
    metrics_Csvs = [f for f in metrics_Csvs if 'conversion_regression_accuracy' in f]

    # zip data files and metrics
    data    = sortOnData(bnames,data_Csvs)
    metrics = sortOnData(bnames,metrics_Csvs)
    data_and_metrics = list(zip(data,metrics))

    # loop over all data
    for i_csv, csv in enumerate(data_and_metrics):

        # make directory for the graphs
        DIR       = os.path.join(METRICS_DIR,'png')
        DIR       = os.path.join(DIR,'rate_fit')
        GRAPH_DIR = os.path.join(DIR, bnames[i_csv])
        if not os.path.exists(GRAPH_DIR):
            os.makedirs(GRAPH_DIR)

        # data dataframe
        data_df    = pd.read_csv(csv[0])
        # metrics dataframe
        metrics_df = pd.read_csv(csv[1])
        # data
        conversion, time, temperature = read_filtrated_datafile(data_df,low,high)
        # read variable units
        timeUnits, tempUnits = read_units(data_df)

        # experimental reaction rate from polynomial conversion
        dadt_polynomial = data2Polrate(csv[0],low,high,pdeg,npoints)
        # experimental reaction rate from actual conversion
        dadt_numerical  = data2Exprate(csv[0],low,high)

        modelNames = metrics_df['model'].tolist()
        ks         = metrics_df['k_arrhenius'].to_numpy()

        # calculate experimental reaction rate
        if fitExp:
            y = dadt_numerical
        else:
            y = dadt_polynomial

        x = time

        # loop over models
        for i_model, modelName in enumerate(modelNames):
            # pick up a model
            model = Model(modelName)
            # choose the corresponding arrhenius rate constant
            k = ks[i_model]

            if modelName not in ['D2','D4']:
                # calculate the modeled differential reaction rate
                tfit = np.linspace(time[0], time[-1], num=npoints)
                yfit = np.array( [ model.alpha(t, k) for t in tfit ] )
                dadt_model = np.array( [k*model.f(a) for a in yfit] )
                yfit = dadt_model
                xfit = tfit
                # export a graph for the fitting of the integral reaction rate
                fig  = plt.figure()
                if fitExp:
                    ext = '_experimental_rate_fit.'
                else:
                    ext = '_polynomial_rate_fit.'
                fname = modelName + '_' + bnames[i_csv] + ext + graph_format
                Plot = os.path.join(GRAPH_DIR,fname)
                plt.scatter(x, y, s=10, label='experimental')
                plt.plot(xfit, yfit, lw=lwidth, label=modelName)
                plt.legend()
                plt.ylabel(r'reaction rate')
                plt.xlabel('time [' + timeUnits +']')
                plt.tight_layout()
                plt.savefig(Plot, format=graph_format, dpi=graph_dpi)
                plt.close() # to avoid memory warnings

def export_kinetic_triplet(OUTPUT_DIR):
    GRAPH_DIR = os.path.join(OUTPUT_DIR, 'desicion')
    if not os.path.exists(GRAPH_DIR):
        os.makedirs(GRAPH_DIR)

    best_model = get_best_model(OUTPUT_DIR)

    WDIR = os.path.join(OUTPUT_DIR,'conversion_regression')

    Csvs = [os.path.join(WDIR,i) for i in os.listdir(WDIR) if 'conversion_regression_accuracy.csv' in i]

    k, T = csv2Arrhenius(Csvs,best_model)

    fit_data = activation_enthalpy(k,T)

    fit_data['best_model'] = best_model

    df = pd.DataFrame.from_dict(fit_data,orient='index')
    df.to_csv(os.path.join(GRAPH_DIR,'kinetic_triplet.csv'))

    x_fit = np.linspace(round(min(T)-10),round(max(T)+10),500)
    k_fit = np.array( [ rateConstant(fit_data['frequency_factor'],fit_data['activation_enthalpy'],T) for T in x_fit ] )

    fig  = plt.figure()
    fname = 'best_model_prediction.' + graph_format

    Plot = os.path.join(GRAPH_DIR,fname)
    plt.scatter(T, k, s=50, label='experimental')
    plt.plot(x_fit, k_fit, lw=lwidth, label='prediction')
    plt.legend()
    plt.xlim(min(x_fit),max(x_fit))
    plt.ylabel('Arrhenius rate constant')
    plt.xlabel('temperature (K)')
    plt.tight_layout()
    plt.savefig(Plot, format=graph_format, dpi=graph_dpi)
    plt.close() # to avoid memory warnings

def graph_isoconversional_enthalpy(OUTPUT_DIR):
    WDIR = os.path.join(OUTPUT_DIR,'isoconversional')
    df = pd.read_csv(os.path.join(WDIR,'isoconversional_energy.csv'))
    fig  = plt.figure()
    fname = 'isoconversional_enthalpy.' + graph_format

    Plot = os.path.join(WDIR,fname)

    fig = plt.figure()
    plt.scatter(df['conversion'], df['activation_enthalpy'], s=50)
    plt.ylabel('activation enthalpy [kJ/mol]')
    plt.xlabel('conversion')
    plt.xlim(0.0,1.0)
    plt.tight_layout()
    plt.savefig(Plot, format=graph_format, dpi=graph_dpi)
    plt.close() # to avoid memory warnings


# -*- coding: utf-8 -*-
import flask
import telebot

from time import sleep

from poems import find_rhyme

try:
    import conf
except ImportError:
    import os

    conf = type(
        "conf",
        (),
        {
            "TOKEN": os.environ["TOKEN"],
            "WEBHOOK_HOST": os.environ["WEBHOOK_HOST"],
            "WEBHOOK_PORT": os.environ["WEBHOOK_PORT"],
        },
    )

WEBHOOK_URL_BASE = "https://{}:{}".format(conf.WEBHOOK_HOST, conf.WEBHOOK_PORT)
WEBHOOK_URL_PATH = "/{}/".format(conf.TOKEN)

bot = telebot.TeleBot(conf.TOKEN)
sleep(1)
bot.set_webhook(url=WEBHOOK_URL_BASE + WEBHOOK_URL_PATH)

app = flask.Flask(__name__)

@bot.message_handler(commands=["start", "help"])
def send_welcome(message):
    bot.send_message(
        message.chat.id,
        "Это бот, который подбирает рифму на ваше сообщение. За основу берутся строчки из https://github.com/IlyaGusev/PoetryCorpus.",
    )

@bot.message_handler(func=lambda m: True)
def send_len(message):
    text = message.text
    chat_id = message.chat.id

    rhyme = find_rhyme(text)
    bot.send_message(chat_id, rhyme)

@app.route("/", methods=["GET", "HEAD"])
def index():
    return "ok"

@app.route(WEBHOOK_URL_PATH, methods=["POST"])
def webhook():
    if flask.request.headers.get("content-type") == "application/json":
        json_string = flask.request.get_data().decode("utf-8")
        update = telebot.types.Update.de_json(json_string)
        bot.process_new_updates([update])
        return ""
    else:
        flask.abort(403)

if __name__ == "__main__":
    import os

    app.debug = True
    port = int(os.environ.get("PORT", 5000))
    app.run(host="0.0.0.0", port=port)


import os
import datetime

TESTS_PATH = os.path.dirname(__file__)
SECONDS_PER_DAY = 30 * 60 + 23 * 3600

# twitter dataset extracted from
path = os.path.join(TESTS_PATH, "workload.txt")

def get_workload():
    with open(path, "r") as f:
        data = f.read()
        workload = data.split(" ")
        return_workload = []
        for i, w in enumerate(workload):
            try:
                return_workload.append(int(w))
            except:
                pass
        return return_workload

def parse_date_str(date_str):
    parts = list(map(int, date_str.split(":")))
    if len(parts) == 3:
        # day:hour:minute format
        day, hour, minute = parts
        return datetime.datetime(2023, 1, day, hour, minute)
    else:
        # day:hour:minute:second format
        day, hour, minute, second = parts
        return datetime.datetime(2023, 1, day, hour, minute, second)

def twitter_workload_generator(days, damping_factor=None):
    workload_all = get_workload()
    if ":" not in days:
        # returns by days
        first, end = list(map(int, days.split("-")))
        # first = (first - 1) * seconds_per_day
        # end = end * seconds_per_day
        if damping_factor is not None:
            workload_all = list(map(lambda l: int(l/damping_factor), workload_all))
        return workload_all[first: end]

    else:
        # return by full date
        start_date_str, end_date_str = days.split("-")
        start_date = parse_date_str(start_date_str)
        end_date = parse_date_str(end_date_str)
        start_index = (
            start_date - datetime.datetime(2023, 1, 1, 0, 0, 0)
        ).total_seconds()
        end_index = (end_date - datetime.datetime(2023, 1, 1, 0, 0, 0)).total_seconds()
        if damping_factor is not None:
            workload_all = list(map(lambda l: int(l/damping_factor), workload_all))
        return workload_all[int(start_index) : int(end_index) + 1]


# -*- coding: UTF-8 -*-
import json
import logging
import traceback

import logzero
import pike.discovery.py as discovery
from flask import Blueprint, Flask, Response, make_response, \
    jsonify
from flask.views import MethodView
from werkzeug.exceptions import HTTPException

_FORMAT = '%(color)s[%(levelname)s] %(message)s%(end_color)s'

LOG = logzero.setup_logger('demo.application',
                           level=logging.INFO,
                           formatter=logzero.LogFormatter(fmt=_FORMAT))

def _catch_all(error):
    if not isinstance(error, HTTPException):
        data = error.data if hasattr(error, 'data') else None
        tb = ''.join(traceback.format_tb(error.__traceback__))
        LOG.info('v' * 78)
        LOG.info(f'{error} {type(error)} \n{tb}'.strip())
        LOG.info('^' * 78)
        code = error.error_code if hasattr(error, 'error_code') else -1
        reason = error.message if hasattr(error, 'message') else str(error)
        retval = dict(msg=reason, data=data, code=code)
        retval['traceback'] = tb
        return Response(json.dumps(retval), 500)
    else:
        # LOG.info(error, exc_info=True)
        return error

def generate_bp(module):
    # 创建蓝图
    _bp = Blueprint('provider', __name__)
    num_routes = 0
    # 自动发现路由（具有 route 属性且值是 str 且具备响应请求的能力的类）
    for v in discovery.get_all_classes(module):
        if hasattr(v, 'route') and isinstance(v.route, str):
            LOG.info(f'Adding route {v.route} from {v.__name__} ...')
            # mapping
            _bp.add_url_rule(v.route, view_func=v.as_view(v.__name__))
            num_routes += 1
    LOG.info(f'#{num_routes} routes has been successfully added.')
    return _bp

def create_app(model):
    _app = Flask(__name__)
    # 创建并注册蓝图
    bp = generate_bp(model)
    _app.register_blueprint(bp)
    # 错误处理
    _app.register_error_handler(Exception, _catch_all)
    return _app

class BaseHandler(MethodView):

    @staticmethod
    def resp_format(code=0, msg='success', data=None):
        if type(data) == Response:
            return data
        return make_response(jsonify({
            'code': code,
            'msg': msg,
            'data': data
        }))


import pandas
import numpy
import matplotlib.pyplot as plt
df=pandas.read_csv('Preprocessed/data_combined.csv')
# print (df[:5])
df['THRESTBPS'] = df['THRESTBPS'].replace(['?'],'120')
df['THRESTBPS'] = df['THRESTBPS'].astype('int64')
v=df.FBS.values=='?'
#randomly filling values with 80% with 0 and 20% with 1s
df.loc[v, 'FBS'] = numpy.random.choice(('0','1'), v.sum(), p=(0.8,0.2))
# print (df['FBS'].value_counts())
df['FBS']=df['FBS'].astype('int64')
df['CHOL'].value_counts().head()
df['CHOL']=df['CHOL'].replace('?','-69')#temporarily replacing ? with -69
df['CHOL']=df['CHOL'].astype('int64')
k=int(df[df['CHOL']!=-69]['CHOL'].mean())
df['CHOL']=df['CHOL'].replace(-69,k)
df['THALACH'].value_counts().head()
df['THALACH']=df['THALACH'].replace('?','-69')#temporarily replacing ? with -69
df['THALACH']=df['THALACH'].astype('int64')
k=int(df[df['THALACH']!=-69]['THALACH'].mean())
df['THALACH']=df['THALACH'].replace(-69,k)
v=df.EXANG.values=='?'
df.loc[v,'EXANG'] = numpy.random.choice(('0','1'), v.sum(), p=(0.61,0.39))
df['EXANG']=df["EXANG"].astype('int64')
df['OLDPEAK']=df['OLDPEAK'].replace('?','-69')#temporarily replacing ? with -69
df['OLDPEAK']=df['OLDPEAK'].astype('float64')
k=df[df['OLDPEAK']!=-69]['OLDPEAK'].mean()
df['OLDPEAK']=df['OLDPEAK'].replace(-69,numpy.round(k,1))
v=df.SLOPE.values=='?'
df.loc[v,'SLOPE'] = numpy.random.choice(('2','1','3'), v.sum(), p=(0.6,0.30,0.10))
# print df['SLOPE'].value_counts()
df['SLOPE']=df['SLOPE'].astype('int64')
v=df.CA.values=='?'
df.loc[v,'CA'] = numpy.random.choice(('0','1','2','3'), v.sum(), p=(0.60,0.20,0.13,0.07))
df['CA']=df['CA'].astype('int64')
# print df['CA'].value_counts()
df['THAL']=df['THAL'].replace('?',-1)
'''
df['THAL']=df['THAL'].replace('?',-1)
for row in df.iterrows():
    if row['THAL']==-1 and row['CATEGORY']>=1:
        df.loc[row.Index, 'ifor'] = 7

    elif row['THAL']==-1 and row['CATEGORY']==0:
        df.loc[row.Index, 'ifor'] = 3
'''
df.loc[(df['THAL']==-1)&(df['CATEGORY']!=0),'THAL']='7'
#print df['THAL'].value_counts()
df.loc[(df['THAL']==-1)&(df['CATEGORY']==0),'THAL']='3'
# print df['THAL'].value_counts()
df['THAL']=df['THAL'].astype('int64')
dummies = pandas.get_dummies(df["CP"],prefix="CP")
df = df.join(dummies)

dummies = pandas.get_dummies(df["RESTECG"],prefix="RESTECG")
df      = df.join(dummies)

dummies = pandas.get_dummies(df["SLOPE"],prefix="SLOPE")
df      = df.join(dummies)

dummies = pandas.get_dummies(df["THAL"],prefix="THAL")
df      = df.join(dummies)

#dummies = pandas.get_dummies(df["EXANG"],prefix="EXANG")
#df = df.join(dummies)

#del df['SEX']
del df['CP']
del df['RESTECG']
del df['SLOPE']
del df['THAL']
#del df['EXANG']
print (df.dtypes)

for g in df.columns:
    if df[g].dtype=='uint8':
        df[g]=df[g].astype('int64')

df.dtypes
df.loc[df['CATEGORY']>0,'CATEGORY']=1
stdcols = ["AGE","THRESTBPS","CHOL","THALACH","OLDPEAK"]
nrmcols = ["CA"]
stddf   = df.copy()
stddf[stdcols] = stddf[stdcols].apply(lambda x: (x-x.mean())/x.std())
stddf[nrmcols] = stddf[nrmcols].apply(lambda x: (x-x.min())/(x.max()-x.min()))
#stddf[stdcols] = stddf[stdcols].apply(lambda x: (x-x.mean())/(x.max()-x.min()))

for g in stdcols:
    print (g,max(stddf[g]),min(stddf[g]))

for g in nrmcols:
    print (g,max(stddf[g]),min(stddf[g]))

df_copy=stddf.copy()
df_copy=df_copy.drop(['CATEGORY'],axis=1)

dat=df_copy.values
labels=df['CATEGORY'].values

from sklearn.model_selection import train_test_split
x_train,x_test,y_train,y_test=train_test_split(dat,labels, test_size=0.25, random_state=42)

from xgboost import XGBClassifier
from sklearn.metrics import accuracy_score
from sklearn.feature_selection import SelectFromModel
from sklearn.ensemble import ExtraTreesClassifier
import xgboost as xgb
from sklearn.model_selection import GridSearchCV,cross_val_score
from sklearn import  metrics

clf = ExtraTreesClassifier()
clf = clf.fit(dat,labels)
g=clf.feature_importances_
c=stddf.drop(['CATEGORY'],axis=1).columns
model = SelectFromModel(clf, prefit=True)
X_new = model.transform(dat)

tx_train,tx_test,ty_train,ty_test=train_test_split(X_new,labels, test_size=0.25, random_state=42)

eval_set = [(x_test, y_test)]
model = XGBClassifier()
model.fit(x_train, y_train, early_stopping_rounds=10, eval_metric="logloss", eval_set=eval_set, verbose=True)
y_pred = model.predict(x_test)
predictions = [round(value) for value in y_pred]
accuracy = accuracy_score(y_test, y_pred)
print("Accuracy: %.2f%%" % (accuracy * 100.0))

from xgboost import plot_importance
from matplotlib import pyplot
plot_importance(model)
pyplot.show()
# from xgboost import plot_tree
# plot_tree(model)
# pyplot.show()
data = xgb.DMatrix(x_train,label=(y_train>0))
params = {
            'booster':'gbtree',
            'objective':'multi:softmax',
            'num_class':2,
            'eta':0.5,
            'max_depth':70,
            'subsample':1.0,
            'min_child_weight':5,
            'colsample_bytree':0.75,
            'scale_pos_weight':0.9,
            'eval_metric':'merror',
            'gamma':0.3,
            'lambda':200
}
num_round = 100
ret = xgb.cv(params,data,num_boost_round=num_round,nfold=10)

print('test-auc: ',1-ret.iloc[num_round-1][0])


import pandas as pd
import numpy as np
import tensorflow as tf
from tensorflow import keras
import os
import warnings
warnings.filterwarnings(action='ignore')
import datetime
# 데이터 입력
df = pd.read_csv("E:\Dacon\study\LSTM\equity_output_005490_H.csv", index_col=False, low_memory = False)
df.eod_date = pd.to_datetime(df.eod_date, format='%Y%m%d')
df = df.set_index("eod_date")

# 트레인, 검증 데이터 분리
from sklearn.model_selection import train_test_split
# 데이터, 레이블 분리
df_X = df.iloc[:,0:99]  # feature
df_Y = df.iloc[:,100:]  # label

X_train, X_test, Y_train, Y_test = train_test_split(df_X, df_Y, test_size = 0.3, shuffle=False)

# LSTM 모델(기본)
# ndarray 형식 변환
X_train = X_train.values
X_test = X_test.values
Y_train = Y_train.values
Y_test = Y_test.values

# timestep에 관련된 시퀀스의 정의가 없음#
print(X_train.shape, Y_train.shape) # timestep = 99, size = 917
print(X_test.shape, Y_test.shape) # timestpe = 99, size = 393
#  Keras의 RNN 계열은 3차원 데이터형식필요 각 차원은( size, timestep, feature) 순으로 reshape 필요
X_train_t = X_train.reshape(X_train.shape[0],99,1)
X_test_t = X_test.reshape(X_test.shape[0],99,1)
print(X_train_t.shape, Y_train.shape)
print(X_test_t.shape, Y_test.shape)
# LSTM 모델 만들기

# 기초 하이퍼파라미터

model = tf.keras.Sequential()
model.add(keras.layers.LSTM(
    units=1,
    input_shape=[X_train_t.shape[1], X_train_t.shape[2]],
    activation = 'relu',
    return_sequences=False
    )
)
model.add(keras.layers.Dense(units =1))

model.compile(
    loss = 'mean_squared_error',
    optimizer='adam'
)
early_stop = keras.callbacks.EarlyStopping(monitor='var_loss', patience=5)
model_path = "E:\\Dacon\\study\\LSTM\\"
filename = os.path.join(model_path, 'test_checkpoint.h5')
checkpoint = keras.callbacks.ModelCheckpoint(filename, monitor='val_loss', verbose=1, save_best_only=True, mode='auto')
history = model.fit(X_train_t, Y_train,
                    epochs=200,
                    batch_size = 16,
                    validation_split= 0.1 ,
                    verbose= 1,
                    shuffle=False,
                    callbacks = [early_stop, checkpoint])

model.load_weights(filename)
pred = model.predict(X_test_t)
Y_test
# Standardization
def data_standardization(x):
    x_np = np.asarray(x)
    return (x_np - x_np.mean()) / x_np.std()

# 너무 작거나 너무 큰 값이 학습을 방해하는 것을 방지하고자 정규화한다
# x가 양수라는 가정하에 최소값과 최대값을 이용하여 0~1사이의 값으로 변환
# Min-Max scaling
def min_max_scaling(x):
    x_np = np.asarray(x)
    return (x_np - x_np.min()) / (x_np.max() - x_np.min() + 1e-7)  # 1e-7은 0으로 나누는 오류 예방차원

# 정규화된 값을 원래의 값으로 되돌린다
# 정규화하기 이전의 org_x값과 되돌리고 싶은 x를 입력하면 역정규화된 값을 리턴한다
def reverse_min_max_scaling(org_x, x):
    org_x_np = np.asarray(org_x)
    x_np = np.asarray(x)
    return (x_np * (org_x_np.max() - org_x_np.min() + 1e-7)) + org_x_np.min()


import json
import sys

class AbstractModel(object):

    def serialize(self, allow_none=False):
        d = vars(self)
        ret = {}
        for k in d:
            if k.startswith("_"):
                continue
            key = d[k]
            if isinstance(key, AbstractModel):
                r = key.serialize(allow_none)
            elif isinstance(d[k], list):
                r = list()
                for tem in d[k]:
                    if isinstance(tem, AbstractModel):
                        r.append(tem.serialize(allow_none))
                    else:
                        r.append(
                            tem.encode("UTF-8") if isinstance(tem, type(u"")) and sys.version_info[0] == 2 else tem)
            else:
                r = d[k].encode("UTF-8") if isinstance(d[k], type(u"")) and sys.version_info[0] == 2 else d[k]
            if allow_none or r is not None:
                ret[k[0] + k[1:]] = r
        return ret

    def from_json_string(self, json_str):
        """Deserialize a JSON formatted str to a Python object"""
        params = json.loads(json_str)
        self._deserialize(params)

    def _deserialize(self, params):
        return None

    def to_json_string(self, *args, **kwargs):
        """Serialize obj to a JSON formatted str, ensure_ascii is False by default"""
        if "ensure_ascii" not in kwargs:
            kwargs["ensure_ascii"] = False
        return json.dumps(self.serialize(allow_none=True), *args, **kwargs)

    def __repr__(self):
        return "%s" % self.to_json_string()


import os
import os.path as osp
from unittest import mock

import numpy as np
from extra_data import RunDirectory, read_machinery

def test_find_proposal(tmpdir):
    prop_dir = osp.join(str(tmpdir), 'SPB', '201701', 'p002012')
    os.makedirs(prop_dir)

    with mock.patch.object(read_machinery, 'DATA_ROOT_DIR', str(tmpdir)):
        assert read_machinery.find_proposal('p002012') == prop_dir

        assert read_machinery.find_proposal(prop_dir) == prop_dir

def test_same_run(mock_spb_raw_run, mock_jungfrau_run, mock_scs_run):
    run_spb = RunDirectory(mock_spb_raw_run)
    run_jf = RunDirectory(mock_jungfrau_run)
    run_scs = RunDirectory(mock_scs_run)

    assert run_spb.is_single_run
    assert run_jf.is_single_run
    assert run_scs.is_single_run

    assert not read_machinery.same_run(run_spb, run_scs, run_jf)

    format_version = run_spb.files[0].format_version
    s1 = run_spb.select_trains(np.s_[:1])
    s2 = run_spb.select_trains(np.s_[:-1])
    s3 = run_spb.select_trains(np.s_[3])

    s4 = run_spb.select('SA1_XTD2_XGM/DOOCS/MAIN', '*')
    s5 = run_spb.select('SPB_IRU_CAM/CAM/SIDEMIC:daqOutput', '*')

    if run_spb.run_metadata()['dataFormatVersion'] != '0.5':
        assert read_machinery.same_run(s1, s2, s3)
        assert read_machinery.same_run(s4, s5)
    else:
        assert not read_machinery.same_run(s1, s2, s3)
        assert not read_machinery.same_run(s4, s5)

    # SourceData
    sd = run_spb['SA1_XTD2_XGM/DOOCS/MAIN']
    sd1 = sd.select_trains(np.s_[:1])
    sd2 = sd.select_trains(np.s_[-1:])
    assert sd.is_single_run
    if sd.run_metadata()['dataFormatVersion'] != '0.5':
        assert read_machinery.same_run(sd1, sd2)
    else:
        assert not read_machinery.same_run(sd1, sd2)


import logging
from math import ceil

from matplotlib import pyplot as plt
from matplotlib import colors
import matplotlib.patches as patches
import numpy as np

from am.segment.image_utils import overlay_source_mask
from am.utils import min_max, time_it

logger = logging.getLogger('am-segm')

def cut_patch(image, y_offset=0, x_offset=0, patch=1000):
    return image[y_offset:y_offset+patch, x_offset:x_offset+patch]

def plot_image(image, figsize=(16, 10), dpi=100, **kwargs):
    fig, ax = plt.subplots(figsize=figsize, dpi=dpi)
    ax.imshow(image, **kwargs)
    return fig, ax

def plot_axis_hist(axis_hist, axis_coords, labels, uniq_labels):
    fig, ax = plt.subplots(1, 1, figsize=(20, 10))
    ax.plot(np.arange(axis_hist.shape[0]), axis_hist, c='black')

    norm = colors.Normalize(vmin=min(uniq_labels), vmax=max(uniq_labels))
    cmap = plt.cm.get_cmap('jet', len(uniq_labels))
    for label in uniq_labels:
        x_min, x_max = min_max(axis_coords[labels == label])
        y_min, y_max = min_max(axis_hist)

        rect = patches.Rectangle(xy=(x_min, y_min),
                                 width=x_max - x_min, height=y_max - y_min,
                                 color=cmap(norm(label)))
        ax.add_patch(rect)

def plot_labels(ax, image, target_axis, axis_coords, labels, uniq_labels):
    norm = colors.Normalize(vmin=min(uniq_labels), vmax=max(uniq_labels))
    cmap = plt.cm.get_cmap('jet', len(uniq_labels))
    for label in uniq_labels:
        if target_axis == 1:
            x_min, x_max = min_max(axis_coords[labels == label])
            y_min, y_max = (0, image.shape[0])
        else:
            x_min, x_max = (0, image.shape[1])
            y_min, y_max = min_max(axis_coords[labels == label])

        rect = patches.Rectangle(xy=(x_min, y_min),
                                 width=x_max - x_min, height=y_max - y_min,
                                 alpha=0.4, color=cmap(norm(label)))
        ax.add_patch(rect)

def plot_image_label_overlay(image, target_axis, axis_coords, labels, uniq_labels):
    fig, ax = plt.subplots(1, 1, figsize=(10, 10))
    ax.imshow(image, cmap='gray', alpha=0.6)
    plot_labels(image, ax, target_axis, axis_coords, labels, uniq_labels)

def shift_center_coords(centers, indices, row_offset, col_offset, patch_size):
    mask = (row_offset < centers[:, 0]) & (centers[:, 0] < row_offset + patch_size) &\
           (col_offset < centers[:, 1]) & (centers[:, 1] < col_offset + patch_size)
    shifted_centers = centers[mask]
    shifted_centers[:, 0] -= row_offset
    shifted_centers[:, 1] -= col_offset
    return shifted_centers, indices[mask]

@time_it
def plot_am_labels(
    mask, centers, labels, source=None, row_offset=None, col_offset=None, patch_size=None
):
    if patch_size:
        centers, labels = shift_center_coords(centers, labels, row_offset, col_offset, patch_size)
        mask = mask[row_offset : (row_offset + patch_size), col_offset : (col_offset + patch_size)]
        if source is not None:
            source = source[
                row_offset : (row_offset + patch_size), col_offset : (col_offset + patch_size)
            ]

    dpi = 100
    figsize = (ceil(mask.shape[1] / dpi), ceil(mask.shape[0] / dpi))
    fig, ax = plt.subplots(figsize=figsize, dpi=dpi)

    overlay = overlay_source_mask(source, mask)
    ax.imshow(np.array(overlay), cmap='viridis', interpolation=None)

    circle_size = (3 * 72 / fig.dpi) ** 2  # circle size (diameter**2) in points
    fontsize = 7.5 * 72 / fig.dpi  # text size in points
    ys, xs = zip(*centers)
    ax.scatter(xs, ys, color='green', s=circle_size, linewidth=0)
    for (y, x), label in zip(centers, labels):
        ax.text(x, y, str(label), color='blue', fontsize=fontsize)

@time_it
def overlay_image_with_am_labels(source, mask, am_centers, acq_indices, path):
    logger.info(f'Overlaying image with AM labels and saving at {path}')
    plot_am_labels(mask=mask, centers=am_centers, labels=acq_indices, source=source)
    plt.axis('off')
    plt.savefig(path, bbox_inches='tight')
    plt.close()

def plot_coord_peaks(hist, peaks, height=None):
    fig, ax = plt.subplots(figsize=(15, 5))
    ax.plot(hist)
    ax.plot(peaks, hist[peaks], 'x')
    if height:
        ax.plot(np.ones_like(hist) * height, '--')


import json
from flask import request, _request_ctx_stack
from functools import wraps
from jose import jwt
from urllib.request import urlopen

AUTH0_DOMAIN = 'jordan-flask-authentication-practice.auth0.com'
ALGORITHMS = ['RS256']
API_AUDIENCE = 'coffee_shop'

class AuthError(Exception):
    """
    Exception subclass to handle authentication errors raised while verifying
    JWT tokens.

    Attributes:
        error (dict: {error code, description}): Human readable string
        describing the exception.
        status_code (int): HTTP error code.
    """

    def __init__(self, error, status_code):
        self.error = error
        self.status_code = status_code

def get_token_auth_header():
    """
    Obtains the access token from the request's header.
    request > header > authorization > bearer & token

    Args:
        NONE: No parameters are required but this function relies on access
        to Flask's request object.

    Returns:
        JWT token string.

    Raises:
        KeyError: Raises an AuthError if the authorization header malformed or
        is missing.
    """

    auth = request.headers.get('Authorization', None)
    if not auth:
        raise AuthError({
            'code': 'authorization_header_missing',
            'description': 'Authorization header is expected.'
        }, 401)

    parts = auth.split()
    if parts[0].lower() != 'bearer':
        raise AuthError({
            'code': 'bearer_missing',
            'description': 'Authorization header must start with "Bearer".'
        }, 401)

    elif len(parts) == 1:
        raise AuthError({
            'code': 'token_missing',
            'description': 'Token not found.'
        }, 401)

    elif len(parts) > 2:
        raise AuthError({
            'code': 'inavlid_token',
            'description': 'Too many authorization headers.'
        }, 401)

    token = parts[1]
    return token

def check_permissions(permission, payload):
    """
    Checks if the requesting user is authorized to perform the action stated
    in the permission paramter.

    Args:
        permission: String value of the requesting permission; 'get:drinks'.
        payload: Dictionary of authorization data.

    Returns:
        True if the requesting permission is authorized.

    Raises:
        KeyError: Raises an AuthError.
    """

    if "permissions" not in payload:
        raise AuthError({
            'code': 'permissions_unavailable',
            'description': 'Permissions were not included in the payload.'
        }, 401)

    if permission not in payload["permissions"]:
        raise AuthError({
            'code': 'invalid_permission',
            'description': 'User does not have authorization; invalid permission provided.'
        }, 401)

    return True

def verify_decode_jwt(token):
    """
    Validates the user's JWT token and returns the contained payload. This
    function uses a public key (rsa_key) to verify the token's validity, which
    is retrieved from appending '/.well-known/jwks.json' to the domain.

    Args:
        token: String containing the JWT token.

    Returns:
        Payload contained in the JWT token IF the token is valid.

    Raises:
        KeyError: Raises an AuthError if the token is not valid.
    """

    # Retrieve token header
    unverified_header = jwt.get_unverified_header(token)
    rsa_key = {}
    if 'kid' not in unverified_header:
        raise AuthError({
            'code': 'invalid_header',
            'description': 'Authorization malformed.'
        }, 401)

    # Retrieve public key (RSA key)
    jsonurl = urlopen(f'https://{AUTH0_DOMAIN}/.well-known/jwks.json') # https://jordan-flask-authentication-practice.auth0.com/.well-known/jwks.json
    jwks = json.loads(jsonurl.read())
    for key in jwks['keys']:
        if key['kid'] == unverified_header['kid']:
            rsa_key = {
                'kty': key['kty'],     # Key type (i.e. RSA)
                'kid': key['kid'],     # Key ID (i.e. M0U5...)
                'use': key['use'],     # Use is to sign (i.e. sig) the token
                'n': key['n'],     # (i.e. u8ptXo...)
                'e': key['e']     # (i.e. AQAB)
            }

    # Decode using public key
    if rsa_key:
        try:
            payload = jwt.decode(
                token,
                rsa_key,
                algorithms=ALGORITHMS,
                audience=API_AUDIENCE,
                issuer='https://' + AUTH0_DOMAIN + '/'
            )

            return payload

        except jwt.ExpiredSignatureError:
            raise AuthError({
                'code': 'token_expired',
                'description': 'Token expired.'
            }, 401)

        except jwt.JWTClaimsError:
            raise AuthError({
                'code': 'invalid_claims',
                'description': 'Incorrect claims. Please, check the audience and issuer.'
            }, 401)
        except Exception:
            raise AuthError({
                'code': 'invalid_header',
                'description': 'Unable to parse authentication token.'
            }, 400)
    raise AuthError({
                'code': 'invalid_header',
                'description': 'Unable to find the appropriate key.'
            }, 400)

def requires_auth(permission=''):
    def requires_auth_decorator(f):
        @wraps(f)
        def wrapper(*args, **kwargs):
            print("token = get_token_auth_header()")
            token = get_token_auth_header()
            print("payload = verify_decode_jwt(token)")
            payload = verify_decode_jwt(token)
            print("check_permissions(permission, payload)")
            check_permissions(permission, payload)
            return f(*args, **kwargs)

        return wrapper
    return requires_auth_decorator


import tensorflow as tf
import numpy as np
import pandas as pd
import pickle
from nltk import tokenize
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.utils import class_weight

from tensorflow.python.keras import models, initializers, regularizers
from tensorflow.python.keras.metrics import BinaryAccuracy, Accuracy
from tensorflow.python.keras.preprocessing.text import Tokenizer
from tensorflow.python.keras.preprocessing.sequence import pad_sequences
from tensorflow.python.keras.layers import Dense, Conv1D, Dropout, Embedding, LSTM, SpatialDropout1D, GlobalMaxPool1D, Bidirectional

from tensorflow.python.keras.callbacks import EarlyStopping
from helper import load_glove_embedding, return_labels, Dataset
from lstm_attention_model import AttentionLSTM, train_part, evaluate, predict_labels

_embedding_index_glove = None

class TextClassifier:

    def __init__(self, tokenizer, label_index, max_word_count = 50000, max_sequence_len = 100,
                 word_embedding_dim = 50, verbose=1):
        self.max_word_count = max_word_count
        self.max_sequence_len = max_sequence_len
        self.embedding_dim = word_embedding_dim
        self.tokenizer = tokenizer
        self.label_index = label_index
        self.history = None
        self.model = None
        self.embedding_matrix_glove = None
        self.verbose = verbose

    def _train_LSTM(self, X_train, y_train, epochs = 5, batch_size = 64, learning_rate = 0.001, reg = 0.01):
        """
        Trains LSTM
        - X_train: Input sequence
        - y_train: Target sequence
        - epochs
        - batch_size
        - learning_rate = Adam optimizer's learning rate
        - reg: Regularization
        Returns :
        - history: Scalar loss
        """

        flatten_y = [category for sublist in y_train for category in sublist]
        class_weights = class_weight.compute_class_weight('balanced', np.unique(flatten_y), flatten_y)
        optim = tf.keras.optimizers.Adam(learning_rate=learning_rate)

        model = models.Sequential()
        model.add(Embedding(input_dim=self.max_word_count, output_dim=self.embedding_dim))
        model.add(SpatialDropout1D(0.2))
        model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
        model.add(Dense(32, activation='relu'))
        model.add(Dense(8, activation='sigmoid'))
        model.compile(loss='binary_crossentropy', optimizer=optim, metrics=[BinaryAccuracy()])

        history = model.fit(X_train, y_train, class_weight = class_weight, epochs = epochs,
                    batch_size=batch_size, validation_split=0.25, verbose = self.verbose,
                    callbacks=[EarlyStopping(monitor='val_loss', patience=3, min_delta=0.0001)])

        self.model = model
        self.history = history.history

    def _train_LSTM_1(self, X_train, y_train, epochs = 5, batch_size = 64, learning_rate = 0.001, reg = 0.01):
        """
        Trains LSTM
        - X_train: Input sequence
        - y_train: Target sequence
        - epochs
        - batch_size
        - learning_rate = Adam optimizer's learning rate
        - reg: Regularization
        Returns :
        - history: Scalar loss
        """
        flatten_y = [category for sublist in y_train for category in sublist]
        class_weights = class_weight.compute_class_weight('balanced', np.unique(flatten_y), flatten_y)
        optim = tf.keras.optimizers.Adam(learning_rate=learning_rate)

        model = models.Sequential()
        model.add(Embedding(self.max_word_count, 64))
        model.add(Bidirectional(LSTM(64, return_sequences=True)))
        model.add(Bidirectional(LSTM(32)))
        model.add(Dense(64, kernel_regularizer=regularizers.l2(reg), activation='relu'))
        model.add(Dropout(0.5))
        model.add(Dense(8, activation='sigmoid'))
        model.compile(loss='binary_crossentropy', optimizer=optim, metrics=[BinaryAccuracy()])

        history = model.fit(X_train, y_train, class_weight = class_weight, epochs = epochs,
            batch_size=batch_size, validation_split=0.25, verbose = self.verbose,
            callbacks=[EarlyStopping(monitor='val_loss', patience=3, min_delta=0.0001)])

        self.model = model
        self.history = history.history

    def _train_CNN(self, X_train, y_train, epochs = 5, batch_size = 64, learning_rate = 0.001, regularization = 0.01):
        """
        Trains CNN
        - X_train: Input sequence
        - y_train: Target sequence
        - epochs
        - batch_size
        - learning_rate = Adam optimizer's learning rate
        - reg: Regularization
        Returns :
        - history: Scalar loss
        """
        flatten_y = [category for sublist in y_train for category in sublist]
        class_weights = class_weight.compute_class_weight('balanced', np.unique(flatten_y), flatten_y)
        optim = tf.keras.optimizers.Adam(learning_rate=learning_rate)

        model = models.Sequential()
        model.add(Embedding(input_dim=self.max_word_count, output_dim=self.embedding_dim))
        model.add(Conv1D(filters=300, kernel_size=3, padding='valid', activation='relu', strides=1))
        model.add(GlobalMaxPool1D())
        model.add(Dense(8, activation='sigmoid'))
        model.compile(loss='binary_crossentropy', optimizer=optim, metrics=[BinaryAccuracy()])
        history = model.fit(X_train, y_train, class_weight = class_weight, epochs = epochs,
                            batch_size=batch_size, validation_split=0.25, verbose = self.verbose,
                            callbacks=[EarlyStopping(monitor='val_loss', patience=3, min_delta=0.0001)])

        self.model = model
        self.history = history.history

    def _train_CNN_Glove(self, X_train, y_train, epochs = 5, batch_size = 64, learning_rate = 0.001, regularization = 0.01):
        """
        Trains CNN
        - X_train: Input sequence
        - y_train: Target sequence
        - epochs
        - batch_size
        - learning_rate = Adam optimizer's learning rate
        - reg: Regularization
        Returns :
        - history: Scalar loss
        """
        flatten_y = [category for sublist in y_train for category in sublist]
        class_weights = class_weight.compute_class_weight('balanced', np.unique(flatten_y), flatten_y)
        optim = tf.keras.optimizers.Adam(learning_rate=learning_rate)

        embedding_matrix = self.create_embedding_matrix()

        model = models.Sequential()
        model.add(Embedding(input_dim=self.max_word_count, output_dim=100,
                            embeddings_initializer = initializers.Constant(embedding_matrix),
                            input_length=self.max_sequence_len, trainable=False))
        model.add(Conv1D(filters=300, kernel_size=3, padding='valid', activation='relu', strides=1))
        model.add(GlobalMaxPool1D())
        model.add(Dense(8, activation='sigmoid'))
        model.compile(loss='binary_crossentropy', optimizer=optim, metrics=[BinaryAccuracy()])
        history = model.fit(X_train, y_train, class_weight = class_weight, epochs = epochs,
                            batch_size=batch_size, validation_split=0.25, verbose = self.verbose,
                            callbacks=[EarlyStopping(monitor='val_loss', patience=3, min_delta=0.0001)])

        self.model = model
        self.history = history.history

    def _train_CNN_Glove_1(self, X_train, y_train, epochs = 5, batch_size = 64, learning_rate = 0.001, regularization = 0.01):
        """
        Trains CNN
        - X_train: Input sequence
        - y_train: Target sequence
        - epochs
        - batch_size
        - learning_rate = Adam optimizer's learning rate
        - reg: Regularization
        Returns :
        - history: Scalar loss
        """
        flatten_y = [category for sublist in y_train for category in sublist]
        class_weights = class_weight.compute_class_weight('balanced', np.unique(flatten_y), flatten_y)
        optim = tf.keras.optimizers.Adam(learning_rate=learning_rate)

        embedding_matrix = self.create_embedding_matrix()

        model = models.Sequential()
        model.add(Embedding(self.max_word_count, 100, embeddings_initializer = initializers.Constant(embedding_matrix),
                            input_length=self.max_sequence_len, trainable=False))
        model.add(Conv1D(filters=128, kernel_size=5, padding='valid', activation='relu', strides=1))
        model.add(Conv1D(filters=256, kernel_size=3, padding='valid', activation='relu', strides=1))
        model.add(GlobalMaxPool1D())
        model.add(Dense(8, activation='sigmoid'))
        model.compile(loss='binary_crossentropy', optimizer=optim, metrics=[BinaryAccuracy()])
        history = model.fit(X_train, y_train, class_weight = class_weight, epochs = epochs,
                            batch_size=batch_size, validation_split=0.25,
                            callbacks=[EarlyStopping(monitor='val_loss', patience=3, min_delta=0.0001)])

        self.model = model
        self.history = history.history

    def _train_MLP(self, X_train, y_train, epochs = 5, batch_size = 64, learning_rate = 0.001, reg = 0.01):
        """
        Trains LSTM
        - X_train: Input sequence
        - y_train: Target sequence
        - epochs
        - batch_size
        - learning_rate = Adam optimizer's learning rate
        - reg: Regularization
        Returns :
        - history: Scalar loss
        """
        flatten_y = [category for sublist in y_train for category in sublist]
        class_weights = class_weight.compute_class_weight('balanced', np.unique(flatten_y), flatten_y)
        optim = tf.keras.optimizers.Adam(learning_rate=learning_rate)

        model = models.Sequential()
        model.add(Dense(512, activation='relu', kernel_regularizer=regularizers.l2(reg), input_shape=(self.max_sequence_len,)))
        model.add(Dropout(0.2))
        for _ in range(3):
            model.add(Dense(256, activation='relu'))
            model.add(Dropout(0.2))
        model.add(Dense(8, activation='sigmoid'))
        model.compile(loss='binary_crossentropy', optimizer=optim, metrics=[BinaryAccuracy()])
        history = model.fit(X_train, y_train, class_weight = class_weight, epochs = epochs,
                            batch_size=batch_size, validation_split=0.25, verbose = self.verbose,
                            callbacks=[EarlyStopping(monitor='val_loss', patience=3, min_delta=0.0001)])

        self.model = model
        self.history = history.history

    def _train_Attention(self, X_train, y_train, epochs = 5, batch_size = 64, learning_rate = 0.001, reg = 0.01):
        """
        Trains BI-LSTM with Attention. Reference: https://www.aclweb.org/anthology/P16-2034/
        Note: this is model has a different api and was included here to be compatible with the cross-validation
        code in the notebook.
        - X_train: Input sequence
        - y_train: Target sequence
        - epochs
        - batch_size
        - learning_rate = Adam optimizer's learning rate
        - reg: Regularization
        Returns :
        - history: Scalar loss
        """
        X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.25, random_state=1)
        # Split data into batches of 64
        train_dset = Dataset(X_train, y_train, batch_size=batch_size, shuffle=True)
        val_dset = Dataset(X_val, y_val, batch_size=batch_size, shuffle=True)

        model = AttentionLSTM(self.max_sequence_len, self.embedding_dim)

        history = train_part(model, train_dset, val_dset,num_epochs=10,
                             is_training=True, learning_rate=learning_rate, verbose=self.verbose)

        # Update the key name for consistency.
        history['binary_accuracy'] = history.pop('accuracy')
        history['val_binary_accuracy'] = history.pop('val_accuracy')

        self.model = model
        self.history = history

    def train(self, X_train, y_train, model_name, epochs = 5, batch_size = 64,
                   learning_rate = 0.001, regularization = 0.01):
        """
        An api for training the model
        - X_train: Input sequence
        - y_train: Target sequence
        - epochs
        - batch_size
        - learning_rate = Adam optimizer's learning rate
        - reg: Regularization
        Returns :
        - history: Scalar loss
        """
        if model_name == 'LSTM':
            self._train_LSTM(X_train, y_train, epochs, batch_size, learning_rate, regularization)
        elif model_name == 'CNN':
            self._train_CNN(X_train, y_train, epochs, batch_size, learning_rate, regularization)
        elif model_name == 'MLP':
            self._train_MLP(X_train, y_train, epochs, batch_size, learning_rate, regularization)
        elif model_name == 'CNN_Glove':
            self._train_CNN_Glove(X_train, y_train, epochs, batch_size, learning_rate, regularization)
        elif model_name == 'Attention':
            self._train_Attention(X_train, y_train, epochs, batch_size, learning_rate, regularization)
        else:
            raise ValueError('Model not defined')

        return self.history

    def predict(self, review):
        """
        This function takes in a list of sentences as input, performs binary classification for each
        of the categories, and selects a sentence for prediction if it's confidence is > 0.5 for a
        particular category.

        Note: The Attention model has a different api and won't be supported here.

        - review: list of sentences to be classified
        - model_name: Model type
        Returns :
        - prediction_list: A list of important sentences in the form of (confidence, category, sentence)
        """
        seq = self.tokenizer.texts_to_sequences(review)
        padded = pad_sequences(seq, maxlen = self.max_sequence_len)

        pred = self.model.predict(padded)

        filtered_indices = np.where(np.array(pred) > 0.5)
        i_review, j_label = filtered_indices[0], filtered_indices[1]

        prediction_list = return_labels(i_review, j_label, pred, review, self.label_index)

        return prediction_list

    def evaluate(self, X_test, y_test):
        """
        Model evaluation on the test set. It uses the binary accuracy metric to compute the score

        Note: The Attention model has a different api and won't be supported here.

        - X_test:
        - y_test:
        Returns :
        - test_loss: Binary cross entropy loss
        - test_acc: binary accuracy score
        """
        test_loss, test_acc = self.model.evaluate(X_test,y_test)

        return test_loss, test_acc

    def create_embedding_matrix(self):
        """
        Model evaluation on the test set. It uses the binary accuracy metric to compute the score

        Note: The Attention model has a different api and won't be supported here.

        - X_test:
        - X_test:
        Returns :
        - test_loss: Binary cross entropy loss
        - test_acc: binary accuracy score
        """
        self.load_glove()

        embedding_matrix = np.zeros((50000, 100))
        for word, index in self.tokenizer.word_index.items():
            if index > 50000 - 1:
                break
            else:
                if word in _embedding_index_glove:
                    embedding_matrix[index] = _embedding_index_glove[word]

        return embedding_matrix

    def load_glove(self):
        """
        This function loads the glove embedding values in the global variable.
        It avoids reloading the same instance over and over.
        """
        global _embedding_index_glove

        if _embedding_index_glove is None:
            _embedding_index_glove =  load_glove_embedding()

def tokenize_data(X, y, max_word_count=50000, max_sequence_len=100):
    """
    This function tokenizes the text blocks, pads the tokenized text blocks to
    have uniform length.
    - X: list of strings
    - y: list of strings
    - max_word_count:
    - max_sequence_len:
    Returns :
    - X_tokenized: Tokenized X
    - y_tokenized: Tokenized y
    - tokenizer: tokenizer object, which contains all the mapping of words and indices
    - label_index: index to word mapping, a dict
    """
    tokenizer = Tokenizer(num_words=max_word_count, filters='!"#$%&()*+,-./:;<=>?@[\]^_`{|}~', lower=True)
    tokenizer.fit_on_texts(X)
    word_index = tokenizer.word_index

    X_tokenized = tokenizer.texts_to_sequences(X)
    X_tokenized = pad_sequences(X_tokenized, maxlen = max_sequence_len)

    mlb = MultiLabelBinarizer()
    y_tokenized = mlb.fit_transform(y)

    label_index = mlb.classes_

    print('Shape of data tensor:', X.shape)
    print('Found %s unique tokens.' % len(word_index))

    return X_tokenized, y_tokenized, tokenizer, label_index

def split_to_sentences(paragraph):
    """
    This function uses the nltk library to split a document into sentences
    - paragraph: String
    Returns :
    - tokenized: list of sentences
    """
    return tokenize.sent_tokenize(paragraph)

def clean_data(text):
    """
    This function cleans up the string by removing trailing spaces and punctutations.
    - text: String
    Returns :
    - text:
    """
    text = text.lower().strip()
    text = text.strip(".")
    # Remove double space
    # Trim input?
    return text

def extract_text_blocks(review):
    """
    This functions splits the string into sentence, then does basic text cleansing
    and flattens the list of sentences.
    - review: Review in the form of a string
    Returns :
    - flat_list: A flattened list of sentences
    """
    sentences = split_to_sentences(review)
    #text_block = [sentence.split(",") for sentence in sentences]
    #flat_list = [clean_data(block) for sublist in text_block for block in sublist]
    #return flat_list
    return sentences


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Plot plastic fluxes to Seychelles by country and sink location
@author: Noam Vogt-Vincent
"""

### TO DO!!!!!
# Sources are currently excluded if they are not in the top X in a source, but are
# in others (should be included)

import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import cmasher as cmr
import matplotlib.colors as colors
import matplotlib.ticker as mticker
import cartopy.crs as ccrs
import cartopy.feature as cfeature
from netCDF4 import Dataset
from glob import glob

##############################################################################
# DIRECTORIES & PARAMETERS                                                   #
##############################################################################

# PARAMETERS
param = {# Analysis parameters
         'ls_d': 3650,      # Sinking timescale (days)
         'lb_d': 40,        # Beaching timescale (days)
         'title': 'Debris sources for zero windage, l(s)=10a, l(b)=20d',

         # Bounds
         'lon_west': 20,
         'lon_east': 130,
         'lat_south': -40,
         'lat_north': 30,
         }

# DIRECTORIES
dirs = {'script': os.path.dirname(os.path.realpath(__file__)),
        'plastic': os.path.dirname(os.path.realpath(__file__)) + '/../PLASTIC_DATA/',
        'grid': os.path.dirname(os.path.realpath(__file__)) + '/../GRID_DATA/',
        'traj': os.path.dirname(os.path.realpath(__file__)) + '/../TRAJ/',
        'fig': os.path.dirname(os.path.realpath(__file__)) + '/../FIGURES/'}

# FILE HANDLES
fh = {'sink_list': dirs['plastic'] + 'sink_list.in',
      'grid':    dirs['grid'] + 'griddata.nc',
      'fig': dirs['fig'] + 'marine_plastic_sources_' + str(param['ls_d']) + '_b' + str(param['lb_d']),
      'data': sorted(glob(dirs['traj'] + 'marine_data_s' + str(param['ls_d']) + '_b' + str(param['lb_d']) + '*.pkl')),
      'cmap': dirs['fig'] + 'cmap_data.pkl'}

##############################################################################
# CALCULATE FLUXES                                                           #
##############################################################################

sink_list = pd.read_csv(fh['sink_list'])
nsink = np.shape(sink_list)[0]

# Grid accumulating fluxes by sink site
# Firstly construct grid
with Dataset(fh['grid'], mode='r') as nc:
    lon = nc.variables['lon_psi'][:]
    lat = nc.variables['lat_psi'][:]
    lon_bnd = np.concatenate([nc.variables['lon_rho'][:], [180]])
    lat_bnd = nc.variables['lat_rho'][:]

    lon = lon[(lon >= param['lon_west'])*(lon <= param['lon_east'])]
    lat = lat[(lat >= param['lat_south'])*(lat <= param['lat_north'])]
    lon_bnd = lon_bnd[(lon_bnd >= param['lon_west'])*(lon_bnd <= param['lon_east'])]
    lat_bnd = lat_bnd[(lat_bnd >= param['lat_south'])*(lat_bnd <= param['lat_north'])]

# Create empty grids for gridding
grid = np.zeros((nsink, len(lat), len(lon)))

for data_fh in fh['data']:
    # if data_fh != fh['data'][0]: # TEMPORARY TIME SAVING HACK, REMOVE!
    #     break

    data = pd.read_pickle(data_fh)
    print(data_fh)

    for sinkidx, sink in enumerate(sink_list['Sink code']):
        # Filter by sink site
        data_sink = data.loc[data['sink_id'] == sink]
        print(sinkidx)

        grid[sinkidx, :, :] += np.histogram2d(data_sink['lon0'].values,
                                              data_sink['lat0'].values,
                                              bins=[lon_bnd, lat_bnd],
                                              weights=data_sink['plastic_flux'].values)[0].T

##############################################################################
# PLOT                                                                       #
##############################################################################

f, ax = plt.subplots(1, 1, figsize=(20, 12), constrained_layout=True,
                     subplot_kw={'projection': ccrs.PlateCarree()})
plt.subplots_adjust(bottom=0, top=1)

data_crs = ccrs.PlateCarree()

mass_flux = ax.pcolormesh(lon_bnd, lat_bnd, np.sum(grid[:,:,:], axis=0)/(len(fh['data'])*1728),
                          norm=colors.LogNorm(vmin=1e-5, vmax=1e-2), cmap=cmr.torch, transform=data_crs)
# Multiplication by years * months per year * particles released per cell per month

ax.spines['geo'].set_linewidth(1)
ax.set_ylabel('Latitude')
ax.set_xlabel('Longitude')
ax.set_aspect('equal')
# ax.set_title('Fraction of mass from cell beaching at Seychelles')

cax1 = f.add_axes([ax.get_position().x1+0.0105,ax.get_position().y0,0.015,ax.get_position().height])

cb1 = plt.colorbar(mass_flux, cax=cax1, pad=0.1)
cb1.set_label('Fraction of mass from cell beaching at Seychelles', size=12)

land_10m = cfeature.NaturalEarthFeature('physical', 'land', '10m',
                                        edgecolor='white', linewidth=0.5,
                                        facecolor='black')
ax.add_feature(land_10m)

gl = ax.gridlines(crs=data_crs, draw_labels=True, linewidth=1, color='black', linestyle='-')
gl.xlocator = mticker.FixedLocator(np.arange(20, 140, 20))
gl.ylocator = mticker.FixedLocator(np.arange(-40, 40, 20))
gl.ylabels_right = False
gl.xlabels_top = False

plt.savefig(fh['fig'] + '_all.png', dpi=300, bbox_inches='tight')

###############################################################################

f, ax = plt.subplots(1, 1, figsize=(20, 12), constrained_layout=True,
                     subplot_kw={'projection': ccrs.PlateCarree()})
plt.subplots_adjust(bottom=0, top=1)

data_crs = ccrs.PlateCarree()

mass_flux = ax.pcolormesh(lon_bnd, lat_bnd, np.sum(grid[:4,:,:], axis=0)/(len(fh['data'])*1728),
                          norm=colors.LogNorm(vmin=1e-5, vmax=1e-2), cmap=cmr.torch, transform=data_crs)
# Multiplication by years * months per year * particles released per cell per month

ax.spines['geo'].set_linewidth(1)
ax.set_ylabel('Latitude')
ax.set_xlabel('Longitude')
ax.set_aspect('equal')
# ax.set_title('Fraction of mass from cell beaching at Seychelles')

cax1 = f.add_axes([ax.get_position().x1+0.0105,ax.get_position().y0,0.015,ax.get_position().height])

cb1 = plt.colorbar(mass_flux, cax=cax1, pad=0.1)
cb1.set_label('Fraction of mass from cell beaching at the Aldabra Group', size=12)

land_10m = cfeature.NaturalEarthFeature('physical', 'land', '10m',
                                        edgecolor='white', linewidth=0.5,
                                        facecolor='black')
ax.add_feature(land_10m)

gl = ax.gridlines(crs=data_crs, draw_labels=True, linewidth=1, color='black', linestyle='-')
gl.xlocator = mticker.FixedLocator(np.arange(20, 140, 20))
gl.ylocator = mticker.FixedLocator(np.arange(-40, 40, 20))
gl.ylabels_right = False
gl.xlabels_top = False

plt.savefig(fh['fig'] + '_AldabraGrp.png', dpi=300, bbox_inches='tight')

###############################################################################

f, ax = plt.subplots(1, 1, figsize=(20, 12), constrained_layout=True,
                     subplot_kw={'projection': ccrs.PlateCarree()})
plt.subplots_adjust(bottom=0, top=1)

data_crs = ccrs.PlateCarree()

mass_flux = ax.pcolormesh(lon_bnd, lat_bnd, np.sum(grid[12:,:,:], axis=0)/(len(fh['data'])*1728),
                          norm=colors.LogNorm(vmin=1e-5, vmax=1e-2), cmap=cmr.torch, transform=data_crs)
# Multiplication by years * months per year * particles released per cell per month

ax.spines['geo'].set_linewidth(1)
ax.set_ylabel('Latitude')
ax.set_xlabel('Longitude')
ax.set_aspect('equal')
# ax.set_title('Fraction of mass from cell beaching at Seychelles')

cax1 = f.add_axes([ax.get_position().x1+0.0105,ax.get_position().y0,0.015,ax.get_position().height])

cb1 = plt.colorbar(mass_flux, cax=cax1, pad=0.1)
cb1.set_label('Fraction of mass from cell beaching at the Seychelles Plateau', size=12)

land_10m = cfeature.NaturalEarthFeature('physical', 'land', '10m',
                                        edgecolor='white', linewidth=0.5,
                                        facecolor='black')
ax.add_feature(land_10m)

gl = ax.gridlines(crs=data_crs, draw_labels=True, linewidth=1, color='black', linestyle='-')
gl.xlocator = mticker.FixedLocator(np.arange(20, 140, 20))
gl.ylocator = mticker.FixedLocator(np.arange(-40, 40, 20))
gl.ylabels_right = False
gl.xlabels_top = False

plt.savefig(fh['fig'] + '_SeychellesPlateau.png', dpi=300, bbox_inches='tight')


# -*- coding: utf-8 -*-
"""
Created on Thu Aug  4 21:39:55 2022

@author: Administrator
"""

import numpy as np

import pandas as pd

import xarray as xr

import glob

from wrf import getvar,latlon_coords

from netCDF4 import Dataset

from metpy.units import units
import metpy.constants as constants
import metpy.calc as mpcalc

'''
This script is used to calculate the three terms on the right side of the equation:
horizontal advection term,
vertical advection term,
non-adiabatic heating term
'''

####################################################################
#####     read data :wind-component ,longitude,latitude,time,level,pv,omega
####################################################################
# f_wind =  r"D:\PV\wrf-read\wrf-cal-pv\wind.nc"

f_wind =  r"/Users/xpji/wind.nc"
d_wind = xr.open_dataset(f_wind)
u = d_wind.u.data*units('m/s')                                      # units('m/s')
v = d_wind.v.data*units('m/s')                                       # units('m/s')

lat = d_wind.lat
lon = d_wind.lon
lev = d_wind.level
time = d_wind.time

f_Q  =   r"/Users/xpji/Q.nc"
d_Q  =  xr.open_dataset(f_Q)

f_pv =  r"/Users/xpji/pv.nc"
pv   = xr.open_dataset(f_pv).pv

f_omega = r"/Users/xpji/omega.nc"
omega  = xr.open_dataset(f_omega).omega
###################################################################################
######  calculate dx\dy\lon\lat
###################################################################################
def  cal_dxdy(file):
    ncfile = Dataset(file)
    P = getvar(ncfile, "pressure")
    lats, lons = latlon_coords(P)
    lon = lons[0]
    lon[lon<=0]=lon[lon<=0]+360
    lat = lats[:,0]
    dx, dy = mpcalc.lat_lon_grid_deltas(lon.data, lat.data)
    return lon,lat,dx,dy

###################################################################################
######    Get the size of latitude and longitude
###################################################################################

path  = r'/Users/xpji/out0-8/'
filelist = glob.glob(path+'wrf*')
filelist.sort()
lon,lat,dx,dy = cal_dxdy(filelist[-1])
nx = lon.shape[0]
ny = lat.shape[0]
####################################################################
#####     calculate  the horizontal and vertical advection of PV
####################################################################

pv_adv = np.full((time.shape[0],lev.shape[0],lat.shape[0],lon.shape[0]),np.nan)

for i in range(time.shape[0]):

    for j in range(lev.shape[0]):

        print(i,j)
        pv_adv[i,j,:,:] = mpcalc.advection(pv[i,j,:,:],u=u[i,j,:,:],v=v[i,j,:,:],dx=dx,dy=dy,x_dim=-1,y_dim=-2)

units.define('PVU = 1e-6 m^2 s^-1 K kg^-1 = pvu')
w = omega.data*units('Pa/s')
P = pv.data*units('PVU').to('m^2 s^-1 K kg^-1')
level = d_wind.level.data*units('hPa')
wdPdp = np.zeros(shape=P.shape,dtype="float32")*(units("m^2 s^-1 K kg^-1")/units("hPa")*units('Pa/s'))
lev_dif = np.gradient(level)
p_dif   = np.gradient(P,axis=1)
for  i in range(40):

    for j in range(27):

        print(i,j)

        wdPdp[i,j,:,:] = w[i,j,:,:]*p_dif[i,j,:,:]/lev_dif[j]
pvvdv = wdPdp.to_base_units()
pvvdv_m  = np.nanmean(pvvdv,axis=0)
pvvdv_a  =( pvvdv-pvvdv_m)
da_nc=xr.Dataset(
data_vars=dict(

    pvvdv=(['time','level','lat','lon'],-np.array(pvvdv.m)),
    pvvdv_ano=(['time','level','lat','lon'],-np.array(pvvdv_a.m)),
    ),
                coords = {
                    'time':time,
                    'level':level.m,
                    'lat':lat,
                    'lon':lon,

                    },
                attrs=dict(description="PV vertical advection(PVU = 1e-6 m^2 s^-1 K kg^-1)",
                            units= 'm^2 s^-2 K kg^-1',
                            equation='-<w*(dpv/dP)>-0907',
                            ),
                )
da_nc.to_netcdf(r'./data/pvvdv_.nc')
###########################################################################################
Q    = d_Q.Q_to.data*units('K/s')
####################################################################
#####     calculate gradient in 3 direction : x,y,p
####################################################################
# calculate  gradient
dv_dif = np.gradient(v,axis=1)
du_dif = np.gradient(u,axis=1)
dq_dif = np.gradient(Q,axis=1)

dv_dp = np.zeros(Q.shape,dtype='float32')*units('m/s')/units('hPa')
du_dp = np.zeros(Q.shape,dtype='float32')*units('m/s')/units('hPa')
dq_dp = np.zeros(Q.shape,dtype='float32')*units('K/s')/units('hPa')
ny=lat.shape[0]
fcc = mpcalc.coriolis_parameter(np.deg2rad(lat.data)).reshape((ny,1))

adv = np.zeros(Q.shape,dtype='float32')*units('K/s')/units('m')*units('m/s')/units('hPa')

dx = dx*units('m')
dy = dy*units('m')
q_dx = np.gradient(Q,axis=-1)
q_dy = np.gradient(Q,axis=-2)
for i in range(40):
    for j in range(27):
        print(i,j)
        dv_dp[i,j] = dv_dif[i,j]/lev_dif[j]
        du_dp[i,j] = du_dif[i,j]/lev_dif[j]
        dq_dp[i,j] = dq_dif[i,j]/lev_dif[j]

for i in range(40):
    for j in range(27):
        adv[i,j] = -1*dv_dp[i,j]*q_dx[i,j]/dx + du_dp[i,j]*q_dy[i,j]/dy

######### cal vertical components
vdv = np.zeros(Q.shape,dtype='float32')*units('K/s')/units('hPa')*units('m/s')/units('m')
v_dx = np.gradient(v, axis=-1)
u_dy = np.gradient(u, axis=-2)
vor_z = np.zeros(Q.shape,dtype='float32')*units('m/s')/units('m')

for i in range(40):
    for j in range(27):
        vor_z[i,j] = v_dx[i,j]/dx - u_dy[i,j]/dy

absvor_z = vor_z+fcc

for i in range(40):
    for j in range(27):
        vdv[i,j] = absvor_z[i,j]*dq_dp[i,j]

heat = (adv+vdv)*constants.g
dia = heat.to_base_units()

dia_m = np.nanmean(dia,axis=0)
dia_ano = dia-dia_m

####################################################################
#####   output as Netcdf
####################################################################
heat_nc=xr.Dataset(
    data_vars=dict(

        Q_tot=(['time','level','lat','lon'],-np.array(dia.m)),
        Q_ana=(['time','level','lat','lon'],-np.array(dia_ano.m))),

        coords = {
                        'time':tim,
                        'level':lev.m,
                        'lat':lat.data,
                        'lon':lon.data,

                        },
                    attrs=dict(description=" diabatic heating ",
                                units= 'm^2 s^-1 K kg^-1'),
                    )

heat_nc.to_netcdf(r'./data/diabatic_heat_20220907.nc')


# -*- coding: utf-8 -*-

#import keras, scikit learn and pandas
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from tensorflow.python import keras
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense, Flatten, Conv2D, Dropout
from sklearn.preprocessing import StandardScaler
from tensorflow.python.keras import optimizers
from tensorflow.python.keras import regularizers
from sklearn.ensemble import RandomForestRegressor
from sklearn import preprocessing as pr
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split

#import the data preProcessed in R as a pandas data frame
fd_tr=pd.read_csv("final_tr_data.csv")
fd_te=pd.read_csv("final_te_data.csv")

#use Pandas data frame manipulation, to make the data suitable for a machine learning algorithm
x_tr=fd_tr.iloc[:,:33]
y_tr=fd_tr.iloc[:,33]

x_te=fd_te

#Cobmine the test and training data so they can both be scaled and encoded
full_x=x_tr.append(x_te)

scaler = StandardScaler()

#Scale the coninous vairables for use in the Neural Net
scale_x=scaler.fit_transform(full_x.iloc[:3000,[0,2,3,6]])
scale_x_te=scaler.transform(full_x.iloc[3000:,[0,2,3,6]])

full_x.iloc[:3000,[0,2,3,6]]=scale_x
full_x.iloc[3000:,[0,2,3,6]]= scale_x_te

# One hot encode the factor variables
full_x=pd.get_dummies(full_x)
x_tr=full_x.iloc[0:3000,:].values
x_te=full_x.iloc[3000:,:].values

#The neural net to be strongly overfitting so we have used both dropout and regularization, the value
# we use for the drop out and regulurization are just based off trial an error

y_tr=y_tr.values

model = Sequential()
model.add(Dropout(0.35))
model.add(Dense(256, activation='relu', kernel_regularizer=regularizers.l2(0.01)))
model.add(Dropout(0.35))
model.add(Dense(128, activation='relu', kernel_regularizer=regularizers.l2(0.01)))
model.add(Dropout(0.35))
model.add(Dense(64, activation='relu', kernel_regularizer=regularizers.l2(0.01)))
model.add(Dropout(0.35))
model.add(Dense(1))
adam=optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
model.compile(optimizer=adam,
              loss='mean_squared_error')
model.fit(x_tr, y_tr,
          batch_size=24,
          epochs=50,
          validation_split = 0.2)

pred=model.predict(x_te)

np.savetxt("nn5.csv",pred,delimiter=",")

#We also implement a Random Forest for prediction, with gird search to find the optimal hyperparameters
parameters={"max_features":[5,10,25,30],"min_samples_split":[2,3,4,5]}
clf = RandomForestRegressor(n_estimators=500, random_state=0)
clf.cv = GridSearchCV(clf, parameters, cv=3)
clf.cv.fit(x_tr,y_tr)
pred1=clf.cv.predict(x_te)

np.savetxt("rfsk1.csv",pred1,delimiter=",")