#! /usr/bin/python
# -*- coding: utf-8 -*-
import os
import math
from collections import defaultdict
import networkx as nx
import matplotlib.pyplot as plt

os.chdir("D:/work_files/pycharm_codes/nlp_assignment")

def load_data():
    beijing_subway_geo_dict = {}  # example: {"草房":[15.26,233.56],...}
    with open("homework/part2/data/beijing_subway_geo.txt", "r", encoding="utf-8") as beijing_subway_geo_file:
        for line in beijing_subway_geo_file.readlines():
            line_strip_split = line.strip().split(":")
            subway_name = line_strip_split[0]
            subway_geo = line_strip_split[1].split(",")
            beijing_subway_geo_dict[subway_name] = (float(subway_geo[0]), float(subway_geo[1]))
    beijing_subway_belong_to_line = {}  # example: {"草房":["6号线"],...}
    beijing_subway_line = {}  # example: {"6号线":["金安桥",...]}
    beijing_subway_link = {}  # example: {"朱辛庄": ["育知路",..]}
    with open("homework/part2/data/beijing_subway_line.txt", "r", encoding="utf-8") as beijing_subway_line_file:
        for line in beijing_subway_line_file.readlines():
            line_strip_split = line.strip().split(":")
            subway_line = line_strip_split[1].split(",")
            beijing_subway_line[line_strip_split[0]] = subway_line
            for beijing_subway in subway_line:
                if beijing_subway_belong_to_line.get(beijing_subway) is None:
                    beijing_subway_belong_to_line[beijing_subway] = []
                    beijing_subway_belong_to_line[beijing_subway].append(line_strip_split[0])
                else:
                    beijing_subway_belong_to_line.get(beijing_subway).append(line_strip_split[0])
            if len(subway_line) >= 2:
                for index in range(len(subway_line) - 1):
                    if beijing_subway_link.get(subway_line[index]) is None:
                        beijing_subway_link[subway_line[index]] = []
                        beijing_subway_link[subway_line[index]].append(subway_line[index + 1])
                    else:
                        beijing_subway_link[subway_line[index]].append(subway_line[index + 1])
                    if beijing_subway_link.get(subway_line[index + 1]) is None:
                        beijing_subway_link[subway_line[index + 1]] = []
                        beijing_subway_link[subway_line[index + 1]].append(subway_line[index])
                    else:
                        beijing_subway_link[subway_line[index + 1]].append(subway_line[index])
    return beijing_subway_geo_dict, beijing_subway_line, beijing_subway_belong_to_line, beijing_subway_link

def geo_distance(origin, destination):
    """
    Calculate the Haversine distance.

    Parameters
    ----------
    origin : tuple of float
        (lat, long)
    destination : tuple of float
        (lat, long)
    Returns
    -------
    distance_in_km : float

    Examples
    --------
     origin = (48.1372, 11.5756)  # Munich
     destination = (52.5186, 13.4083)  # Berlin
     round(distance(origin, destination), 1) 504.2
    """
    lat1, lon1 = origin
    lat2, lon2 = destination
    radius = 6371  # km

    dlat = math.radians(lat2 - lat1)
    dlon = math.radians(lon2 - lon1)
    a = (math.sin(dlat / 2) * math.sin(dlat / 2) +
         math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) *
         math.sin(dlon / 2) * math.sin(dlon / 2))
    c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
    d = radius * c
    return d

def plt_beijing_subway_line():
    """
    绘制北京地铁路线图
    :return: 无返回值
    """
    beijing_subway_geo_dict, beijing_subway_line, beijing_subway_belong_to_line, beijing_subway_link_src = load_data()
    beijing_subway_link = defaultdict(list)
    beijing_subway_link.update(beijing_subway_link_src)
    beijing_subway_graph = nx.Graph(beijing_subway_link)
    nx.draw(beijing_subway_graph, beijing_subway_geo_dict, with_labels=False, node_size=10,font_size=6)
    plt.show()

def first_search(graph, start, rule="depth_first"):
    """
    用深度优先或者广度优先的搜索策略搜索整个图
    :param graph: dict,图结构，形如 {"北京":["太原","沈阳"],...}
    :param start: string,开始地名
    :param rule: string,规则,比如depth_first,breath_first
    :return: 遍历后的元素
    """
    visited = [start]
    seen = set()
    while visited:
        froninter = visited.pop()  # 删除列表中最后一个值,并返回最后一个值
        if froninter in seen: continue
        for successor in graph[froninter]:
            if successor in seen: continue
            print(successor)
            if rule == "depth_first":
                visited = visited + [successor]  # 我们每次扩展都扩展最新发现的点 -> depth first
            else:
                visited = [successor] + visited  # 我们每次扩展都先考虑已经发现的 老的点 -> breath first
            # 所以说，这个扩展顺序其实是决定了我们的深度优先还是广度优先
        seen.add(froninter)
    return seen

def search(start, destination, connection_graph, sort_candidate):
    """
    给定初始位置和结束位置，以及图结构和排序规则，利用排序规则获取最优分布路径。
    :param start: string,开始位置
    :param destination: string,结束位置
    :param connection_graph: dict,图结构,形如{"a":["b","c"]}
    :param sort_candidate: 排序函数
    :return: path, [], 最优路径
    """
    pathes = [[start]]

    visited = set()

    while pathes:
        path = pathes.pop(0)
        froninter = path[-1]

        if froninter in visited: continue

        successors = connection_graph[froninter]

        for node in successors:
            if node in path: continue
            new_path = path + [node]
            pathes.append(new_path)
            if node == destination: return new_path
        visited.add(froninter)
        pathes = sort_candidate(pathes)  # 我们可以加一个排序函数 对我们的搜索策略进行控制

def transfer_station_first(pathes):
    """
    经过站数最少
    :param pathes: [] 路径列表,[path1,path2]
    :return: function
    """
    return sorted(pathes, key=len)  # 升序

def transfer_as_much_possible(pathes):
    """
    经过站数最多
    :param pathes:  [] 路径列表,[path1,path2]
    :return: function
    """
    return sorted(pathes, key=len, reverse=True)

def get_path_distance(path):
    """
    获取路径距离
    :param path:[] 整个路径
    :return: 距离和
    """
    distance = 0.0
    for i in range(len(path) - 1):
        distance += geo_distance(path[i], path[i + 1])
    return distance

def shortest_path_first(pathes):
    """
    最短路径距离
    :param pathes:  [] 路径列表,[path1,path2]
    :return: function
    """
    if len(pathes) <= 1: return pathes
    return sorted(pathes, key=get_path_distance)

def main():
    plt_beijing_subway_line()

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# coding: utf-8

# In[83]:

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
from sklearn import linear_model
import seaborn as sns
from scipy.stats import ttest_ind

# In[84]:

covid_data = pd.read_csv("Copy of COVID_Data.csv")
mobility_data = pd.read_csv("US_Mobility_Data.csv")

# First step is loading the data into python We will use both covid tracking data and Google's mobility data. Now we have to clean the data to get only entries that we want.

# In[85]:

covid_usa = covid_data[covid_data['location']=='United States']
covid_usa = covid_usa[['date','total_cases','new_cases','total_deaths','new_deaths','reproduction_rate']]

mobility_data = mobility_data.fillna(0)
mobility_data = mobility_data[mobility_data['sub_region_1']==0]
mobility_data = mobility_data[['country_region_code','country_region','sub_region_1',
                                   'date','retail_and_recreation_percent_change_from_baseline',
                                   'parks_percent_change_from_baseline','transit_stations_percent_change_from_baseline',
                                   'workplaces_percent_change_from_baseline']]

# Here we have simplified the data into something that we can actually use. The COVID Data has been simplified to just contain values pertaining to the United States. We also took certain columns from both the COVID Data and Mobility Data to have only columns we wish to analyze.

# Now we must merge the two dataframes together on the common column of date.

# In[86]:

mobility_data = mobility_data.reset_index(drop=True)
covid_usa = covid_usa.reset_index(drop=True)
data_list = [covid_usa,mobility_data]
combined_data = covid_usa.append(mobility_data)
combined_data = combined_data.groupby(['date']).sum()
combined_data.fillna(0,inplace=True)
combined_data['Time']= np.arange(len(combined_data))
combined_data.head(300)

# Here is the combined dataset, on the column of date. Obviously all of the NaN values have been changed to zero, and the join was done as an outer. The next step would be to plot some scatters of our data to see how it is distributed. There was also a Time column added that counts the entries and is supposed to simulate date. The ultimate goal will be to try and see how well reproduction rate correlates with each of the mobility data, and then see if we can predict reproduction with the mobility.

# In[87]:

plt.plot(combined_data['Time'],combined_data['reproduction_rate'])
plt.xlabel('Time (in days)')
plt.ylabel('Reproduction Rate')
plt.title('Reproduction Rate Versus Time')
plt.show()

# In[88]:

plt.plot(combined_data['Time'],combined_data['workplaces_percent_change_from_baseline'])
plt.xlabel('Time (in days)')
plt.ylabel('Workplace Mobility')
plt.title('Workplace Mobility Versus Time')
plt.show()

# In[89]:

plt.plot(combined_data['Time'],combined_data['transit_stations_percent_change_from_baseline'])
plt.xlabel('Time (in days)')
plt.ylabel('Transit Mobility')
plt.title('Transit Mobility Versus Time')
plt.show()

# In[90]:

plt.plot(combined_data['Time'],combined_data['retail_and_recreation_percent_change_from_baseline'])
plt.xlabel('Time (in days)')
plt.ylabel('Retail & Recreation Mobility')
plt.title('Retail & Recreation Mobility Versus Time')
plt.show()

# In[91]:

plt.plot(combined_data['Time'],combined_data['parks_percent_change_from_baseline'])
plt.xlabel('Time (in days)')
plt.ylabel('Park Mobility')
plt.title('Park Mobility Versus Time')
plt.show()

# While none of the mobility graphs map exactly the same as the reproduction rate of the virus; it can be observed that they are somewhat similar. What we can do now is run some correlations on the data, to see which of the mobility columns correlate the best with reproduction rate.

# In[92]:

transit_corr = combined_data[['reproduction_rate','transit_stations_percent_change_from_baseline']].corr()
workplace_corr = combined_data[['reproduction_rate','workplaces_percent_change_from_baseline']].corr()
parks_corr = combined_data[['reproduction_rate','parks_percent_change_from_baseline']].corr()
retail_corr = combined_data[['reproduction_rate','retail_and_recreation_percent_change_from_baseline']].corr()

# In[93]:

sns.heatmap(transit_corr,annot=True)

# In[94]:

sns.heatmap(workplace_corr,annot=True)

# In[95]:

sns.heatmap(parks_corr,annot=True)

# In[96]:

sns.heatmap(retail_corr,annot=True)

# From the four correlation heatmaps above we can see that the highest correlation with reproduction rate occurs with transit mobility, or the movement of the population in the USA on average on public transportation. Now that we have this, it may be more effective for us to take the 7-Day average for each of these columns, then try to graph the two of them on the same plot. It also would help if we normalized the columns.

# In[97]:

combined_data['Reproduction 7-Day Avg'] = combined_data['reproduction_rate'].rolling(window=7).mean()
combined_data['Transit Mobility 7-Day Avg'] = combined_data['transit_stations_percent_change_from_baseline'].rolling(window=7).mean()

cols_to_norm = ['Reproduction 7-Day Avg','Transit Mobility 7-Day Avg']
combined_data[cols_to_norm] = combined_data[cols_to_norm].apply(lambda x: (x - x.min()) / (x.max() - x.min()))

# In[98]:

plt.plot('Time','Reproduction 7-Day Avg',data=combined_data,color='skyblue')
plt.plot('Time','Transit Mobility 7-Day Avg',data=combined_data,color='red')
plt.xlabel('Time (in days)')
plt.ylabel('Normalized Column Values')
plt.title('Comparison of 7-Day Average Reproduction Rate and Transit Mobility')
plt.legend()

# As you can see, the first step was to create two new columns: Reproduction Rate 7-Day Rolling Average and Transit Mobility 7-Day Rolling Average. Once the two columns were created using panda's pd.rolling function, setting the window to 7 for a 7-Day roll, and taking the mean, we now have to normalize the two columns to bring them to the same scale. mobility's range was much larger than the reproduction rate. Looking at the plot of the two lines it can be said that reproduction rate almost acts opposite of mobility in transits. The next step in our process is to create a linear regression model to hopefully be able to make a prediction of where the reproduction rate will go. After that we could try and average out all of the mobilities and see if that is a better predictor. But to start, we can make a model to predict reproductive rate using: transit mobility, new cases,total cases, and new deaths.

# In[99]:

combined_data = combined_data.fillna(0)
predictors_1 = combined_data[['total_cases','new_cases','new_deaths','Transit Mobility 7-Day Avg']]
response_1 = combined_data['Reproduction 7-Day Avg']

regression_1 = linear_model.LinearRegression()
model_1 = regression_1.fit(predictors_1,response_1)

res1 = ttest_ind(predictors_1, response_1).pvalue

table_1 = {'Variable':['total_cases','new_cases','new_deaths','Transit Mobility 7-Day Avg'],
           'Coefficients':[model_1.coef_],
           'P-Values':[res1]

          }

print(
    'Variable:     ','total_cases    ','new_cases    ','new_deaths     ','Transit Mobility 7-Day Avg '
    '\nCoeficients: ',model_1.coef_,
    '\nP=Values:    ',res1,
    '\nIntercept: ',model_1.intercept_,
    '\nR-Squared: ',model_1.score(predictors_1,response_1))

# Off this first model we can see what our Beta values will be and the intercept. But it is clear that the R-Squared value is not too impressive, clocking in at only about 19.64%. This means that only 19.64% of the variation in reproduction rate is explained by these predictors.

# There are three steps here to try and make our model better. First, we will make a model consisting solely of our transit mobility 7-Day Average predictor. Next, we will get the seven day averages of each of the mobility data columns and make models of that, and average them together and use that as a predictor. lastly, we will take the 7-Day average of new cases and new deaths.

# In[100]:

predictors_2 = combined_data[['Transit Mobility 7-Day Avg']]
response_2 = combined_data['Reproduction 7-Day Avg']

regression_2 = linear_model.LinearRegression()
model_2 = regression_2.fit(predictors_2,response_2)

res2 = ttest_ind(predictors_2, response_2).pvalue

table_1 = {'Variable':['total_cases','new_cases','new_deaths','Transit Mobility 7-Day Avg'],
           'Coefficients':[model_2.coef_],
           'P-Values':[res2]

          }

print(
    'Variable:     ','total_cases    ','new_cases    ','new_deaths     ','Transit Mobility 7-Day Avg '
    '\nCoeficients: ',model_2.coef_,
    '\nP=Values:    ',res2,
    '\nIntercept: ',model_2.intercept_,
    '\nR-Squared: ',model_2.score(predictors_2,response_2))

# Above is a model that just consists of the sole predictor of 'Transit Mobility 7-Day Avg' to predict the Reproduction Rate 7-Day Average. It is apparent that getting rid of the other predictors did not have a detrimental affect on the R-Squared value, now coming in slightly lower than before at 0.1753. The next step would be to take the 7-Day averages of all other mobility columns, and using each of them as separate predictors for the reproduction rate.

# In[101]:

combined_data['Park Mobility 7-Day Avg'] = combined_data['parks_percent_change_from_baseline'].rolling(window=7).mean()
combined_data['Workplace Mobility 7-Day Avg'] = combined_data['workplaces_percent_change_from_baseline'].rolling(window=7).mean()
combined_data['Retail Mobility 7-Day Avg'] = combined_data['retail_and_recreation_percent_change_from_baseline'].rolling(window=7).mean()
combined_data = combined_data.fillna(0)

# In[102]:

cols_to_norm_2 = ['Park Mobility 7-Day Avg','Workplace Mobility 7-Day Avg','Retail Mobility 7-Day Avg']
combined_data[cols_to_norm_2]=combined_data[cols_to_norm_2].apply(lambda x: (x - x.min()) / (x.max() - x.min()))

combined_data['Mean Mobility'] = combined_data[['Park Mobility 7-Day Avg','Workplace Mobility 7-Day Avg','Retail Mobility 7-Day Avg','Transit Mobility 7-Day Avg']].mean(1)

# Above is the pre-processing for the mobility data so we can get each of the mobility's 7-Day averages, and also create another new column taking the average of all four normalized mobilities. Let's now create a plot with all four of the mobilities, and the average to see what that looks like.

# In[103]:

plt.plot('Time','Park Mobility 7-Day Avg',data=combined_data,color='green')
plt.plot('Time','Retail Mobility 7-Day Avg',data=combined_data,color='blue')
plt.plot('Time','Workplace Mobility 7-Day Avg',data=combined_data,color='red')
plt.plot('Time','Transit Mobility 7-Day Avg',data=combined_data,color='skyblue')
plt.plot('Time','Mean Mobility',data=combined_data,color='black',linewidth=3)
plt.xlabel('Time (in days)')
plt.title('Plotted 7-Day Average Mobilities')
plt.legend(bbox_to_anchor=(0,0),loc="upper right")

# Now that we have all the desired columns, and have plotted all of the mobilities, we can now create another predictive model using all of the mobilities.

# In[65]:

predictors_3 = combined_data[['Park Mobility 7-Day Avg','Retail Mobility 7-Day Avg','Transit Mobility 7-Day Avg','Workplace Mobility 7-Day Avg']]
response_3 = combined_data['Reproduction 7-Day Avg']

regression_3 = linear_model.LinearRegression()
model_3 = regression_3.fit(predictors_3,response_3)

res3 = ttest_ind(predictors_3, response_3).pvalue

table_1 = {'Variable':['total_cases','new_cases','new_deaths','Transit Mobility 7-Day Avg'],
           'Coefficients':[model_3.coef_],
           'P-Values':[res3]

          }

print(
    'Variable:     ','total_cases    ','new_cases    ','new_deaths     ','Transit Mobility 7-Day Avg '
    '\nCoeficients: ',model_3.coef_,
    '\nP=Values:    ',res3,
    '\nIntercept: ',model_3.intercept_,
    '\nR-Squared: ',model_3.score(predictors_3,response_3))

# Using all four of the mobilities in our model actually gives us the best R-Squared value thus far, clocking in at 23.18% roughly. The same process was followed as before to make this linear model. Give the predictors their own list from the data, giving the response its assigned column, and then using the LinearRegression() function. The next step would be to try and see how well the column averaging all of the mobilities together fairs in terms of creating an affecting model.

# In[67]:

predictors_4 = combined_data[['Mean Mobility']]
response_4 = combined_data['Reproduction 7-Day Avg']

regression_4 = linear_model.LinearRegression()
model_4 = regression_4.fit(predictors_4,response_4)

res4 = ttest_ind(predictors_4, response_4).pvalue

table_1 = {'Variable':['total_cases','new_cases','new_deaths','Transit Mobility 7-Day Avg'],
           'Coefficients':[model_4.coef_],
           'P-Values':[res4]

          }

print(
    'Variable:     ','total_cases    ','new_cases    ','new_deaths     ','Transit Mobility 7-Day Avg '
    '\nCoeficients: ',model_4.coef_,
    '\nP=Values:    ',res4,
    '\nIntercept: ',model_4.intercept_,
    '\nR-Squared: ',model_4.score(predictors_4,response_4))

#
#
#

# In[63]:

x_data = combined_data['Time'].values.reshape(-1,1)
y_data = combined_data['Reproduction 7-Day Avg'].values.reshape(-1,1)

linear_regressor = linear_model.LinearRegression()

linear_regressor.fit(x_data,y_data)
Y_Pred = linear_regressor.predict(x_data)

plt.scatter(x_data,y_data)
plt.plot(x_data,Y_Pred,color='red')
plt.show()

# Above is a linear regression thatpython best sees fit to the Reproduction Data. It is clear to see that even the best fit line concerning no other variables still has a great deal of variance around the line. This is due to the spike which could be attributed to the viral spread, then the sharp dip attributed to social distancing guidlelines.

# In[ ]:


"""merchantの基本的な特徴量
"""
import os
import gc
import datetime
import numpy as np
import pandas as pd

import CONST
from features import FeatureBase

from utils import print_null
from utils import impute_na
from utils import clipping_outliers

class _701_Merchant(FeatureBase):
    fin = [os.path.join(CONST.INDIR, "merchants.feather"),
           os.path.join(CONST.INDIR, "historical_transactions.feather"),
           os.path.join(CONST.INDIR, "new_merchant_transactions.feather"),
           ]
    pref = "_701_merchant_"

    def create_feature_impl(self, df_list, random_state):
        base = pd.concat([
            df_list[1][['card_id', 'merchant_id']],
            df_list[2][['card_id', 'merchant_id']]
        ], axis=0)
        df = df_list[0]
        del df_list
        gc.collect()

        df = df.replace([np.inf, -np.inf], np.nan)
        # Average sales null
        null_cols = ['avg_purchases_lag3',
                     'avg_sales_lag3',
                     'avg_purchases_lag6',
                     'avg_sales_lag6',
                     'avg_purchases_lag12',
                     'avg_sales_lag12']

        for col in null_cols:
            df[col] = df[col].fillna(df[col].mean())

        # Category 2
        df['category_2'] = impute_na(df, df, 'category_2')

        # Sales cut
        sales_cut = df['most_recent_sales_range'].value_counts().sort_values(ascending=False).values
        sales_cut = sales_cut / np.sum(sales_cut)
        for i in range(1, len(sales_cut)):
            sales_cut[i] = sales_cut[i] + sales_cut[i - 1]

        # Purchases cut
        purchases_cut = df['most_recent_purchases_range'].value_counts().sort_values(ascending=False).values
        purchases_cut = purchases_cut / np.sum(purchases_cut)
        for i in range(1, len(purchases_cut)):
            purchases_cut[i] = purchases_cut[i] + purchases_cut[i - 1]

        # Discretize data
        discretize_cols = ['avg_purchases_lag3', 'avg_sales_lag3', 'avg_purchases_lag6', 'avg_sales_lag6',
                           'avg_purchases_lag12', 'avg_sales_lag12']

        for col in discretize_cols:
            categories = pd.qcut(df[col].values, sales_cut, duplicates='raise').categories.format()
            df[col], intervals = pd.qcut(df[col], 5, labels=['A', 'B', 'C', 'D', 'E'], retbins=True, duplicates='raise')
            print('Discretize for %s:' % col)
            print(categories)

        map_cols = discretize_cols + ['most_recent_purchases_range', 'most_recent_sales_range']
        for col in map_cols:
            df[col] = df[col].map({'A': 5, 'B': 4, 'C': 3, 'D': 2, 'E': 1})

        # https://www.kaggle.com/raddar/towards-de-anonymizing-the-data-some-insightsの変換
        df['trans_numerical_1'] = np.round(df['numerical_1'] / 0.009914905 + 5.79639, 0)
        df['trans_numerical_2'] = np.round(df['numerical_2'] / 0.009914905 + 5.79639, 0)

        # # numerical_cols = ['numerical_1', 'numerical_2']
        # # for col in numerical_cols:
        # #     df = clipping_outliers(df, df, col)
        #
        # for col in numerical_cols:
        #     b = df[col].unique()
        #     df[col] = df[col].apply(lambda x: 0 if x == b[0] else (1 if x in b[1:4] else 2))

        df = df.drop(columns=['avg_purchases_lag3', 'avg_sales_lag3', 'avg_purchases_lag6', 'avg_sales_lag6'])
        base = base.merge(df, on='merchant_id', how='left')
        base.drop(columns=['merchant_id'], inplace=True)

        aggs = {}
        aggs['merchant_group_id'] = ['nunique']
        aggs['merchant_category_id'] = ['nunique']
        aggs['subsector_id'] = ['nunique']
        aggs['city_id'] = ['nunique']
        aggs['state_id'] = ['nunique']
        aggs['numerical_1'] = ['sum', 'mean', 'max', 'min', 'var', 'skew', 'nunique']
        aggs['numerical_2'] = ['sum', 'mean', 'max', 'min', 'var', 'skew', 'nunique']
        aggs['category_1'] = ['sum', 'mean']
        aggs['category_4'] = ['sum', 'mean']
        aggs['most_recent_sales_range'] = ['sum', 'mean', 'max', 'min', 'var', 'skew']
        aggs['most_recent_sales_range'] = ['sum', 'mean', 'max', 'min', 'var', 'skew']
        aggs['active_months_lag3'] = ['sum', 'mean', 'max', 'min', 'var', 'skew']
        aggs['active_months_lag6'] = ['sum', 'mean', 'max', 'min', 'var', 'skew']
        aggs['active_months_lag12'] = ['sum', 'mean', 'max', 'min', 'var', 'skew']
        aggs['avg_sales_lag12'] = ['sum', 'mean', 'max', 'min', 'var', 'skew']
        aggs['avg_purchases_lag12'] = ['sum', 'mean', 'max', 'min', 'var', 'skew']

        feat = base.groupby(['card_id']).agg(aggs)
        feat.columns = ['-'.join(col).strip() for col in feat.columns.values]

        return feat.reset_index()

if __name__ == '__main__':
    trn, tst = _701_Merchant().create_feature()


import numpy as np
import torch

from PIL import Image
from abc import ABC, abstractmethod
from diffusers import (
    AutoencoderKL,
    DiffusionPipeline,
)
from tqdm.auto import tqdm
from transformers import CLIPTextModel, CLIPTokenizer
from typing import Any, List, Optional, Tuple, Union

from stablefused.typing import (
    PromptType,
    OutputType,
    Scheduler,
    SchedulerType,
    UNetType,
)
from stablefused.utils import (
    denormalize,
    load_model_from_cache,
    normalize,
    numpy_to_pil,
    numpy_to_pt,
    pil_to_numpy,
    pt_to_numpy,
    resolve_scheduler,
    save_model_to_cache,
)

class BaseDiffusion(ABC):
    def __init__(
        self,
        model_id: str = None,
        tokenizer: CLIPTokenizer = None,
        text_encoder: CLIPTextModel = None,
        vae: AutoencoderKL = None,
        unet: UNetType = None,
        scheduler: SchedulerType = None,
        torch_dtype: torch.dtype = torch.float32,
        device="cuda",
        use_cache=True,
        *args,
        **kwargs,
    ) -> None:
        self.device: str = device
        self.torch_dtype: torch.dtype = torch_dtype
        self.model_id: str = model_id

        self.tokenizer: CLIPTokenizer
        self.text_encoder: CLIPTextModel
        self.vae: AutoencoderKL
        self.unet: UNetType
        self.scheduler: SchedulerType
        self.vae_scale_factor: int

        if model_id is None:
            if (
                tokenizer is None
                or text_encoder is None
                or vae is None
                or unet is None
                or scheduler is None
            ):
                raise ValueError(
                    "Either (`model_id`) or (`tokenizer`, `text_encoder`, `vae`, `unet`, `scheduler`) must be provided."
                )

            self.tokenizer = tokenizer
            self.text_encoder = text_encoder
            self.vae = vae
            self.unet = unet
            self.scheduler = scheduler
        else:
            model = DiffusionPipeline.from_pretrained(
                model_id, torch_dtype=torch_dtype, *args, **kwargs
            )
            self.tokenizer = model.tokenizer
            self.text_encoder = model.text_encoder
            self.vae = model.vae
            self.unet = model.unet
            self.scheduler = model.scheduler

        self.to(self.device)
        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)

        if use_cache and model_id is not None:
            model = load_model_from_cache(model_id, None)
            if model is None:
                save_model_to_cache(self)
            else:
                print("here")
                self.share_components_with(model)

    def to(self, device: str) -> "BaseDiffusion":
        """
        Move model to specified compute device.

        Parameters
        ----------
        device: str
            The device to move the model to. Must be one of `cuda` or `cpu`.
        """
        self.device = device
        self.text_encoder = self.text_encoder.to(device)
        self.vae = self.vae.to(device)
        self.unet = self.unet.to(device)
        return self

    def share_components_with(self, model: "BaseDiffusion") -> None:
        """
        Share components with another model. This allows for sharing of the
        different internal components of the model, such as the text encoder,
        VAE, and UNet. This is useful for reducing memory usage when using
        multiple diffusion pipelines with the same checkpoint at the same time.

        Parameters
        ----------
        model: BaseDiffusion
            The model to share components with.
        """
        self.device = model.device
        self.torch_dtype = model.torch_dtype
        self.tokenizer = model.tokenizer
        self.text_encoder = model.text_encoder
        self.vae = model.vae
        self.unet = model.unet
        self.scheduler = model.scheduler
        self.vae_scale_factor = model.vae_scale_factor

    def set_scheduler(self, scheduler: Scheduler) -> None:
        """
        Set the scheduler for the diffusion pipeline.

        Parameters
        ----------
        scheduler: SchedulerType
            The scheduler to use for the diffusion pipeline.
        """
        self.scheduler = resolve_scheduler(scheduler, self.scheduler.config)

    def enable_attention_slicing(self, slice_size: Optional[int] = -1) -> None:
        """
        Enable attention slicing. By default, the attention head is sliced in half.
        This is a good tradeoff between memory and performance.

        Parameters
        ----------
        slice_size: int
            The size of the attention slice. If -1, the attention head is sliced in
            half. If None, attention slicing is disabled.
        """
        if slice_size == -1:
            slice_size = self.unet.config.attention_head_dim // 2
        self.unet.set_attention_slice(slice_size)

    def disable_attention_slicing(self) -> None:
        """Disable attention slicing."""
        self.unet.set_attention_slice(None)

    def enable_slicing(self) -> None:
        """
        Allow tensor slicing for vae decode step. This will cause the vae to split
        the input tensor to compute decoding in multiple steps. This will save
        memory and allow for larger batch sizes, but will affect performance slightly.
        """
        self.vae.enable_slicing()

    def disable_slicing(self) -> None:
        """Disable tensor slicing for vae decode step."""
        self.vae.disable_slicing()

    def enable_tiling(self) -> None:
        """
        Allow tensor tiling for vae. This will cause the vae to split the input tensor
        into tiles to compute encoding/decoding in several steps. This will save a large
        amount of memory and allow processing larger images, but will affect performance.
        """
        self.vae.enable_tiling()

    def disable_tiling(self) -> None:
        """Disable tensor tiling for vae."""
        self.vae.disable_tiling()

    @staticmethod
    def validate_input(
        prompt: PromptType = None,
        negative_prompt: PromptType = None,
        image_height: int = None,
        image_width: int = None,
        start_step: int = None,
        num_inference_steps: int = None,
        strength: float = None,
    ) -> None:
        """
        Validate input parameters.
        TODO: This needs to be removed and improved. More checks need to be added.

        Parameters
        ----------
        prompt: PromptType
            The prompt(s) to condition on.
        negative_prompt: PromptType
            The negative prompt(s) to condition on.
        image_height: int
            The height of the image to generate.
        image_width: int
            The width of the image to generate.
        start_step: int
            The step to start inference from.
        num_inference_steps: int
            The number of inference steps to perform.
        strength: float
            The strength of the noise mixing when performing LatentWalkDiffusion.
        """
        if image_height is not None and image_width is not None:
            if image_height % 8 != 0 or image_width % 8 != 0:
                raise ValueError(
                    "`image_height` and `image_width` must a multiple of 8"
                )
        if negative_prompt is not None:
            if type(prompt) is not type(negative_prompt):
                raise TypeError(
                    "Type of `prompt` and `negative_prompt` must be the same"
                )
            if isinstance(prompt, list) and len(prompt) != len(negative_prompt):
                raise ValueError(
                    "Length of `prompt` list and `negative_prompt` list should match"
                )
        if start_step is not None:
            if num_inference_steps is None:
                raise ValueError(
                    "`num_inference_steps` must be provided if `start_step` is provided"
                )
            if start_step < 0 or start_step >= num_inference_steps:
                raise ValueError(
                    "`start_step` must be in the range [0, `num_inference_steps` - 1]"
                )
        if strength is not None:
            if strength < 0 or strength > 1:
                raise ValueError("`strength` must be in the range [0.0, 1.0]")

    @abstractmethod
    def embedding_to_latent(self, *args: Any, **kwargs: Any) -> Any:
        """
        Abstract method for converting an embedding to a latent vector. This method
        must be implemented by all subclasses.
        """
        pass

    @abstractmethod
    def __call__(self, *args: Any, **kwargs: Any) -> Any:
        """
        Abstract method for performing inference. This method must be implemented
        by all subclasses.
        """
        pass

    def random_tensor(self, shape: Union[List[int], Tuple[int]]) -> torch.FloatTensor:
        """
        Generate a random tensor of the specified shape.

        Parameters
        ----------
        shape: List[int] or Tuple[int]
            The shape of the random tensor to generate.

        Returns
        -------
        torch.FloatTensor
            A random tensor of the specified shape on the same device and dtype
            as model.
        """
        rand_tensor = torch.randn(shape, device=self.device, dtype=self.torch_dtype)
        return rand_tensor

    def prompt_to_embedding(
        self,
        prompt: PromptType,
        negative_prompt: Optional[PromptType] = None,
    ) -> torch.FloatTensor:
        """
        Convert a prompt or a list of prompts into a text embedding.

        Parameters
        ----------
        prompt: PromptType
            The prompt or a list of prompts to convert into an embedding. Used
            for conditioning.
        negative_prompt: Optional[PromptType]
            A negative prompt or a list of negative prompts, by default None.
            Use for unconditioning. If not provided, an empty string ('') will
            be used to generate the unconditional embeddings.

        Returns
        -------
        torch.FloatTensor
            A text embedding generated from the given prompt(s) and, if provided,
            the negative prompt(s).
        """

        if negative_prompt is not None and type(negative_prompt) is not type(prompt):
            raise TypeError(
                f"`negative_prompt` must have the same type as `prompt` ({type(prompt)}), but found {type(negative_prompt)}"
            )

        if isinstance(prompt, str):
            batch_size = 1
            prompt = [prompt]
            if negative_prompt is not None:
                negative_prompt = [negative_prompt]
        elif isinstance(prompt, list):
            batch_size = len(prompt)
        else:
            raise TypeError("`prompt` must be a string or a list of strings")

        # Tokenize the prompt(s)
        text_input = self.tokenizer(
            prompt,
            padding="max_length",
            max_length=self.tokenizer.model_max_length,
            truncation=True,
            return_tensors="pt",
        )

        # Enable use of attention_mask if the text_encoder supports it
        if (
            hasattr(self.text_encoder.config, "use_attention_mask")
            and self.text_encoder.config.use_attention_mask
        ):
            attention_mask = text_input.attention_mask.to(self.device)
        else:
            attention_mask = None

        # Generate text embedding
        text_embedding = self.text_encoder(
            text_input.input_ids.to(self.device), attention_mask=attention_mask
        )[0]

        # Unconditioning input is an empty string if negative_prompt is not provided
        if negative_prompt is None:
            unconditioning_input = [""] * batch_size
        else:
            unconditioning_input = negative_prompt

        # Tokenize the unconditioning input
        unconditioning_input = self.tokenizer(
            unconditioning_input,
            padding="max_length",
            max_length=self.tokenizer.model_max_length,
            truncation=True,
            return_tensors="pt",
        )

        # Generate unconditional embedding
        unconditional_embedding = self.text_encoder(
            unconditioning_input.input_ids.to(self.device),
            attention_mask=attention_mask,
        )[0]

        # Concatenate unconditional and conditional embeddings
        embedding = torch.cat([unconditional_embedding, text_embedding])
        return embedding

    def classifier_free_guidance(
        self,
        noise_prediction: torch.FloatTensor,
        guidance_scale: float,
        guidance_rescale: float,
    ) -> torch.FloatTensor:
        """
        Apply classifier-free guidance to noise prediction.

        Parameters
        ----------
        noise_prediction: torch.FloatTensor
            The noise prediction tensor to which guidance will be applied.
        guidance_scale: float
            The scale factor for applying guidance to the noise prediction.
        guidance_rescale: float
            The rescale factor for adjusting the noise prediction based on
            guidance. Based on findings in Section 3.4  of [Common Diffusion
            Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf).

        Returns
        -------
        torch.FloatTensor
            The noise prediction tensor after applying classifier-free guidance.
        """

        # Perform guidance
        noise_unconditional, noise_prompt = noise_prediction.chunk(2)
        noise_prediction = noise_unconditional + guidance_scale * (
            noise_prompt - noise_unconditional
        )

        # Skip computing std if guidance_rescale is 0
        if guidance_rescale > 0:
            std_prompt = noise_prompt.std(
                dim=list(range(1, noise_prompt.ndim)), keepdim=True
            )
            std_prediction = noise_prediction.std(
                dim=list(range(1, noise_prediction.ndim)), keepdim=True
            )
            noise_prediction_rescaled = noise_prediction * (std_prompt / std_prediction)
            noise_prediction = (
                noise_prediction * (1 - guidance_rescale)
                + noise_prediction_rescaled * guidance_rescale
            )

        return noise_prediction

    def latent_to_image(
        self, latent: torch.FloatTensor, output_type: str
    ) -> OutputType:
        """
        Convert a latent tensor to an image in the specified output format.

        Parameters
        ----------
        latent: torch.FloatTensor
            The latent tensor to convert into an image.
        output_type: str
            The desired output format for the image. Should be one of [`pt`, `np`, `pil`].

        Returns
        -------
        OutputType
            An image in the specified output format.
        """
        if output_type not in ["pt", "np", "pil"]:
            raise ValueError("`output_type` must be one of [`pt`, `np`, `pil`]")

        image = self.vae.decode(
            latent / self.vae.config.scaling_factor, return_dict=False
        )[0]
        image = denormalize(image)

        if output_type == "pt":
            return image

        image = pt_to_numpy(image)

        if output_type == "np":
            return image

        image = numpy_to_pil(image)
        return image

    def image_to_latent(
        self,
        image: Union[Image.Image, List[Image.Image], np.ndarray, torch.Tensor],
    ) -> torch.FloatTensor:
        """
        Convert an image or a list of images into a latent tensor.

        Parameters
        ----------
        image: Union[Image.Image, List[Image.Image], np.ndarray, torch.Tensor]
            The input image(s) to convert into a latent tensor. Supported types are
            `PIL.Image.Image`, `np.ndarray`, and `torch.Tensor`.

        Returns
        -------
        torch.FloatTensor
            A latent tensor representing the input image(s).
        """
        if (
            not isinstance(image, Image.Image)
            and not isinstance(image, list)
            and not isinstance(image, np.ndarray)
            and not isinstance(image, torch.Tensor)
        ):
            raise TypeError(
                "`image` type must be one of (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`). Other types are not supported yet"
            )
        if isinstance(image, Image.Image):
            image: List[Image.Image] = [image]

        if isinstance(image[0], Image.Image):
            image: np.ndarray = pil_to_numpy(image)

        if isinstance(image[0], np.ndarray):
            image: torch.FloatTensor = numpy_to_pt(image)

        image = image.to(self.device)
        latent = (
            self.vae.encode(image).latent_dist.sample() * self.vae.config.scaling_factor
        )

        return latent

    def resolve_output(
        self,
        latent: torch.FloatTensor,
        output_type: str,
        return_latent_history: bool,
    ) -> Union[OutputType, List[OutputType]]:
        """
        Resolve the output from the latent based on the provided output options.

        Parameters
        ----------
        latent: torch.FloatTensor
            The latent tensor representing the content to be resolved.
        output_type: str
            The desired output format. Should be one of [`latent`, `pt`, `np`, `pil`].
        return_latent_history: bool
            If True, it means that the input latent tensor contains a tensor of latent
            tensor for each inference step. This requires decoding each latent tensor
            and returning a list of images. If False, decoding occurs directly.

        Returns
        -------
        Union[OutputType, List[OutputType]]
            The resolved output based on the provided latent vector and options.
        """
        if output_type not in ["latent", "pt", "np", "pil"]:
            raise ValueError(
                "`output_type` must be one of [`latent`, `pt`, `np`, `pil`]"
            )

        if output_type == "latent":
            return latent

        if return_latent_history:
            # Transpose latent tensor from [num_steps, batch_size, *latent_dim] to
            # [batch_size, num_steps, *latent_dim].
            # This is done so that the history of latent vectors for each prompt
            # is returned as a row instead of a column. It is what the user would
            # intuitively expect.
            latent = torch.transpose(latent, 0, 1)
            image = [
                self.latent_to_image(l, output_type)
                for _, l in list(enumerate(tqdm(latent)))
            ]

            if output_type == "pt":
                image = torch.stack(image)
            elif output_type == "np":
                image = np.stack(image)
            else:
                # output type is "pil" so we can just return as a python list
                pass
        else:
            image = self.latent_to_image(latent, output_type)

        return image


import pandas as pd
from sklearn.metrics import accuracy_score, f1_score
import pickle
import json

def main():
    # open saved model
    with open("model.pickle","rb") as f:
        model = pickle.load(f)

    # open test data
    data = pd.read_csv("data_test.csv")

    # do some processing
    X,y = process(data)

    # get metrics from predictions
    metrics = test(X,y,model)

    # save to JSON file
    save_result(metrics)

def process(data):
    # boring preprocessing
    X,y = data.iloc[:,:-1], data.iloc[:,-1]

    return X,y

def test(X,y,model):
    # using model to predict
    y_predicted = model.predict(X)

    # gathering metrics
    metrics = {
        'accuracy': round(accuracy_score(y,y_predicted),4),
        'f1_score': round(f1_score(y,y_predicted),4)
    }

    return metrics

def save_result(metrics):
    # save results to JSON file
    with open("metrics.json",'w') as f:
        json.dump(metrics,f)

if __name__ == "__main__":
    main()


import pymysql.cursors
from PyQt5 import uic,QtWidgets
from time import sleep as delay

SqlConexao = pymysql.connect(
            host = 'localhost',
            user = 'root',
            password= '',
            database = 'estacionamento',
            cursorclass= pymysql.cursors.DictCursor)

# SELECT (((saida - entrada)/100)*6) FROM carros WHERE placa ='PNL1234'; PAGAMENTO VAI SER IMPLEMENTADO

def Consultar():
    ConsultaSQL = SqlConexao.cursor()
    Consulta = ('SELECT placa FROM carros WHERE placa = %s')
    placa = input('Informe a placa do veiculo: ')
    print("#" * 10)
    print('Consultando...')
    print("#" * 10)
    delay(1.3)
    ConsultaSQL.execute(Consulta,({placa}))
    ApresentaConsulta = ConsultaSQL.fetchall()
    for Placa in ApresentaConsulta:
        print("#" * 10)
        print(f'{Placa} - PLACA CADASTRADA')
        print("#" * 10)
        op = int(input('1 - Nova Consulta | 2 - Retornar ao Menu : '))
        if op == 1:
            Consultar()
        if op == 2:
            Return()
    delay(0.4)
    print("#" * 10)
    print(f'{placa} - NAO CADASTRADA')
    print("#" * 10)
    op = int(input('1 - Nova Consulta | 2 - Retornar ao Menu : '))
    if op == 1:
        Consultar()
    if op == 2:
        Return()
    ConsultaSQL.close()
    SqlConexao.close()

def Apagar():
    ApagaSql = SqlConexao.cursor()
    Apaga = ('DELETE FROM carros WHERE placa = %s')
    placa = input("Informe a placa do veiculo:")
    ApagaSql.execute(Apaga,({placa}))
    confirma = int(input(f'Os a serem removidos são {placa} . Deseja confirmar? (1 - SIM / 2 - NÃO) '))
    if confirma == 1:
        SqlConexao.commit()
    else:
        Apagar()
    ApagaSql.close()
    SqlConexao.close()

def Modificar():
    ModificaSql = SqlConexao.cursor()
    Modifica = ("UPDATE carros SET placa = %s , modelo = %s WHERE cod =%s")
    placa = input("Informe a placa do veiculo: ")
    modelo = input("Informe o modelo do veiculo: ")
    cod = int(input("Informe o cod de registro : "))
    ModificaSql.execute(Modifica, ({placa}, {modelo},{cod}))
    confirma = int(input(f'Os dados serão atualizados {placa} e {modelo}. Deseja confirmar? (1 - SIM / 2 - NÃO) '))
    if confirma == 1:
        SqlConexao.commit()
    else:
        Modificar()
    ModificaSql.close()
    SqlConexao.close()

def Todos():
    ConsultaSQL = SqlConexao.cursor()
    Listando = ('SELECT * FROM carros')
    print('Consultando...')
    delay(1.3)
    ConsultaSQL.execute(Listando)
    ApresentaConsulta = ConsultaSQL.fetchall()
    for Placa in ApresentaConsulta:
        print(Placa)
    ConsultaSQL.close()
    SqlConexao.close()

def RegistroOperador():
    TelaCadastro.show()
    placa = TelaCadastro.Placa_Line.text()
    HoraEntrada = TelaCadastro.Entrada_Line.text()
    OperadorSql = SqlConexao.cursor()
    OperadorEntrada = ("INSERT INTO carros(placa,entrada)"
                "VALUES (%s,%s)")
    info = (str(placa),str(HoraEntrada))
    OperadorSql.execute(OperadorEntrada,info)
#    confirma = int(input(f'Os dados cadastrados são {placa} e {HoraEntrada}. Deseja confirmar? (1 - SIM / 2 - NÃO)'))
#    if confirma == 1:
    confirmar = SqlConexao.commit()
    PRINCIPAL.TelaCadastro.Confirma_btt.clicked.connect(confirmar)
    print(f'Os dados cadastrados são {placa} e {HoraEntrada}')
#       print('CADASTRANDO...')
#        delay(0.3)
#       Return()
#    else:
#        RegistroOperador()
    OperadorSql.close()
    SqlConexao.close()

def SaidaOperador():
    Consultar()
    delay(0.9)
    SaidaSql = SqlConexao.cursor()
    Saida = ("INSERT INTO carros(saida)"
                "VALUES (%s)")
    HoraSaida = input("Informe o modelo do veiculo:")
    SaidaSql.execute(Saida, ({HoraSaida},))
    confirma = int(input(f'Os dados cadastrados são {HoraSaida} . Deseja confirmar? (1 - SIM / 2 - NÃO)'))
    if confirma == 1:
        SqlConexao.commit()
        Return()
    else:
        SaidaOperador()
    SaidaSql.close()
    SqlConexao.close()

def ConsultaOperador():
    ConsultaSQL = SqlConexao.cursor()
    Consulta = ('SELECT placa,entrada FROM carros WHERE placa = %s')
    placa = input('Informe a placa do veiculo: ')
    print("#" * 10)
    print('Consultando...')
    print("#" * 10)
    delay(1.3)
    ConsultaSQL.execute(Consulta, ({placa}))
    ApresentaConsulta = ConsultaSQL.fetchall()
    for Placa in ApresentaConsulta:
        print("#" * 10)
        print(f'{Placa} - PLACA CADASTRADA')
        print("#" * 10)
        op = int(input('1 - Nova Consulta | 2 - Retornar ao Menu : '))
        if op == 1:
            Consultar()
        if op == 2:
            Return()
    delay(0.4)
    print("#" * 10)
    print(f'{placa} - NAO CADASTRADA')
    print("#" * 10)
    op = int(input('1 - Nova Consulta | 2 - Retornar ao Menu : '))
    if op == 1:
        Consultar()
    if op == 2:
        Return()
    ConsultaSQL.close()
    SqlConexao.close()

def Return():
    import OPERADOR
    OPERADOR.Menu()


#! /usr/bin/env python
# coding: utf-8

from flask import Flask
from flask import render_template, request, url_for, redirect
from model.webpayment import WebPaymentModel
from model.qrcode import QrCodeModel
from model.database import DatabaseModel
from model.mapping import MappingModel
from model.form import qrCodeGenerateForm

from datetime import datetime, timezone, timedelta
import pprint
import requests

app = Flask(__name__)

@app.route("/", methods=['GET', 'POST'])
def main():

  form = qrCodeGenerateForm()

  return render_template("index.html", form = form)

@app.route("/qrcode", methods=["POST"])
def qrcode():

  if request.method == "POST":

    webpayment = WebPaymentModel()
    dbm = DatabaseModel()
    qrcode = QrCodeModel()

    code = webpayment.createCode(request.form)

    # pprint.pprint(code)

    # insert GeneratedPaymentData to DB
    dbm.insertData(code)

    # Generate to Qrcode from PaymentData
    qrcode_image_name = qrcode.generateQrcode(code["data"]["url"], code["data"]["codeId"])

    # epoch to JST
    JST = timezone(timedelta(hours=+9), 'JST')
    expiryDate = datetime.fromtimestamp(code["data"]["expiryDate"], JST)
    requestedAt = datetime.fromtimestamp(code["data"]["requestedAt"], JST)

    return render_template("qrcode.html",
      code = code,
      qrcode_image_name = qrcode_image_name,
      expiryDate = expiryDate,
      requestedAt = requestedAt
    )

@app.route("/list")
def list():

  dbm = DatabaseModel()
  mm = MappingModel()

  qr_list_s_src = dbm.selectData()

  # create dict from list
  qr_list_s = mm.createListMapDict(qr_list_s_src)

  # pprint.pprint(qr_list_s)

  return render_template("list.html", qr_list_s = qr_list_s)

if __name__ == "__main__":
    app.run(host="0.0.0.0")


#
# Author: Brendan Reidy
# Email: bcreidy@email.sc.edu
# Date created: Sep 10 2022
# Last Modified: Sep 11 2022
#

import tensorflow as tf
import json
import TransformerModel
import re
WORD = re.compile(r"\w+")
mappedWeights = []
outputMap = []
def word2vec(word):
    from collections import Counter
    from math import sqrt

    # count the characters in word
    cw = Counter(word)
    # precomputes a set of the different characters
    sw = set(cw)
    # precomputes the "length" of the word vector
    lw = sqrt(sum(c*c for c in cw.values()))

    # return a tuple
    return cw, sw, lw

def cosdis(v1, v2):
    v1 = word2vec(v1)
    v2 = word2vec(v2)
    # which characters are common to the two words?
    common = v1[1].intersection(v2[1])
    # by definition of cosine distance we have
    return sum(v1[0][ch]*v2[0][ch] for ch in common)/v1[2]/v2[2]

def from_hub_encoder(hub_encoder, configPath, strategy=None):
    with open(configPath) as json_file:
        data = json.load(json_file)
    n_layers = data["num_hidden_layers"]
    num_heads = data["num_attention_heads"]
    vocab_size = data["vocab_size"]
    seq_len = data["max_position_embeddings"]
    n_segments = data["type_vocab_size"]
    d_model = data["hidden_size"]
    intermediate_size = data["intermediate_size"]
    activation = data["hidden_act"]

    with strategy.scope():
        x = tf.keras.layers.Input(shape=(128), dtype=tf.float32, name="input_word_ids", ragged=False)
        seg = tf.keras.layers.Input(shape=(128), dtype=tf.float32, name="input_type_ids", ragged=False)
        mask = tf.keras.layers.Input(shape=(128), dtype=tf.float32, name="input_mask", ragged=False)
        custom_encoder = TransformerModel.BERT(n_layers, num_heads, vocab_size, seq_len, n_segments, d_model, intermediate_size, activation=activation, name="transformer")(x, seg, mask)
        encoder_model = tf.keras.Model(inputs=[x, seg, mask], outputs=[custom_encoder])
    inject_weights(hub_encoder, encoder_model, n_layers, num_heads)
    return encoder_model

def removeFromList(name):
    global mappedWeights
    if name in mappedWeights:
        mappedWeights.remove(name)
    else:
        print("ERROR", name, "not in list")
def getWeightByName(model, name, exact=False):
    for weight in model.weights:
        if exact:
            if weight.name == name:
                return weight
        elif name in weight.name:
            return weight
    return None

def setWeightByName(model, name, inWeight, pseudoName=None):
    global outputMap
    closest = -9999999999999999999
    closestVal = None
    for i, weight in enumerate(model.weights):
        sim = cosdis(name, weight.name)
        if sim > closest:
            closest = sim
            closestVal = weight.name
        if name in weight.name:
            assert weight.shape==inWeight.shape
            tempName = weight.name
            model.weights[i].assign(inWeight)
            if pseudoName is None:
                outputMap.append((inWeight.name, weight.name))
            else:
                outputMap.append((pseudoName, weight.name))
            removeFromList(weight.name)
            return
    raise Exception("ModelConverter was unable to find layer: " + name + "\nDid you mean " + str(closestVal))

def injectEmbeddings(fromModel, toModel):
    word_embeddings = getWeightByName(fromModel, "word_embeddings")
    setWeightByName(toModel, "word_embeddings", word_embeddings)
    position_embedding = getWeightByName(fromModel, "position_embedding")
    setWeightByName(toModel, "position_embedding", position_embedding)
    type_embeddings = getWeightByName(fromModel, "type_embeddings")
    setWeightByName(toModel, "type_embeddings", type_embeddings)

    layer_norm_gamma = getWeightByName(fromModel, "embeddings/layer_norm/gamma")
    setWeightByName(toModel, "transformer/bert_embedding/layer_normalization/gamma", layer_norm_gamma)
    layer_norm_beta = getWeightByName(fromModel, "embeddings/layer_norm/beta")
    setWeightByName(toModel, "transformer/bert_embedding/layer_normalization/beta", layer_norm_beta)
    print("Successfuly injected embedding values")

def injectMHA(fromModel, toModel, num_heads, layer=0):
    attn_layer_norm_gamma = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/self_attention_layer_norm/gamma")
    attn_layer_norm_beta = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/self_attention_layer_norm/beta")
    out_layer_norm_gamma = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/output_layer_norm/gamma")
    out_layer_norm_beta = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/output_layer_norm/beta")
    setWeightByName(toModel, "layer_" + str(layer) + "/attention_layer_norm/gamma", attn_layer_norm_gamma)
    setWeightByName(toModel, "layer_" + str(layer) + "/attention_layer_norm/beta", attn_layer_norm_beta)
    setWeightByName(toModel, "layer_" + str(layer) + "/output_layer_norm/gamma", out_layer_norm_gamma)
    setWeightByName(toModel, "layer_" + str(layer) + "/output_layer_norm/beta", out_layer_norm_beta)

    intermediate_kernel = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/intermediate/kernel")
    intermediate_bias = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/intermediate/bias")
    setWeightByName(toModel, "layer_" + str(layer) + "/intermediate/kernel", intermediate_kernel)
    setWeightByName(toModel, "layer_" + str(layer) + "/intermediate/bias", intermediate_bias)

    output_kernel = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/output/kernel")
    output_bias = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/output/bias")
    setWeightByName(toModel, "layer_" + str(layer) + "/kernel_out:", output_kernel)
    setWeightByName(toModel, "layer_" + str(layer) + "/bias_out:", output_bias)

    query_kernel = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/self_attention/query/kernel")
    query_bias = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/self_attention/query/bias")
    key_kernel = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/self_attention/key/kernel")
    key_bias = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/self_attention/key/bias")
    value_kernel = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/self_attention/value/kernel")
    value_bias = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/self_attention/value/bias")
    attn_output_kernel = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/self_attention/attention_output/kernel")
    attn_output_kernel_name = attn_output_kernel.name
    attn_output_bias = getWeightByName(fromModel, "transformer/layer_" + str(layer) + "/self_attention/attention_output/bias")

    attn_output_kernel = tf.reshape(attn_output_kernel, (attn_output_kernel.shape[1]*num_heads, attn_output_kernel.shape[2]))
    setWeightByName(toModel, "layer_" + str(layer) + "/mha/mhaattention_output_kernel:", attn_output_kernel, pseudoName=attn_output_kernel_name)
    setWeightByName(toModel, "layer_" + str(layer) + "/mha/mhaattention_output_bias:", attn_output_bias)

    for h in range(num_heads):
        queryTempK = query_kernel[:,h:h+1,:]
        queryTempK = tf.reshape(queryTempK, (queryTempK.shape[0], queryTempK.shape[2]))
        queryTempB = query_bias[h:h+1,:]
        queryTempB = tf.reshape(queryTempB, (queryTempB.shape[1]))

        keyTempK = key_kernel[:,h:h+1,:]
        keyTempK = tf.reshape(keyTempK, (keyTempK.shape[0], keyTempK.shape[2]))
        keyTempB = query_bias[h:h+1,:]
        keyTempB = tf.reshape(keyTempB, (keyTempB.shape[1]))

        valueTempK = value_kernel[:,h:h+1,:]
        valueTempK = tf.reshape(valueTempK, (valueTempK.shape[0], valueTempK.shape[2]))
        valueTempB = value_bias[h:h+1,:]
        valueTempB = tf.reshape(valueTempB, (valueTempB.shape[1]))

        setWeightByName(toModel, "layer_" + str(layer) + "/mha/mhasdp_" + str(h) + "/mhasdp_" + str(h) + "_kernel_q", queryTempK, pseudoName=query_kernel.name)
        setWeightByName(toModel, "layer_" + str(layer) + "/mha/mhasdp_" + str(h) + "/mhasdp_" + str(h) + "_bias_q", queryTempB, pseudoName=query_bias.name)

        setWeightByName(toModel, "layer_" + str(layer) + "/mha/mhasdp_" + str(h) + "/mhasdp_" + str(h) + "_kernel_k", keyTempK, pseudoName=key_kernel.name)
        setWeightByName(toModel, "layer_" + str(layer) + "/mha/mhasdp_" + str(h) + "/mhasdp_" + str(h) + "_bias_k", keyTempB, pseudoName=key_kernel.name)

        setWeightByName(toModel, "layer_" + str(layer) + "/mha/mhasdp_" + str(h) + "/mhasdp_" + str(h) + "_kernel_v", valueTempK, pseudoName=value_kernel.name)
        setWeightByName(toModel, "layer_" + str(layer) + "/mha/mhasdp_" + str(h) + "/mhasdp_" + str(h) + "_bias_v", valueTempB, pseudoName=value_bias.name)

def inject_weights(fromModel, toModel, n_layers, num_heads):
    global mappedWeights
    global outputMap
    mappedWeights = []
    for weight in toModel.weights:
        mappedWeights.append(weight.name)
    injectEmbeddings(fromModel, toModel)
    for layer in range(n_layers):
        injectMHA(fromModel, toModel, num_heads, layer=layer)

    pooler_kernel = getWeightByName(fromModel, "pooler_transform/kernel")
    pooler_bias = getWeightByName(fromModel, "pooler_transform/bias")
    setWeightByName(toModel, "transformer/transformerpooler_transform_kernel", pooler_kernel)
    setWeightByName(toModel, "transformer/transformerpooler_transform_bias", pooler_bias)
    showOuputMap(outdir=toModel.name + "_mapping.log")

def showOuputMap(outdir=None):
    global outputMap
    global mappedWeights
    if not outdir is None:
        with open(outdir, 'w') as fp:
            for b,a in outputMap:
                fp.write("[X] " +  a + " -> " + b + "\n")
            for a in mappedWeights:
                fp.write("[_] " +  a + " -> \n")
    else:
        for b,a in outputMap:
            print("[X]", a, "->", b)
        for a in mappedWeights:
            print("[_]", a, "->")

def getTrainableParams(model):
    totalSize = 0
    for weight in model.weights:
        currSize = 1
        if "Variable:0" in weight.name:
            continue
        for axis in weight.shape:
            currSize*=axis
        totalSize+=currSize
    return totalSize


# -*- coding: utf-8 -*-
"""Profile - Select Business Type"""
from requests import Response, Session

from directory_tests_shared import BusinessType, PageType, Service, URLs
from tests.functional.utils.context_utils import Actor
from tests.functional.utils.request import Method, check_response, make_request

SERVICE = Service.PROFILE
NAME = "Select business type"
TYPE = PageType.FORM
URL = URLs.PROFILE_ENROL_SELECT_BUSINESS_TYPE.absolute
EXPECTED_STRINGS = [
    "Select your business type",
    "My business is registered with Companies House",
    "I'm a sole trader or I represent another type of UK ",
    "I'm a UK taxpayer but do not represent a business",
    "My business or organisation is not registered in the UK",
]

BUSINESS_TYPES = {
    BusinessType.COMPANIES_HOUSE: "companies-house-company",
    BusinessType.SOLE_TRADER: "non-companies-house-company",
    BusinessType.CHARITY: "non-companies-house-company",
    BusinessType.PARTNERSHIP: "non-companies-house-company",
    BusinessType.OTHER: "non-companies-house-company",
    BusinessType.INDIVIDUAL: "not-company",
    BusinessType.OVERSEAS_COMPANY: "overseas-company",
}

def go_to(session: Session) -> Response:
    headers = {
        "Referer": f"{URLs.PROFILE_ENROL.absolute}?next={URLs.DOMESTIC_LANDING_UK.absolute}"
    }
    return make_request(Method.GET, URL, headers=headers, session=session)

def should_be_here(response: Response):
    check_response(response, 200, body_contains=EXPECTED_STRINGS)

def submit(actor: Actor, business_type: BusinessType) -> Response:
    session = actor.session
    headers = {"Referer": URLs.PROFILE_ENROL_SELECT_BUSINESS_TYPE.absolute}
    data = {
        "csrfmiddlewaretoken": actor.csrfmiddlewaretoken,
        "choice": BUSINESS_TYPES[business_type],
    }

    return make_request(
        Method.POST,
        URL,
        session=session,
        headers=headers,
        files=data,
        no_filename_in_multipart_form_data=True,
    )


import argparse
import concurrent.futures
import itertools
import logging
import pathlib
import sys

import numpy as np

from . import data
from . import utils

logger = logging.getLogger(__name__)

def _do_convert(args):
    mesh = data.load_mesh(args.input)
    data.save_mesh(args.output, mesh)

def _do_shoot(args):
    mesh = data.load_mesh(str(args.input))

    has_min_holes = args.min_holes is not None
    has_max_holes = args.max_holes is not None
    if has_min_holes != has_max_holes:
        raise ValueError("--min-holes and --max-holes must be set together")
    n_holes = ((args.min_holes, args.max_holes)
               if has_min_holes
               else args.holes)

    has_min_dropout = args.min_dropout is not None
    has_max_dropout = args.max_dropout is not None
    if has_min_dropout != has_max_dropout:
        raise ValueError(
            "--min-dropout and --max-dropout must be set together")
    dropout = ((args.min_dropout, args.max_dropout)
               if has_min_dropout
               else args.dropout)

    mask_faces = (np.load(args.mask) if args.mask is not None
                  else None)
    faces = None if mask_faces is None else mesh.faces
    point_indices = utils.shoot_holes(mesh.vertices,
                                      n_holes,
                                      dropout,
                                      mask_faces=mask_faces,
                                      faces=faces)
    shot = utils.remove_points(mesh, point_indices)
    shot.save(str(args.output))

def identify_meshes(dir_):
    """List meshes and identify the challenge/track in a directory tree."""
    meshes_track1 = list(dir_.glob("**/*_normalized.npz"))
    meshes_track2 = list(dir_.glob("**/fusion_textured.npz"))
    meshes_challenge2 = list(dir_.glob("**/model_*.obj"))
    if meshes_track1:
        meshes = sorted(meshes_track1)
        challenge = 1
        track = 1
    elif meshes_track2:
        meshes = sorted(meshes_track2)
        challenge = 1
        track = 2
    elif meshes_challenge2:
        meshes = sorted(meshes_challenge2)
        challenge = 2
        track = None
    else:
        meshes = []
        challenge = None
        track = None

    return meshes, challenge, track

def shoot_helper(mesh_index,
                 path,
                 n_holes,
                 dropout,
                 shape_seeds,
                 n_meshes,
                 input_dir,
                 output_dir,
                 mask_dir=None,
                 ):
    logger.info(f"{mesh_index + 1}/{n_meshes} -------- TEST ------------ processing {path}")

    rel_path = path.relative_to(input_dir)

    # Lazily load the mesh and mask when it is sure they are needed.
    mesh_cached = None
    mask_cached = None

    def make_name_suffix(shape_index, n_shapes):
        if n_shapes == 1:
            return "partial"
        else:
            # Assuming 0 <= shape_index <= 99.
            return f"partial-{shape_index:02d}"

    def load_mask(rel_path):
        mask_name = f"{rel_path.stem}-mask.npy"
        mask_rel_path = rel_path.with_name(mask_name)
        mask_path = mask_dir / mask_rel_path
        logger.info(f"loading mask {mask_path}")
        mask_faces = np.load(mask_path)
        return mask_faces

    n_shapes = len(shape_seeds)
    for shape_index, shape_seed in enumerate(shape_seeds):
        out_name_suffix = make_name_suffix(shape_index, n_shapes)
        out_name = f"{rel_path.stem}-{out_name_suffix}.npz"
        out_rel_path = rel_path.with_name(out_name)
        out_path = output_dir / out_rel_path

        if out_path.exists():
            logger.warning(f"shape exists, skipping {out_path}")
            continue

        logger.info(f"generating shape {shape_index + 1}/{n_shapes}")

        out_path.parent.mkdir(parents=True, exist_ok=True)

        if mesh_cached is None:
            mesh_cached = data.load_mesh(str(path))
        if mask_cached is None and mask_dir is not None:
            mask_cached = load_mask(rel_path)

        mesh = mesh_cached
        mask = mask_cached

        logger.info(f"TEST shape seed = {shape_seed}")
        shape_rng = np.random.default_rng(shape_seed)
        point_indices = utils.shoot_holes(mesh.vertices,
                                          n_holes,
                                          dropout,
                                          mask_faces=mask,
                                          faces=mesh.faces,
                                          rng=shape_rng)
        partial = utils.remove_points(mesh, point_indices)

        logger.info(f"{shape_index + 1}/{n_shapes} saving {out_path}")
        partial.save(str(out_path))

def _do_shoot_dir(args):
    """Generate partial data in a directory tree of meshes.

    An independent PRNG is used for each partial shape. Independent seeds are
    created in advance for each partial shape. The process is thus reproducible
    and can also be interrupted and resumed without generating all the previous
    shapes.
    """
    input_dir = args.input_dir
    output_dir = args.output_dir
    mask_dir = args.mask_dir
    seed = args.seed
    n_holes = args.holes
    dropout = args.dropout
    n_shapes = args.n_shapes
    n_workers = args.n_workers

    logger.info("generating partial data in directory tree")
    logger.info(f"input dir = {input_dir}")
    logger.info(f"output dir = {output_dir}")
    logger.info(f"mask dir = {mask_dir}")
    logger.info(f"seed = {seed}")
    logger.info(f"holes = {n_holes}")
    logger.info(f"dropout = {dropout}")
    logger.info(f"n_shapes = {n_shapes}")
    logger.info(f"n_workers = {n_workers}")

    mesh_paths, challenge, track = identify_meshes(input_dir)
    if challenge is None:
        raise ValueError(f"could not identify meshes in {input_dir}")
    logger.info(f"detected challenge {challenge} track {track}")
    n_meshes = len(mesh_paths)
    logger.info(f"found {n_meshes} meshes")

    logger.info(f"setting random seed {seed}")
    rng = np.random.default_rng(seed)

    # Create in advance the individual initial random states for each partial
    # shape to be generated so that the process of generation can be
    # - resumed without regenerating the previous shapes,
    # - parallelised.
    seeds = rng.integers(1e12, size=(n_meshes, n_shapes))

    mesh_indices = range(n_meshes)
    with concurrent.futures.ProcessPoolExecutor(max_workers=n_workers) as executor:
        executor.map(
            shoot_helper,
            mesh_indices,
            mesh_paths,
            itertools.repeat(n_holes),
            itertools.repeat(dropout),
            seeds.tolist(),
            itertools.repeat(n_meshes),
            itertools.repeat(input_dir),
            itertools.repeat(output_dir),
            itertools.repeat(mask_dir),
        )

def _parse_args():
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()

    parser_convert = subparsers.add_parser(
        "convert",
        help="Convert between mesh formats.",
    )
    parser_convert.add_argument("input", type=pathlib.Path)
    parser_convert.add_argument("output", type=pathlib.Path)
    parser_convert.set_defaults(func=_do_convert)

    parser_shoot = subparsers.add_parser(
        "shoot",
        help="Generate partial data with the shooting method.",
    )
    parser_shoot.add_argument("input", type=pathlib.Path)
    parser_shoot.add_argument("output", type=pathlib.Path)
    parser_shoot.add_argument(
        "--holes", type=int, default=40,
        help="Number of holes to shoot.",
    )
    parser_shoot.add_argument(
        "--min-holes", type=int, default=None,
        help="Minimum number of holes to generate."
             " (Supersedes --holes and requires --max-holes.)",
    )
    parser_shoot.add_argument(
        "--max-holes", type=int, default=None,
        help="Maximum number of holes to generate."
             " (Supersedes --holes and requires --min-holes.)",
    )
    parser_shoot.add_argument(
        "--dropout", type=float, default=2e-2,
        help="Proportion of points of the mesh to remove in a single hole.",
    )
    parser_shoot.add_argument(
        "--min-dropout", type=float, default=None,
        help="Minimum proportion of points of the mesh to remove in a single "
             "hole."
             " (Supersedes --dropout and requires --max-dropout.)",
    )
    parser_shoot.add_argument(
        "--max-dropout", type=float, default=None,
        help="Maximum proportion of points of the mesh to remove in a single "
             "hole."
             " (Supersedes --dropout and requires --min-dropout.)",
    )
    parser_shoot.add_argument(
        "--mask", type=pathlib.Path,
        help=" (optional) Path to the mask (.npy) to generate holes only on"
             " regions considered for evaluation.",
    )
    parser_shoot.set_defaults(func=_do_shoot)

    parser_shoot_dir = subparsers.add_parser(
        "shoot_dir",
        help="Generate partial data with the shooting method for a directory"
             " tree of meshes."
    )
    parser_shoot_dir.add_argument("input_dir", type=pathlib.Path)
    parser_shoot_dir.add_argument("output_dir", type=pathlib.Path)
    parser_shoot_dir.add_argument(
        "--mask-dir",
        type=pathlib.Path,
        help=" (optional) Directory tree with the masks (.npy). If defined,"
             " the partial data is created only on the non-masked faces of the"
             " meshes. (Only valid for challenge 1.)",
    )
    parser_shoot_dir.add_argument(
        "--holes",
        type=int,
        default=40,
        help="Number of holes to shoot.",
    )
    parser_shoot_dir.add_argument(
        "--dropout",
        type=float,
        default=2e-2,
        help="Proportion of points of the mesh to remove in a single hole.",
    )
    parser_shoot_dir.add_argument(
        "--seed",
        type=int,
        default=None,
        help="Initial state for the pseudo random number generator."
             " If not set, the initial state is not set explicitly.",
    )
    parser_shoot_dir.add_argument(
        "-n", "--n-shapes",
        type=int,
        default=1,
        help="Number of partial shapes to generate per mesh."
             " If n = 1 (default), the shape is saved as"
             " '<mesh_name>-partial.npz'."
             " If n > 1, the shapes are saved as '<mesh_name>-partial-XY.npz'"
             ", with XY as 00, 01, 02... (assuming n <= 99).",
    )
    parser_shoot_dir.add_argument(
        "--n-workers",
        type=int,
        default=None,
        help="Number of parallel processes. By default, the number of"
             " available processors.",
    )
    parser_shoot_dir.set_defaults(func=_do_shoot_dir)

    args = parser.parse_args()

    # Ensure the help message is displayed when no command is provided.
    if "func" not in args:
        parser.print_help()
        sys.exit(1)

    return args

def main():
    args = _parse_args()
    args.func(args)

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    main()


#Libraries
from collections import deque

#TQ=2
time_quantum = 2

class Prs:
    def __init__(self, name, AT, RT):
        self.name = name
        self.AT = AT
        self.RT = RT
        self.t_t = 0
        self.CPU = -1

    def __repr__(self):
        return self.name

#        Pid  AT  BT
p1 = Prs('P1', 0, 5)
p2 = Prs('P2', 1, 4)
p3 = Prs('P3', 2, 2)
p4 = Prs('P4', 4, 1)
P = [p1, p2, p3, p4]
end_times = {Prs.name: 0 for Prs in P}
WT = {Prs.name: 0 for Prs in P}
R_T = {Prs.name: 0 for Prs in P}
queue = deque()
run_prs = None  # Tracks running Prs in the CPU
P_s = 0  # Tracks the time running Prs spent in the CPU

for t in range(12):
    if t == 0:
        for p in P:
            if p.AT == 0:
                queue.append(p)

    if run_prs == None:
        if len(queue) > 0:
            run_prs = queue.popleft()
            if run_prs.CPU == -1:
                run_prs.CPU = t
            P_s = 0

    for p in P:
        if p.AT == t + 1:
            queue.append(p)

    if run_prs is not None:
        run_prs.t_t += 1
        P_s += 1
        if run_prs.RT == run_prs.t_t:
            end_times[str(run_prs)] = t + 1
            WT[str(run_prs)] = end_times[str(run_prs)] - run_prs.AT - run_prs.RT
            R_T[str(run_prs)] = run_prs.CPU - run_prs.AT
            run_prs = None
            P_s = 0
        elif P_s >= time_quantum:
            queue.append(run_prs)
            run_prs = None
            P_s = 0

print()
print("G A N T T-C H A R T")
print(end_times)
print()

print("Prs\t AT\t BT\t TAT\t WT\t RT")
t_TAT = 0
t_WT = 0
t_RT = 0
for p in P:
    tat = end_times[p.name] - p.AT
    wt = WT[p.name]
    rt = R_T[p.name]
    t_TAT += tat
    t_WT += wt
    t_RT += rt
    print(p.name, "\t", p.AT, "\t", p.RT, "\t", tat, "\t", wt, "\t", rt)
m = len(P)
A_tat = t_TAT / m
A_wt = t_WT / m
A_rt = t_RT / m

#AvErAgE
print("A tat: ", A_tat)
print("A wt: ", A_wt)
print("A rt: ", A_rt)


import labrad
import numpy as np
import matplotlib

from matplotlib import pyplot
from scipy.optimize import curve_fit
from scipy.stats import chi2

cxn = labrad.connect()
dv = cxn.data_vault
#datasets = ['2105_00','2112_12', ('2121_24','2123_16')]

dv.cd(['','Experiments','729Experiments','RamseyDephase','2012Dec20','2105_00'])
dv.open(1)
data1 = dv.get().asarray.transpose()

dv.cd(['','Experiments','729Experiments','RamseyDephase','2012Dec20','2112_12'])
dv.open(1)
data2 = dv.get().asarray.transpose()

dv.cd(['','Experiments','729Experiments','RamseyDephase','2012Dec20','2121_24'])
dv.open(1)
data3_1 = dv.get().asarray

dv.cd(['','Experiments','729Experiments','RamseyDephase','2012Dec20','2123_16'])
dv.open(1)
data3_2 = dv.get().asarray

data3 = np.concatenate((data3_1,data3_2), axis = 0).transpose()

figure = pyplot.figure()
pyplot.title('Ramsey Fringes')
pyplot.plot(data1[0],data1[1], 'k', label = 'S+1/2D+1/2 with line triggering 2012Dec20 2105_00')
pyplot.plot(data2[0],data2[1], 'b', label = 'S+1/2D+1/2 with no line triggering 2012Dec20 2112_12')
pyplot.plot(data3[0],data3[1], 'r', label = 'S+1/2D+5/2 with line triggering 2012Dec20 2121_24 + 2123_16')

pyplot.ylabel('Excitation Probability')
pyplot.xlabel(r'Delay Time between two $\pi / 2$ pulses, $\mu s$')
pyplot.legend()
pyplot.show()


from flask import Flask, jsonify
from flask_restful import Api, Resource, reqparse
from os import environ
import redis
import json
import uuid

app = Flask(__name__)
api = Api(app)

CONN = redis.Redis(
    host=environ.get('REDIS_HOST'),
    port=environ.get('REDIS_PORT'),
    db=environ.get('REDIS_DB'))

class LongTaskResource(Resource):
    parser = reqparse.RequestParser()
    parser.add_argument('number', type=int, help='number type is ianteger.')
    TASK_PREFIX = 'long-task-'

    def post(self):
        args = self.parser.parse_args()
        number = args.get('number')
        queue_id = '{}{}'.format(self.TASK_PREFIX, str(uuid.uuid4()))
        msg = {'number': number, 'uuid': queue_id}

        CONN.lpush('my-queue', json.dumps(msg))

        response = jsonify(result_id=queue_id)
        response.status_code = 200
        return response

class TaskResultResource(Resource):
    parser = reqparse.RequestParser()
    parser.add_argument('result_id', type=str, help='result_id is str')

    def get(self):
        args = self.parser.parse_args()
        result_id = args.get('result_id')
        value = CONN.hget('result', result_id)
        if value:
            data = json.loads(value.decode('utf-8'))
            response = jsonify({'task_status': 'finish', 'data': data})
        else:
            response = jsonify({'task_status': 'wait'})
        response.cache_control.max_age = 60
        return response

api.add_resource(LongTaskResource, '/task')
api.add_resource(TaskResultResource, '/result')

if __name__ == '__main__':
    app.run(port=5000, host='0.0.0.0', debug=True)


import os
import re
from pathlib import Path
from re import Match
from typing import Optional

from app.records import NameFormats

class Location:
    _path: Path

    def __init__(self, path_arg: str | Path) -> None:
        self.path = path_arg

    @property
    def path(self) -> Path:
        return self._path

    @path.setter
    def path(self, path_arg: str | Path) -> None:
        if isinstance(path_arg, str):
            if path_arg == "":
                self._path = Path()
                return

            path_exp_user: str = os.path.expanduser(path_arg)
            path_exp_vars: str = os.path.expandvars(path_exp_user)

            self._path = Path(path_exp_vars).resolve()

        if isinstance(path_arg, Path):
            self._path = path_arg

    @property
    def is_valid(self) -> bool:
        """
        Checks if ._path is initialized properly and not using its default value (i.e. for empty strings).
        """
        if str(self._path) == ".":
            return False
        return self._path.is_dir()

    @property
    def is_media_dir(self) -> bool:
        is_media_dir: bool = re.match(r"^/media/.+", str(self._path)) is not None
        is_test_temp_media_dir: bool = (
            re.match(r"^/tmp/pytest.+media$", str(self._path)) is not None
        )
        return is_media_dir or is_test_temp_media_dir

    @property
    def name(self) -> str:
        return self.__class__.__name__

class Source(Location):
    pass

class Target(Location):
    existing_backup: Optional[Path] = None
    backup_name: Optional[str] = None

    @property
    def path(self) -> Path:
        return super().path

    @path.setter
    def path(self, path_arg: str | Path) -> None:
        Location.path.fset(self, path_arg)
        if self.is_valid:
            self._path = self._get_contained_backup_dir(self._path)

    def _get_contained_backup_dir(self, path: Path) -> Path:
        matches: list[str] = sorted(
            [str(path) for path in path.iterdir() if self._matches_backup_dir(path)]
        )
        if len(matches) > 0:
            return path / matches[0]
        return path

    def _matches_backup_dir(self, item: Path) -> bool:
        match: Match[str] | None = re.match(r".+/[Bb]ackup[s]*$", str(item))
        return match is not None and item.is_dir()

    @property
    def content(self) -> list[str]:
        directory: Path
        if self.existing_backup is not None:
            directory = self.existing_backup
        else:
            directory = self._path
        return sorted([self._add_slash_to_dir(path) for path in directory.iterdir()])

    def _add_slash_to_dir(self, path: Path) -> str:
        if path.is_dir():
            return f"{path.name}/"
        else:
            return path.name

    @property
    def content_dirs(self) -> list[str]:
        return sorted(
            [item[0 : len(item) - 1 :] for item in self.content if item[len(item) - 1] == "/"]
        )

    def select_existing_backup(
        self, ignore_datetime: bool, name_arg: int, name_formats: NameFormats
    ) -> None:
        if ignore_datetime:
            if name_arg <= 3:
                name_to_find: str = name_formats.select(1)
            else:
                name_to_find: str = name_formats.select(4)

            self._select_latest_existing_backup(name_to_find)

        else:
            if self.backup_name in self.content_dirs:
                self.existing_backup = self.path / self.backup_name

    def _select_latest_existing_backup(self, name_to_find: str) -> None:
        matches: list[str] = [
            dir for dir in sorted(self.content_dirs, reverse=True) if dir.startswith(name_to_find)
        ]
        self.existing_backup = self.path / matches[0] if matches else None


import logging

import numpy as np
import pytest

from rtCommon.bidsCommon import (
    adjustTimeUnits,
    getDicomMetadata,
    getNiftiData,
    loadBidsEntities,
    metadataFromProtocolName,
)

logger = logging.getLogger(__name__)

# Test time units are adjusted correctly from DICOM (msec) to BIDS (sec)
def testTimeUnitAdjustment():
    rtKey = 'RepetitionTime'
    etKey = 'EchoTime'
    metadata = {}

    # Test values that are correct numbers, but strings and need to be converted
    rtValidButString = '50'
    etValidButString = '.5'
    metadata[rtKey] = rtValidButString
    metadata[etKey] = etValidButString

    adjustTimeUnits(metadata)
    assert metadata[rtKey] == float(rtValidButString)
    assert metadata[etKey] == float(etValidButString)

    # Test values above max, but convertible
    rtConvertible = 1000
    etConvertible = 10
    metadata[rtKey] = rtConvertible
    metadata[etKey] = etConvertible

    adjustTimeUnits(metadata)
    assert metadata[rtKey] == rtConvertible / 1000.0
    assert metadata[etKey] == etConvertible / 1000.0

    # Test values above max, but convertible
    rtAboveMax = 1000 * 100 + 1
    etAboveMax = 1000 * 1 + 1
    metadata[rtKey] = rtAboveMax
    metadata[etKey] = etAboveMax

    with pytest.raises(ValueError):
        adjustTimeUnits(metadata)

    # Test values within max
    rtWithinMax = 50
    etWithinMax = .5
    metadata[rtKey] = rtWithinMax
    metadata[etKey] = etWithinMax

    adjustTimeUnits(metadata)
    assert metadata[rtKey] == rtWithinMax
    assert metadata[etKey] == etWithinMax

    # Test missing values
    metadata[rtKey] = None
    metadata[etKey] = None

    adjustTimeUnits(metadata)
    assert metadata[rtKey] is None
    assert metadata[etKey] is None

# Test metadata is correctly extracted from a DICOM to public and private
# dictionaries by ensuring a sample of public keys have the right value
def testMetadataExtraction(dicomImage, dicomMetadataSample):
    with pytest.raises(TypeError):
        getDicomMetadata("this isn't a pydicom dataset")

    # Test a sampling of field names and values extracted by hand
    metadata = getDicomMetadata(dicomImage, kind='all')
    for field, value in dicomMetadataSample.items():
        assert metadata.get(field) == str(value)

# Ensure entitity dictionary is loaded and parsed properly
# Expected dictionary format:
#   key: Full entity name, all lowercase
def testEntitiesDictGeneration():
    entities = loadBidsEntities()

    # Ensure entity count correct
    # Manually summed from bids.json and derivatives.json, which are on Github
    # at bids-standard/pybids/bids/layout/config/
    NUM_ENTITIES = 38
    assert len(entities) == NUM_ENTITIES

    # Ensure case correct
    for key in entities.keys():
        assert key.islower()

    # Check a sample of important keys are present
    importantKeySample = ["subject", "task", "session", "datatype"]
    for key in importantKeySample:
        assert key in entities.keys()

# Test BIDS fields in a DICOM ProtocolName header field are properly parsed
def testParseProtocolName():
    # ensure nothing spurious is found in strings without BIDS fields
    assert metadataFromProtocolName("") == {}
    assert metadataFromProtocolName("this ain't bids") == {}
    assert metadataFromProtocolName("nor_is_this") == {}
    assert metadataFromProtocolName("still-aint_it") == {}

    protocolName = "func_ses-01_task-story_run-01"
    expectedValues = {'session': '01', 'task': 'story', 'run': '1'}

    parsedValues = metadataFromProtocolName(protocolName)

    for key, expectedValue in expectedValues.items():
        # pybids 0.15.1 changed the parsing of the 'run' entity
        # from run-0*(\d+) to run-(\d+), so the previous version
        # removed leading zeros from run number, but newer version
        # now includes leading zeros
        if key == 'run':
            assert parsedValues[key].lstrip('0') == expectedValue
        else:
            assert parsedValues[key] == expectedValue

# Test correct Nifti data is extracted
def testGetNiftiData(sample4DNifti1):
    extracted = getNiftiData(sample4DNifti1)
    fromRawDataobj = np.asanyarray(sample4DNifti1.dataobj,
                                   dtype=sample4DNifti1.dataobj.dtype)

    assert np.array_equal(extracted, fromRawDataobj)