# %%
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import sklearn
import seaborn as sns
import math
plt.style.use('ggplot')
plt.rcParams['axes.edgecolor'] = 'black'
plt.rcParams['font.sans-serif'] = ['Arial Unicode MS']

# %%
shiduW=pd.read_excel('土壤湿度2012-2022各月份.xls')
youjiwuO=pd.read_excel('内蒙古自治区锡林郭勒盟典型草原不同放牧强度土壤碳氮监测数据集（2012年8月15日-2020年8月15日）.xlsx')

# %%
shiduW.columns

# %%
plt.figure(figsize=(6,5))
shidulen=len(shiduW)
xticklabelsname=[str(shiduW.iloc[i,0])+'年'+str(shiduW.iloc[i,1])+'月' for i in range(len(shiduW))]
plt.plot(list(range(shidulen)),[math.log10(i) for i in shiduW.iloc[:,4]])
plt.plot(list(range(shidulen)),[math.log10(i) for i in shiduW.iloc[:,5]])
plt.plot(list(range(shidulen)),[math.log10(i) for i in shiduW.iloc[:,6]])
plt.plot(list(range(shidulen)),[math.log10(i) for i in shiduW.iloc[:,7]])
plt.xticks(list(range(0,shidulen,11)),xticklabelsname[::11],rotation='vertical')
plt.legend(['10cm','40cm','100cm','200cm'])
plt.ylabel('湿度（kg/m2)')
plt.yticks(list(range(1,3)),['10mm','100mm'])

# %%
# 模拟生成一组实验数据
x = [1,3,5,7]
y = [0.41,0.38,0.43,0.46]
plt.figure(figsize=(8,8))
fig, ax = plt.subplots()
ax.plot(x, y, 'b--')
ax.set_xlabel('x')
ax.set_ylabel('y')

# 二次拟合
coef = np.polyfit(x, y, 2)
y_fit = np.polyval(coef, x)
ax.plot(x, y_fit, 'g')
# 找出其中的峰值/对称点
if coef[0] != 0:
    x0 = -0.5 * coef[1] / coef[0]
    x0 = round(x0, 2)
    ax.plot([x0]*5, np.linspace(min(y),max(y),5),'r--')
    print(x0)
else:
    raise ValueError('Fail to fit.')
plt.xlabel('放牧强度')
plt.ylabel('沙漠化指数SM')
plt.show()
coef

# %%
import math
def f(x):
    res=0
    res+=0.68/(0.04*math.pow(2.7,-x*x/128)+1.36)
    print(0.68/(0.04*math.pow(2.7,-x*x/128)+1.36))
    res+=0.00375*x*x-0.002*x+0.422
    return res

temp=list(range(50))
x=[i/100 for i in temp]
y=[f(i) for i in x]
plt.plot(x,y)
plt.xlabel('放牧强度')
plt.ylabel('SM+B')


import numpy as np
import torch
import torch.nn.functional as F
import os
import matplotlib.pyplot as plt
import inspect

def FDD_BSA(U_flex, x, W2_CX, W2_CY, W2_CZ, Eigens2, TrackDOFs, Nloc, weight, ToOneV, inputDim, outputDim, D, DOF_exc, DOF_flex, ComDevice, switch, g=9.8):
    batch_size = U_flex.size(0)
    nt = U_flex.size(1)
    nf = U_flex.size(2)
    # DOF_exc = 3
    # DOF_rigid = 9
    # DOF_flex = (1 / 3) * (u.shape[-1] - 3 * DOF_rigid)

    # Extracting and ToOne actions for all parameters
    Excitation, U_flex, dU_flex, ddU_flex = ToOne_Action_BSA(ToOneV, x, U_flex, D, DOF_exc, DOF_flex)

    # for i in range(0, ddU_rigid.size(-1)):
    #     plt.figure(i)
    #     plt.plot(ddU_rigid[0, :, i].to(torch.device('cpu')).numpy())
    #     plt.show()
    # Coupling Points Information
    K = 5e6
    C = 1e6
    ConnectNum = 3
    alpha_k = 0.001
    alpha_c = 0.001
    nDOF = TrackDOFs.size(0)
    TrackDOFs = TrackDOFs.long()

    # Computing General Mode Forces for flexible system (Compute the connection displacement and velocity first)
    ConnectDis = torch.cat((Excitation[:, :, 0].unsqueeze(-1).repeat(1, 1, nDOF) - torch.matmul(U_flex, W2_CX.transpose(0, 1)),
                            Excitation[:, :, 1].unsqueeze(-1).repeat(1, 1, nDOF) - torch.matmul(U_flex, W2_CY.transpose(0, 1)),
                            Excitation[:, :, 2].unsqueeze(-1).repeat(1, 1, nDOF) - torch.matmul(U_flex, W2_CZ.transpose(0, 1))), dim=-1)
    ConnectVel = torch.cat((Excitation[:, :, 3].unsqueeze(-1).repeat(1, 1, nDOF) - torch.matmul(dU_flex, W2_CX.transpose(0, 1)),
                            Excitation[:, :, 4].unsqueeze(-1).repeat(1, 1, nDOF) - torch.matmul(dU_flex, W2_CY.transpose(0, 1)),
                            Excitation[:, :, 5].unsqueeze(-1).repeat(1, 1, nDOF) - torch.matmul(dU_flex, W2_CZ.transpose(0, 1))), dim=-1)
    # print('The connection information dimension is {}'.format(ConnectDis.shape))

    Gvec = torch.matmul(K * ConnectDis + C * ConnectVel,
                        torch.cat((W2_CX, W2_CY, W2_CZ), dim=0))

    # Encoding physics laws for Flexible modes
    Du_flex = ddU_flex - Gvec + Eigens2.squeeze(-1) * U_flex + (Eigens2.squeeze(-1) * alpha_k + alpha_c) * dU_flex

    # Applying ODEs weights for all equations, the variable TargetMin is equal to r in Algorithm 1.
    TargetMin = 0.02

    Du_flex *= TargetMin / weight.unsqueeze(-2)[:, :, :U_flex.size(-1)].to(ComDevice)

    # Computing magnitude information for all DU
    Du_Magnitude = torch.cat((torch.std(Du_flex, [0, 1]), torch.mean(Du_flex, [0, 1])))

    # Regenerate the data tensor for derivatives output
    MeanV, StdV = ToOneV[0, 1:, 0], ToOneV[0, 1:, 3]
    DOF_de = DOF_exc + DOF_flex
    dU_flex = (dU_flex - MeanV[DOF_de: DOF_de + DOF_flex]) / StdV[DOF_de: DOF_de + DOF_flex]
    DOF_dde = DOF_de + DOF_flex
    ddU_flex = (ddU_flex - MeanV[DOF_dde: DOF_dde + DOF_flex]) / StdV[DOF_dde: DOF_dde + DOF_flex]

    if switch == 1:
        Derivative_Data = torch.cat([dU_flex, ddU_flex], dim=-1)
    else:
        Derivative_Data = torch.zeros(1, 1).to(ComDevice)
    return Du_flex, Du_Magnitude, Derivative_Data

def BSA_PINO_loss(U_flex, x, weight, ToOneV, inputDim, outputDim, D, DOF_exc, DOF_flex, W2_CX, W2_CY, W2_CZ, Eigens2, TrackDOFs, Nloc, ComDevice, switch):
    batchsize = U_flex.size(0)
    nt = U_flex.size(1)
    U_flex = U_flex.reshape(batchsize, nt, outputDim)

    Du_flex, Du_Magnitude, Derivative_Data = FDD_BSA(U_flex, x, W2_CX, W2_CY, W2_CZ, Eigens2, TrackDOFs, Nloc, weight, ToOneV, inputDim, outputDim, D, DOF_exc, DOF_flex, ComDevice, switch, g=9.8)
    # criterion = torch.nn.SmoothL1Loss()
    criterion = torch.nn.L1Loss()
    loss_f = criterion(Du_flex, torch.zeros_like(Du_flex))
    return loss_f, Du_Magnitude, Derivative_Data

def Var_Encoder_BSA(x):
    Excitation = x[:, 1:-1, 1:]   # Excitations
    return Excitation

def ToOne_Action_BSA(ToOneV, x, U_flex, D, DOF_exc, DOF_flex):
    Excitation = Var_Encoder_BSA(x)
    MeanV, StdV = ToOneV[0, 1:, 0], ToOneV[0, 1:, 3]
    # ReNorm for Excitations, Flexible modes and Rigid motions
    Excitation = Excitation * StdV[:DOF_exc] + MeanV[:DOF_exc]
    U_flex = U_flex * StdV[DOF_exc:DOF_exc + DOF_flex] + MeanV[DOF_exc:DOF_exc + DOF_flex]

    # Derivatives computation
    nt = U_flex.size(1)
    dt = D / nt

    dU_flex = (U_flex[:, 2:, :] - U_flex[:, :-2, :]) / (2 * dt)
    ddU_flex = (U_flex[:, 2:, :] - 2 * U_flex[:, 1:-1, :] + U_flex[:, :-2, :]) / (dt ** 2)

    # Reshape the solution of the system
    U_flex = U_flex[:, 1:-1, :]

    return Excitation, U_flex, dU_flex, ddU_flex


# IMPORTS
from pathlib import Path
from isort import file
import numpy as np
import matplotlib.pyplot as plt
plt.rcParams.update({
    'axes.titlesize': 'medium',
    'axes.labelsize': 'small',
    'legend.title_fontsize' : 'small',
    }
)

# import datetime
from utils.atab import Atab

from pprint import pprint
import pandas as pd
from ipdb import set_trace as dbg

from utils.parse_plot_synop_ch import total_score_range, cat_total_score_range
from utils.check_params import check_params

def collect_relevant_files(file_prefix, file_postfix, debug, source_path, parameter):
    """Collect all files corresponding to current parameter in 'corresponding_files_dict'.

    Args:
        file_prefix (str): prefix of files we're looking for (i.e. total_scores)
        file_postfix (str): postfix of files we're looking for (i.e. .dat)
        debug (bool): add debug messages command prompt
        source_path (Path): path to directory, where source files are
        parameter (str): parameter of interest

    Returns:
        dict: dictionary containig all available lead time range (ltr) dataframes for parameter
    """
    # collect the files, corresponding to this parameter in the corresponding files dict.
    # the keys in this dict are the available lead time ranges for the current parameter.
    corresponding_files_dict = {}

    # for dbg purposes:
    files_list = []

    for file_path in source_path.glob(f"{file_prefix}*{parameter}{file_postfix}"):
        if file_path.is_file(): # check, that the corresponding path belongs to a file and not to a sub-directory

            # lt_range = key for corresponding_files_dict
            # TODO: change here, if ltr is longer than 5 chars
            lt_range = file_path.name[len(file_prefix):len(file_prefix)+5]

            # extract header & dataframe
            header = Atab(file=file_path, sep=" ").header
            df = Atab(file=file_path, sep=" ").data

            # clean df
            df = df.replace(float(header["Missing value code"][0]), np.NaN)

            df.set_index(keys='Score', inplace=True)

            # add information to dict
            corresponding_files_dict[lt_range] = {
                                                    # 'path':file_path,
                                                    'header':header,
                                                    'df':df
                                                }

            # add path of file to list of relevant files
            files_list.append(file_path)
    if debug:
        print(f"\nFor parameter: {parameter} these files are relevant:\n")
        pprint(files_list)
        print(f"\nThese files have been parsed & combined in the 'corresponding_files_dict'. Each key (lt-range) has a subdict with two keys: {corresponding_files_dict['19-24'].keys()}\n")

    return corresponding_files_dict

# enter directory / read total_scores files / call plotting pipeline
def _total_scores_pipeline(
    params_dict,
    plot_scores,
    plot_params,
    plot_cat_scores,
    plot_cat_params,
    plot_cat_thresh,
    file_prefix,
    file_postfix,
    input_dir,
    output_dir,
    season,
    model_version,
    grid,
    debug,
) -> None:
    """Read all ```ATAB``` files that are present in: data_dir/season/model_version/<file_prefix><...><file_postfix>
        Extract relevant information (parameters/scores) from these files into a dataframe.
        Rows --> Scores | Columns --> Stations | For each parameter, a separate station_scores File exists.

    Args:
        parameters (list): parameters, for which plots should be generated (i.e. CLCT, DD_10M, FF_10M, PMSL,...). part of file name
        file_prefix (str): prefix of files (i.e. time_scores)
        file_postfix (str): postfix of files (i.e. '.dat')
        input_dir (str): directory to seasons (i.e. /scratch/osm/movero/wd)
        output_dir (str): output directory (i.e. plots/)
        season (str): season of interest (i.e. 2021s4/)
        model_version (str): model_version of interest (i.e. C-1E_ch)
        scores (list): list of scores, for which plots should be generated
        debug (bool): print further comments & debug statements
    """
    print(f"\n--- initialising total scores pipeline")

    # tmp; define debug = True, to show debug statements for total_scores only
    debug = True

    source_path = Path(f"{input_dir}/{season}/{model_version}")
    for parameter in params_dict:
        corresponding_files_dict = collect_relevant_files(file_prefix, file_postfix, debug, source_path, parameter)

        # pass dict to plotting pipeline
        _generate_total_scores_plot(
            data=corresponding_files_dict,
            parameter=parameter,
            plot_scores=plot_scores,
            plot_params=plot_params,
            plot_cat_scores=plot_cat_scores,
            plot_cat_params=plot_cat_params,
            plot_cat_thresh=plot_cat_thresh,
            # TODO: add ens params/thresh/scores
            output_dir=output_dir,
            grid=grid,
            debug=debug,
        )
    return

############################################################################################################################
######################################### PLOTTING PIPELINE FOR TOTAL SCORES PLOTS #########################################
############################################################################################################################

def _set_ylim(param, score, ax, debug):
    # define limits for yaxis if available
    regular_param = (param, "min") in total_score_range.columns
    regular_score = score in total_score_range.index

    if regular_param and regular_score:
        lower_bound = total_score_range[param]["min"].loc[score]
        upper_bound = total_score_range[param]["max"].loc[score]
        # if debug:
        #     print(
        #         f"found limits for {param}/{score} --> {lower_bound}/{upper_bound}"
        # )
        if lower_bound != upper_bound:
            ax.set_ylim(lower_bound, upper_bound)

    # TODO: add computation of y-lims for cat & ens scores

    return

def _customise_ax(parameter, score, x_ticks,grid, ax):
    """Apply cosmetics to current ax.

    Args:
        parameter (str): current parameter
        score (str): current score
        x_ticks (list): list of x-ticks labels (lead time ranges, as strings)
        grid (bool): add grid to ax
        ax (Axes): current ax
    """
    if grid:
        ax.grid(which='major', color='#DDDDDD', linewidth=0.8)
        ax.grid(which='minor', color='#EEEEEE', linestyle=':', linewidth=0.5)
        ax.minorticks_on()

    ax.tick_params(axis='both', which='major', labelsize=8)
    ax.tick_params(axis='both', which='minor', labelsize=6)
    ax.set_title(f"{parameter}: {score}")
    ax.set_xlabel(f"Lead-Time Range (h)")
    ax.legend(fontsize=6)
    ax.set_xticks(range(len(x_ticks)), x_ticks)
    return

def _clear_empty_axes(subplot_axes, idx):
    # remove empty ``axes`` instances
    i = 1
    while (idx%4+i) < 4:
        ax = subplot_axes[idx%4+i]
        ax.axis('off')
        i += 1
    return

def _save_figure(output_dir, filename):
    print(f"---\t\tsaving: {output_dir}/{filename[:-1]}.png")
    plt.savefig(f"{output_dir}/{filename[:-1]}.png")
    plt.clf()
    return

def _generate_total_scores_plot(
    data,
    parameter,
    plot_params,
    plot_scores,
    plot_cat_scores,
    plot_cat_params,
    plot_cat_thresh,
    output_dir,
    grid,
    debug,
):
    """Generate Total Scores Plot."""
    if debug:
        print(f"--- starting plotting pipeline")
        print(f"---\t1) map parameter (i.e. TD_2M_KAL --> TD_2M*)")
    # get correct parameter, i.e. if parameter=T_2M_KAL --> param=T_2M*
    param = check_params(param=parameter, verbose=False) # TODO: change False back to debug

    if debug:
        print(f"---\t2) check if output_dir exists (& create it if necessary)")
    # check (&create) output directory for total scores plots
    # output_dir = f"{output_dir}/total_scores"
    if not Path(output_dir).exists():
        Path(output_dir).mkdir(parents=True, exist_ok=True)

    if debug:
        print(f"---\t3) initialise figure with a 2x2 subplots grid")
    # create 2x2 subplot grid
    fig, ((ax0, ax1),(ax2, ax3)) = plt.subplots(nrows=2, ncols=2, tight_layout=True, figsize=(10, 10), dpi=200)

    subplot_axes = {0:ax0, 1:ax1, 2:ax2, 3:ax3} # hash map to access correct axes later on

    # ltr_unsorted  ->  unsorted lead time ranges
    # ltr_sorted    ->  sorted lead time ranges (used for x-tick-labels later on )
    ltr_unsorted, ltr_sorted = list(data.keys()), []
    ltr_start_times_sorted = sorted([int(lt.split('-')[0]) for lt in ltr_unsorted])
    for idx, ltr_start in enumerate(ltr_start_times_sorted):
        for ltr in ltr_unsorted:
            if ltr.startswith(str(ltr_start).zfill(2)):
                ltr_sorted.insert(idx, ltr)

    # re-name & create x_int list, s.t. np.arrays are plottet against each other
    x_ticks = ltr_sorted
    x_int = list(range(len(ltr_sorted)))

    if debug:
        print(f"---\t4) create x-axis")
        print(f"---\t\tUnsorted ltr list:\t\t{ltr_unsorted}")
        print(f"---\t\tSorted ltr start times list:\t{ltr_start_times_sorted}")
        print(f"---\t\tSorted ltr list (= x-ticks):\t{ltr_sorted}")
        print(f"---\t\tx_int =\t\t\t\t{x_int}")

    # extract header from data & create title
    header = data[ltr_sorted[-1]]['header']
    footer = f"Model: {header['Model version'][0]} | Period: {header['Start time'][0]} - {header['End time'][0]} | © MeteoSwiss"
    unit = header["Unit"][0]

    # initialise filename
    filename = f"total_scores_{parameter}_"

    # scores = params_dict[parameter] # this is a list of lists

    ############################################################################
    ##################### REGULAR SCORES PLOTTING PIPELINE #####################
    ############################################################################
    if debug:
        print(f"---\t5) plot REGULAR parameter/scores (Because, regular & categorical scores should not be mixed on the same figure.)")

    if plot_scores and plot_params:
        regular_scores = plot_scores.split(',')
        # the idx of a score, maps the score to the corresponding subplot axes instance
        for idx, score in enumerate(regular_scores):
            # if debug:
            #     print(f"--- plotting:\t{param}/{score}")

            multiplt=False

            # save filled figure & re-set necessary for next iteration
            if idx > 0 and idx%4==0:
                # add title to figure
                plt.suptitle(
                    footer,
                    horizontalalignment="center",
                    verticalalignment="top",
                    fontdict={
                        "size": 6,
                        "color": "k",
                    },
                    bbox= dict(
                        facecolor='none',
                        edgecolor='grey',
                    ),
                )
                _save_figure(output_dir=output_dir, filename=filename)
                fig, ((ax0, ax1),(ax2, ax3)) = plt.subplots(nrows=2, ncols=2, tight_layout=True, figsize=(10, 10), dpi=200)
                subplot_axes = {0:ax0, 1:ax1, 2:ax2, 3:ax3}
                # reset filename
                filename = f"total_scores_{parameter}_"

            # get ax, to add plot to
            ax = subplot_axes[idx%4]
            ax.set_xlim(x_int[0], x_int[-1])
            ax.set_ylabel(f"{score.upper()} ({unit})")

            # plot two scores on one sub-plot
            if '/' in score:
                multiplt = True
                scores = score.split('/')
                filename += f"{scores[0]}_{scores[1]}_"
                _set_ylim(param=param, score=scores[0], ax=ax, debug=debug)

                # get y0, y1 from dfs
                if debug and idx==0:
                    print(f"---\t6) collect the data corresponding to {score} from all dataframes in the data dict in y-list")
                    print(f"---\t7) plot y-list against x_int")
                y0, y1 = [], []
                for ltr in ltr_sorted:
                    y0.append(data[ltr]['df']['Total'].loc[scores[0]])
                    y1.append(data[ltr]['df']['Total'].loc[scores[1]])

                # plot y0, y1
                ax.plot(x_int, y0, color="red", linestyle="-", marker='^', fillstyle='none', label=f"{scores[0].upper()}")
                ax.plot(x_int, y1, color="k", linestyle="-", marker='D', fillstyle='none', label=f"{scores[1].upper()}")

            # plot single score on sub-plot
            if not multiplt:
                filename += f"{score}_"
                _set_ylim(param=param, score=score, ax=ax, debug=debug)

                y = []
                # extract y from different dfs
                for ltr in ltr_sorted:
                    ltr_score = data[ltr]['df']['Total'].loc[score]
                    y.append(ltr_score)

                ax.plot(x_int, y, color="k", linestyle="-", marker='D', fillstyle='none', label=f"{score.upper()}")

            # customise grid, title, xticks, legend of current ax
            _customise_ax(parameter=parameter, score=score, x_ticks=x_ticks, grid=True, ax=ax)

            # save figure, if this is the last score
            if idx == len(regular_scores)-1:
                # add title to figure
                plt.suptitle(
                    footer,
                    horizontalalignment="center",
                    verticalalignment="top",
                    fontdict={
                        "size": 6,
                        "color": "k",
                    },
                    bbox= dict(
                        facecolor='none',
                        edgecolor='grey',
                    ),
                )
                # clear empty subplots
                _clear_empty_axes(subplot_axes=subplot_axes, idx=idx)
                # save & clear figure
                _save_figure(output_dir=output_dir, filename=filename)
    ############################################################################
    ################### CATEGORICAL SCORES PLOTTING PIPELINE ###################
    ############################################################################
    # remark: include thresholds for categorical scores
    if debug:
        print(f"---\t10) repeat plotting pipeline for categorical params/scores/thresh combinations")

    print(plot_cat_params)
    print(plot_cat_scores)
    print(plot_cat_thresh)
    print(plot_cat_params and plot_cat_scores and plot_cat_thresh)

    if plot_cat_params and plot_cat_scores and plot_cat_thresh:
        print(f"--- should now create total scores plots for all cat params/scores")
        cat_params = plot_cat_params.split(',')  # categorical parameters: TOT_PREC12,TOT_PREC6,TOT_PREC1,CLCT,T_2M,T_2M_KAL,TD_2M,TD_2M_KAL,FF_10M,FF_10M_KAL,VMAX_10M6,VMAX_10M1
        cat_scores = plot_cat_scores.split(',')  # categorical scores: FBI,MF,POD,FAR,THS,ETS
        cat_threshs = plot_cat_thresh.split(':') # categorical thresholds: 0.1,1,10:0.2,1,5:0.2,0.5,2:2.5,6.5:0,15,25:0,15,25:-5,5,15:-5,5,15:2.5,5,10:2.5,5,10:5,12.5,20:5,12.5,20
        cat_params_dict = {cat_param: [] for cat_param in cat_params}
        for param, threshs in zip(cat_params, cat_threshs):
            # first append all scores w/o thresholds to parameter
            for score in plot_scores.split(','):
                if '/' in score:
                    cat_params_dict[param].append(score.split('/'))
                else:
                    cat_params_dict[param].append([score])

            # afterwards append all scores that have a threshold in their name to current to parameter
            thresholds = threshs.split(',')
            for threshold in thresholds:
                for score in cat_scores:
                    if '/' in score:
                        cat_params_dict[param].append([x+f"({threshold})" for x in score.split('/')])

                    else:
                        cat_params_dict[param].append([f"{score}({threshold})"])

        if True:
            print("Categorical Parameter Dict: ")
            pprint(cat_params_dict)

    # TODO: implement the total scores pipeline for categorical scores as well

    ############################################################################
    #################### ENSEMBLE SCORES PLOTTING PIPELINE #####################
    ############################################################################
    # remark: include thresholds for categorical scores
    print(f"---\t11) repeat plotting pipeline for ensemble params/scores/thresh combinations")
    # TODO: implement the total scores pipeline for categorical scores as well

    ############################################################################

    return


from functools import reduce
from random import randint

class Packet():
	def __init__(self, stickers):
		self.stickers = stickers

	@classmethod
	def fromRandomSelection(cls, book):
		stickers = set()
		while len(stickers) < book.packetSize:
			s = randint(1, book.collectionSize)
			stickers.add(s)
		return cls(stickers)

	@classmethod
	def fromSpecifiedStickers(cls, book, stickers):
		return cls(stickers)

class StickerBook():
	def __init__(self, name, collectionSize, packetSize):
		self.name = name
		self.collectionSize = collectionSize
		self.packetSize = packetSize

	def __str__(self):
		return "Sticker Book {} with {} stickers in packets of {}".format(self.name, self.collectionSize, self.packetSize)

	def printPacket(self):
		return Packet.fromRandomSelection(self)

class StickerCollection():
	def __init__(self, book, collectionId=None, observer=None):
		self.collectionId = collectionId
		self.book = book
		self.observer=observer
		self.stickers = dict()
		self.packetsBought = 0

	def __str__(self):
		return "Packets Bought: {}, Stickers: {}".format(self.packetsBought, self.stickers)

	def receivePacket(self, packet):
		self.packetsBought += 1
		for s in packet.stickers:
			self.receiveSticker(s)

	def receiveSticker(self, whichSticker):
		if whichSticker in self.stickers:
			self.stickers[whichSticker] += 1
		else:
			self.stickers[whichSticker] = 1
		if self.observer:
			self.observer(self.collectionId, whichSticker, self.stickers[whichSticker])

	def giveAwaySticker(self, whichSticker, recipient):
		if (whichSticker in self.stickers):
			self.stickers[whichSticker] -= 1
			if (self.stickers[whichSticker] == 0):
				del(self.stickers[whichSticker])
			recipient.receiveSticker(whichSticker)
			return True
		return False

	def getSwaps(self):
		swaps = set(filter(lambda x: self.stickers[x] > 1, self.stickers.keys()))
		return swaps

	def isComplete(self):
		return len(self.stickers) == self.book.collectionSize

class GroupOfCollections():
	def __init__(self, book, collections, swapFunction):
		self.book = book
		self.collections = collections
		self.swapsMade = 0
		self.swapFunction = swapFunction

	@classmethod
	def withBookCountAndSwapFunction(cls, book, numberCollections, swapFunction):
		collections = []
		for x in range(numberCollections):
			c = StickerCollection(collectionId=x, book=book)
			collections.append(c)

		return cls(book, collections, swapFunction)

	@classmethod
	def withBookCollectionsAndSwapFunction(cls, book, collections, swapFunction):
		return cls(book, collections, swapFunction)

	def buyPackets(self):
		for c in self.collections:
			if not c.isComplete():
				packet = self.book.printPacket()
				c.receivePacket(packet)

	def runSwaps(self):
		for c in self.collections:
			for d in self.collections:
				if (c is not d):
					self.swapsMade += self.swapFunction(c, d)

	def getPacketsBought(self):
		packetsBought = map(lambda x: x.packetsBought, self.collections)
		total = reduce((lambda x, y: x + y), packetsBought)
		return total

	def isComplete(self):
		for c in self.collections:
			if not c.isComplete():
				return False
		return True

class SwapFunctions():
	# Returns all the swap functions in a dictionary, each function is given a name.
	@staticmethod
	def all():
		allFunctions = []
		allFunctions.append(SwapFunctions.noSwap)
		allFunctions.append(SwapFunctions.transactionalSwap)
		allFunctions.append(SwapFunctions.generousSwap)
		return allFunctions

	# Given two collectors, this function works out which ones each needs that the other has as a swap
	# It returns both sets, which can be unpacked by the caller
	@staticmethod
	def getRespectiveNeeds(first, second):
		# Get the list of swaps
		firstSwaps = first.getSwaps()
		secondSwaps = second.getSwaps()

		# Compose a set of stickers that can be swapped
		firstHasSecondNeeds = set(filter(lambda x: x not in second.stickers, firstSwaps))
		secondHasFirstNeeds = set(filter(lambda x: x not in first.stickers, secondSwaps))

		return firstHasSecondNeeds, secondHasFirstNeeds

	# A swap handler which simply doesn't do swapping, used as a 'control' in measurements
	@staticmethod
	def noSwap(first, second):
		return 0

	# This method works out what both collectors need, and what swaps are available.
	# It then swaps an equal number of stickers from first to second and vice versa
	# This is transactional because a collector only gives with the expectation of receiving
	@staticmethod
	def transactionalSwap(first, second):
		swapsMade = 0

		# Check that both people still need stickers, quit early
		if first.isComplete() or second.isComplete():
			return swapsMade

		# Work out what everyone needs
		firstHasSecondNeeds, secondHasFirstNeeds = SwapFunctions.getRespectiveNeeds(first, second)

		# While there are stickers left in both piles
		while (len(firstHasSecondNeeds) > 0) and (len(secondHasFirstNeeds) > 0):
			# Pick one of mine and theirs to swap
			fromFirst = firstHasSecondNeeds.pop()
			fromSecond = secondHasFirstNeeds.pop()
			swapsMade += 1

			first.giveAwaySticker(fromFirst, second)
			second.giveAwaySticker(fromSecond, first)
			swapsMade += 1

		return swapsMade

	# In this method, when two people are swapping, they pay no attention to how many they receive compared to how many they give.
	# This is 'generous' because each collector seeks to help the other with their swaps regardless of what they get back
	@staticmethod
	def generousSwap(first, second):
		swapsMade = 0

		# Check that at least one of the collectors needs more stickers
		if first.isComplete() and second.isComplete():
			return swapsMade

		# Work out what everyone needs
		firstHasSecondNeeds, secondHasFirstNeeds = SwapFunctions.getRespectiveNeeds(first, second)

		# First gives all required swaps to second
		for fromFirst in firstHasSecondNeeds:
			first.giveAwaySticker(fromFirst, second)
			swapsMade += 1

		# Second gives all required swaps to first
		for fromSecond in secondHasFirstNeeds:
			second.giveAwaySticker(fromSecond, first)
			swapsMade += 1

		return swapsMade


import numpy as np
import tensorflow as tf

from pdb import set_trace

log_2_e = 1.44269504089  # Constant to convert to binary entropies
epsilon = 1e-12

class MetricsTracker(object):

	def __init__(self):
		self.total_samples = 0
		self.b_ce = tf.keras.metrics.BinaryCrossentropy()
		self.b_acc = tf.keras.metrics.BinaryAccuracy()
		self.preds = np.array([])
		self.flush()

	def flush(self, flush_totals=False):
		if flush_totals:
			self.total_samples = 0
		self.preds = np.array([])
		self.b_acc.reset_states()
		self.b_ce.reset_states()

	def add_observation(self, targets, predictions, loss):
		# Compute overall statistics, gathering types and predictions accordingly
		num_samples = targets.shape[0]
		self.total_samples += int(num_samples)
		self.b_acc.update_state(targets, predictions)
		self.b_ce.update_state(targets, predictions)
		self.preds = np.append(self.preds, predictions.numpy())

	def get_stats(self):
		return self.total_samples, "{0:.3f}".format(self.b_ce.result() * log_2_e), "{0:.2%}".format(
			self.b_acc.result())

	def get_acc(self):
		return self.b_acc.result()

	def get_bce(self):
		return self.b_ce.result() * log_2_e

if __name__ == '__main__':
	import pickle
	import matplotlib.pyplot as plt
	import numpy as np
	from sklearn.metrics import auc, accuracy_score

	y_preds, y_true = [], []
	with open('../data/valid_shared_transformer.pkl', 'rb') as f:
		data = pickle.load(f)
		y_preds, y_true = data
	convert = lambda x: 1 if x > .50 else 0

	print(accuracy_score(y_true, [convert(x) for x in y_preds]))
	""" if aligned and say is aligned
		precision tp / (tp + fp)
		recall tp / (tp + fn)
	"""
	plt.subplots(2, 2)
	data = []
	roc = []
	ys = sorted(list(zip(y_true, y_preds)), key=lambda x: x[1])
	for threshold in np.arange(min(y_preds), max(y_preds), .01):
		fn, tp, fp, tn = 0, 0, 0, 0
		for t, p in ys:
			if t == 1.0:
				if p >= threshold:
					tp += 1
				else:
					fn += 1
			if t == 0.0:
				if p >= threshold:
					fp += 1
				else:
					tn += 1

		recall = tp / (tp + fn)
		precision = tp / (tp + fp)
		tpr = recall
		fpr = fp / (fp + tn)
		data.append((precision, recall, threshold))
		roc.append((tpr, fpr, threshold))

	recalls = [x[1] for x in data]
	precisions = [x[0] for x in data]

	plt.subplot(2, 2, 1)
	plt.plot(recalls, precisions, marker='.')
	plt.xlabel('Recall', fontsize='small')
	plt.ylabel('Precision', fontsize='small')
	plt.title('Aligned and Is Aligned', fontdict={'fontsize': 7})

	tprs = [x[0] for x in roc]
	fprs = [x[1] for x in roc]
	p = plt.subplot(2, 2, 2)
	plt.plot(fprs, tprs, marker='.')
	plt.xlabel('False Positive Rate ' + 'AUC {:.3f}'.format(auc(fprs, tprs)), fontsize='small')
	plt.ylabel('True Positive Rate', fontsize='small')
	plt.title('Aligned and Is Aligned', fontdict={'fontsize': 7})

	""" if not aligned and say is not aligned
		precision tp / (tp + fp)
		recall tp / (tp + fn)
	"""
	data = []
	roc = []
	ys = sorted(list(zip(y_true, y_preds)), key=lambda x: x[1])
	for threshold in np.arange(min(y_preds) + .01, max(y_preds), .01):
		fn, tp, fp, tn = 0, 0, 0, 0
		for t, p in ys:
			if t == 0.0:
				if p < threshold:
					tp += 1
				else:
					fn += 1
			if t == 1.0:
				if p < threshold:
					fp += 1
				else:
					tn += 1

		recall = tp / (tp + fn)
		precision = tp / (tp + fp)
		tpr = recall
		fpr = fp / (fp + tn)
		data.append((precision, recall, threshold))
		roc.append((tpr, fpr, threshold))

	recalls = [x[1] for x in data]
	precisions = [x[0] for x in data]
	plt.subplot(2, 2, 3)
	plt.plot(recalls, precisions, marker='.')
	plt.xlabel('Recall', fontsize='small')
	plt.ylabel('Precision', fontsize='small')
	plt.title('Not Aligned and Is Not Aligned', fontdict={'fontsize': 7})

	tprs = [x[0] for x in roc]
	fprs = [x[1] for x in roc]
	plt.subplot(2, 2, 4)
	plt.plot(fprs, tprs, marker='.')
	plt.xlabel('False Positive Rate ' + 'AUC {:.3f}'.format(auc(fprs, tprs)), fontsize='small')

	plt.ylabel('True Positive Rate', fontsize='small')

	plt.title('Not Aligned and Is Not Aligned', fontdict={'fontsize': 7})
	plt.tight_layout()
	plt.show()


from typing import List
from collections import defaultdict

class Solution:
    def leastBricks(self, wall: List[List[int]]) -> int:

        h = defaultdict(int)
        m = len(wall)
        for i in range(m):
            s = 0
            for j in range(len(wall[i]) - 1):

                s += wall[i][j]
                h[s] += 1

        return m - max(h.values() or [0])


from inspect import signature, getargspec
from dash.dependencies import Input
from dash.exceptions import PreventUpdate
from dash import callback_context

from callback_structs import *
from visualiser import Visualiser
from abstract_dash import AbstractDash

assets_ignore = '.*bootstrap.*'
class DashBoard(AbstractDash):
    def __init__(self, name):
        super().__init__(Visualiser(), name, assets_ignore=assets_ignore)
        self._build_app()

    def run(self):
        self.app.run_server(debug=True)

    def _build_app(self):
        # Add Options
        form_elements, identifiers = self._create_form_elements(self.visualiser, id_prefix=id_prefix)
        update_i.update(identifiers)

        gns = [{"label": c, "value": c} for c in []]
        inp = (self.create_dropdown(load_s["gns"].component_id, gns, multi=True,placeholder="Load Design") +
               self.create_line_break(10) + self.create_button(load_i.component_id,"Submit"))
        acc_elements = [("Load Design", inp)]
        load_accordion = self.create_accordion("proj_accordion", acc_elements)

        form_div = self.create_div(graph_type_o["id"].component_id, form_elements)
        options = self.create_sidebar(not_modifier_identifiers["sidebar_id"], "Options", form_div, className="col sidebar")
        figure = self.visualiser.empty_graph(graph_id)
        graph = self.create_div(update_o["graph_id"].component_id, [figure], className="col")
        graph = self.create_div(load_o.component_id, graph)
        legend = self.create_div(update_o["legend_id"].component_id,[], className="col sidebar")
        elements = options+graph+legend
        container = self.create_div("row-main", elements, className="row flex-nowrap no-gutters")
        self.app.layout = self.create_div("main", load_accordion+container, className="container-fluid")[0]

        # Bind the callbacks
        def load_inner(click,gns):
            return self.load(click,gns)

        def update_graph_inner(*args):
            return self.update_graph(args)

        def export_img_inner(*args):
            return self.export_graph_img(*args)

        self.add_callback(load_inner, [load_i], [load_o],load_s.values())
        self.add_callback(update_graph_inner, update_i.values(), update_o.values())
        self.add_callback(export_img_inner,export_img_i, export_img_o)
        self.build()

    def load(self,click,gns):
        if not gns or not isinstance(self.visualiser, Visualiser):
            raise PreventUpdate()
        ctx = callback_context
        if ctx.triggered:
            self.visualiser.set_design_names(gns)
            figure = self.visualiser.build(graph_id=graph_id)
            d = self.create_div(update_o["graph_id"].component_id, figure, className="col")
            return d
        else:
            raise PreventUpdate()

    def update_graph(self, *args):
        if not isinstance(self.visualiser, Visualiser):
            raise PreventUpdate()
        args = args[0]
        for index, setter_str in enumerate(args):
            if setter_str is not None:
                try:
                    setter = getattr(self.visualiser, setter_str, None)
                    parameter = None
                except TypeError:
                    # Must be a input element rather than a checkbox.
                    # With annonymous implementation this is tough.
                    to_call = list(update_i.keys())[index]
                    parameter = setter_str
                    setter = getattr(self.visualiser, to_call, None)
                if setter is not None:
                    try:
                        if parameter is not None and len(getargspec(setter).args) > 1:
                            setter(parameter)
                        else:
                            setter()
                    except Exception as ex:
                        print(ex)
                        raise PreventUpdate()
        try:
            figure, legend = self.visualiser.build(graph_id=graph_id, legend=True)
            legend = self.create_legend(legend)
            return [figure], legend
        except Exception as ex:
            print(ex)
            raise PreventUpdate()

    def export_graph_img(self, get_jpg_clicks, get_png_clicks, get_svg_clicks):
        action = 'store'
        input_id = None
        ctx = callback_context
        if ctx.triggered:
            input_id = ctx.triggered[0]["prop_id"].split(".")[0]
            if input_id != "tabs":
                action = "download"
        else:
            raise PreventUpdate()
        return [{'type': input_id, 'action': action}]

    def _create_form_elements(self, visualiser, style={}, id_prefix=""):
        default_options = [visualiser.set_full_graph_view,
                           visualiser.set_concentric_layout,
                           visualiser.add_node_no_labels,
                           visualiser.add_edge_no_labels,
                           visualiser.add_standard_node_color,
                           visualiser.add_standard_edge_color]

        options = self._generate_options(visualiser)
        removal_words = ["Add", "Set"]
        elements = []
        identifiers = {}
        for k, v in options.items():
            name = self._beautify_name(k)
            identifier = id_prefix + "_" + k
            element = []

            if isinstance(v, (int, float)):
                min_v = int(v/1.7)
                max_v = int(v*1.7)
                default_val = int((min_v + max_v) / 2)
                step = 1

                element += (self.create_heading_6("", name) +
                            self.create_slider(identifier, min_v, max_v, default_val=default_val, step=step))
                identifiers[k] = Input(identifier, "value")

            elif isinstance(v, dict):
                removal_words = removal_words + \
                    [word for word in name.split(" ")]
                inputs = []
                default_button = None
                for k1, v1 in v.items():
                    label = self._beautify_name(k1)
                    label = "".join(
                        "" if i in removal_words else i + " " for i in label.split())
                    inputs.append({"label": label, "value": k1})
                    if v1 in default_options:
                        default_button = k1

                element = (self.create_heading_6(k, name) +
                           self.create_radio_item(identifier, inputs, value=default_button))
                identifiers[k] = Input(identifier, "value")

            breaker = self.create_horizontal_row(False)
            elements = elements + \
                self.create_div(identifier + "_contamanual_toolbariner",
                                element, style=style)
            elements = elements + breaker

        exports = self.create_heading_4("export_img_heading", "Image Export")
        for e_input in export_img_i:
            exports += self.create_button(e_input.component_id,
                                          className="export_img_button")
            exports += self.create_line_break()
        export_div = self.create_div("export_data_container", exports, style=style)

        return (elements + export_div, identifiers)

    def _beautify_name(self, name):
        name_parts = name.split("_")
        name = "".join([p.capitalize() + " " for p in name_parts])
        return name

    def _generate_options(self, visualiser):
        blacklist_functions = ["empty_graph",
                               "build",
                               "mode",
                               "view",
                               "node_size",
                               "edge_color",
                               "node_color",
                               "copy_settings",
                               "get_design_names",
                               "edge_shape",
                               "edge_text",
                               "node_shape",
                               "node_text",
                               "set_design_names",
                               "get_load_predicates",
                               "get_loaded_design_names"]

        options = {"view": {},
                   "layout": {}}

        for func_str in dir(visualiser):
            if func_str[0] == "_":
                continue
            func = getattr(visualiser, func_str, None)

            if func is None or func_str in blacklist_functions or not callable(func):
                continue

            if len(signature(func).parameters) > 0:
                # When there is parameters a slider will be used.
                # Some Paramterised setters will return there default val if one isnt provided.
                default_val = func()
                if default_val is None:
                    default_val = 1
                options[func_str] = default_val
            else:
                # When no params radiobox.
                if func_str.split("_")[-1] == "preset":
                    option_name = "preset"

                elif func_str.split("_")[-1] == "view":
                    option_name = "view"

                elif func_str.split("_")[-1] == "mode":
                    option_name = "mode"

                elif func_str.split("_")[-1] == "layout":
                    option_name = "layout"

                elif func_str.split("_")[-1] == "connect":
                    option_name = "connect"

                elif "node" in func_str:
                    option_name = "node" + "_" + func_str.split("_")[-1]

                elif "edge" in func_str:
                    option_name = "edge" + "_" + func_str.split("_")[-1]
                elif func_str.split("_")[-1] == "level":
                    option_name = "detail"
                else:
                    option_name = "misc"

                if option_name not in options.keys():
                    options[option_name] = {func_str: func}
                else:
                    options[option_name][func_str] = func
        return options


from scrapy import Spider
from scrapy.selector import HtmlXPathSelector
from imdb.items import IMDBItem, Field

class IMDBSpider(Spider):
    name = 'imdb'
    allowed_domains = ["imdb.com"]
    start_urls = [
        "https://www.imdb.com/chart/top"
    ]

    def parse(self, response):
        hxs = HtmlXPathSelector(response)
        sites = hxs.select('//tbody[@class="lister-list"]/tr')
        items = []
        for site in sites:
            item = IMDBItem()
            item['title'] = site.select('td[@class="titleColumn"]/a/text()').extract()
            item['year'] = site.select('td[@class="titleColumn"]/span/text()').extract()
            item['rating'] = site.select('td[@class="ratingColumn imdbRating"]/strong/text()').extract()
            items.append(item)
        return items


import os, csv
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker

engine = create_engine(os.getenv("DATABASE_URL"))
db = scoped_session(sessionmaker(bind=engine))

file = open("/Dataset/books.csv")
reader = csv.reader(file)
for isbn, title, author, year in reader:

    db.execute("INSERT INTO books (isbn, title, author, year) VALUES (:isbn, :title, :author, :year)",
                {"isbn": isbn,
                 "title": title,
                 "author": author,
                 "year": year})

    print(f"Added book {title} to database.")
    db.commit()

return render_template("/HTML/BookMain.html")


import os
import webbrowser
from flask import Flask, flash, request, redirect, url_for, send_from_directory
from werkzeug.utils import secure_filename

UPLOAD_FOLDER = 'assets'
ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg', 'gif', 'mp4', 'mov', 'heic'}

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

def allowed_file(filename):
    return '.' in filename and \
        filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

@app.route('/', methods=['GET', 'POST'])
def upload_file():
    if request.method == 'POST':
        # check if the post request has the file part
        if 'file' not in request.files:
            flash('No file part')
            return redirect(request.url)

        file = request.files['file']
        # If the user does not select a file, the browser submits an
        # empty file without a filename.
        if file.filename == '':
            flash('No selected file')
            return redirect(request.url)

        if file and allowed_file(file.filename):
            filename = secure_filename(file.filename)
            file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
            url = f"{UPLOAD_FOLDER}/{filename}"
            print(url)
            webbrowser.open_new(f"file://{os.getcwd()}/{url}")
            return redirect(url_for('download_file', name=filename))

    html = '''
    <!doctype html>
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Upload new File</title>

Upload new File

    <form method=post enctype=multipart/form-data>
      <input type=file name=file>
      <input type=submit value=Upload>
    </form>
    '''
    return html

@app.route('/uploads/<name>')
def download_file(name):
    print(f"File received: {name}")
    return send_from_directory(app.config["UPLOAD_FOLDER"], name)


import importlib
import inspect
from typing import Any, List, Type

def get_subclass_types_from_module(module_path, type: Type) -> List:
    """Retrieve a list of types that are subclass of the other type parameter.

    Args:
        module_path (str): The absolute path to the module
        type (Type): The type that this class should be a subclass of.

    Returns:
        List: A list of types that derive from the subclass.
    """
    module_instance = importlib.import_module(module_path)
    return [obj for name, obj in inspect.getmembers(module_instance) if inspect.isclass(obj) and issubclass(obj, type)]

def create_type_from_module_path(module_type_path: str, **kwargs) -> Any:
    """Instantiate the exporter from the type definition as a string

    Args:
        module_type_path (str): The fully qualified definition for the exporter type.

    Raises:
        ValueError: If the exporter type was not defined.
        ValueError: If it's not a fully qualified type declaration.
        ValueError: If the module instane is invalid.
        ValueError: If the exporter interface type was not retrievable from the module instance.

    Returns:
        ExporterInterface: The instantiated exporter.
    """
    if module_type_path is None:
        raise ValueError("The exporter type is invalid or null")

    if "." not in module_type_path:
        raise ValueError(
            "Full type definition is required for the exporter in order to create.")

    module_path_expanded = module_type_path.split('.')
    type_module = module_path_expanded[:-1]
    type_class_name = module_path_expanded[-1]

    if module_path_expanded is None:
        raise ValueError("Failed to expand the exporter type definition")

    type_module: str = '.'.join(module_type_path.split('.')[-1])
    type_module_instance = importlib.import_module(type_module)

    if not type_module_instance:
        raise ValueError("The module was not found")

    module_type = getattr(type_module_instance, type_class_name)
    if not module_type:
        raise ValueError("Failed to retrieve the interface type")

    module_type_instance = module_type(**kwargs)
    if module_type_instance is None:
        raise ValueError("The exporter instance is invalid or null")

    return module_type_instance


from asg.intermediate_lang import *
from asg.entities import *
from lark import Lark, Transformer, Tree, Token
import os
from asg.grammar import *

class XSchemSymbolTransformer(Transformer):
    """
    Transform the parsed tree to an SExpressionList object. Also handles strings and literals.
    """

    def __init__(self, name, raw_data):
        super().__init__()
        self.name = name
        self.raw_data = raw_data

    def string(self, string: List[Token]) -> str:
        if len(string) > 0:
            return string[0].value
        else:
            return ""

    def literal(self, literal: List[Token]) -> Literal:
        return Literal(literal[0].value)

    def number(self, num: List[Token]) -> float or int:
        if "." in num[0].value:
            return float(num[0].value)
        else:
            return int(num[0].value)

    def value(self, value: List[Token]):
        if type(value[0]) == Token:
            return value[0].value
        else:
            return value[0]

    def key(self, value: List[Token]):
        if type(value[0]) == Token:
            return value[0].value
        else:
            return value[0]

    def key_value_pair(self, pair):
        return tuple(pair)

    def block(self, block: List[Tree or Token]):
        res = []
        for block_member in block:
            if type(block_member) == Tree:
                res += block_member.children
        return res

    def file(self, file: List[Tree]):
        pin_locations = {}
        inputs = []
        outputs = []

        ul_corner = Point(0, 0)
        lr_corner = Point(0, 0)
        for statement in file:
            if type(statement) == Tree:
                if type(statement.children[0]) == Tree:
                    if statement.children[0].data == "rectangle":
                        body = statement.children[1].children
                        properties = body[-1]
                        properties_dict = {}
                        for prop in properties:
                            if type(prop) == tuple:
                                properties_dict[prop[0].value] = prop[1].value

                        # Skip rectangles that don't look like pins
                        if (
                            "name" not in properties_dict
                            or "dir" not in properties_dict
                        ):
                            continue

                        body = body[:-1]
                        pin_corner_1 = Point(body[1], body[2])
                        pin_corner_2 = Point(body[3], body[4])
                        pin_location = (pin_corner_1 + pin_corner_2) / 2
                        pin_locations[properties_dict["name"]] = pin_location

                        if properties_dict["dir"] == "in":
                            inputs.append(properties_dict["name"])
                        if properties_dict["dir"] == "out":
                            outputs.append(properties_dict["name"])

                        # If this pin is outside the bounding box, move the box to include it
                        if pin_location.x < ul_corner.x:
                            ul_corner.x = pin_location.x
                        if pin_location.y < ul_corner.y:
                            ul_corner.y = pin_location.y
                        if pin_location.x > lr_corner.x:
                            lr_corner.x = pin_location.x
                        if pin_location.y > lr_corner.y:
                            lr_corner.y = pin_location.y

        raw_data = "\n".join(self.raw_data.split("\n")[1:])
        return LibrarySymbol(
            self.name,
            self.name,
            pin_locations,
            inputs,
            outputs,
            raw_data,
            [],
            BoundingBox(ul_corner, lr_corner),
        )

def xschem_to_il(library_dir: List[os.DirEntry]):
    """
    Convert a directory of Xschem symbol files to an internal representation
    :param library_dir: A list of directory entries for a directory which contains Xschem symbol files
    :return: A representation of the library as [LibraryComponent]s
    """

    # Get paths relative to the location of this file, not the root of the module
    script_dir = os.path.dirname(os.path.realpath(__file__))
    res = LibraryIL()
    for file in library_dir:
        if file.is_file() and os.path.splitext(file.path)[-1] == ".sym":
            symbol_name = os.path.splitext(os.path.split(file.path)[-1])[0]
            with open(file.path) as sym_file:
                with open(os.path.join(script_dir, "xschem.lark")) as grammar:
                    parser = Lark(
                        grammar.read() + "\n",
                        parser="lalr",
                        transformer=XSchemSymbolTransformer(
                            symbol_name, sym_file.read().strip()
                        ),
                        start="file",
                    )
                sym_file.seek(0)
                res.symbols[symbol_name] = parser.parse(sym_file.read())
    return res


#!/usr/bin/env python3

import argparse
import logging
import sys

import slack

logging.basicConfig(level=logging.INFO)
LOGGER = logging.getLogger(__name__)

def idb_excepthook(type, value, tb):
    """Call an interactive debugger in post-mortem mode

    If you do "sys.excepthook = idb_excepthook", then an interactive debugger
    will be spawned at an unhandled exception
    """
    if hasattr(sys, 'ps1') or not sys.stderr.isatty():
        sys.__excepthook__(type, value, tb)
    else:
        import pdb, traceback
        traceback.print_exception(type, value, tb)
        print
        pdb.pm()

@slack.RTMClient.run_on(event='message')
def handle_messages(**payload):
    """
    Does a lot. Fucking decorators.
    1. if the message has react-able strings, react to the message appropriately
    2. if the message starts with @botname and has a command i've planned for, do thing.
    """
    data = payload['data']
    channel_id = data['channel']
    thread_ts = data['ts']

    LOGGER.debug(payload)

    if not data['text'].startswith("MOONRITUAL"):
        return
    response = "ðŸ§ž"

    webclient = payload['web_client']
    webclient.chat_postMessage(
        channel=channel_id,
        text=response,
        timestamp=thread_ts
    )

def main(*args, **kwargs):
    parser = argparse.ArgumentParser(
        description="The Dicussion Archiver")
    parser.add_argument(
        "--debug", "-d", action='store_true',
        help="Include debugging output")
    parser.add_argument("--verbose", "-v", action="store_true", help="Include verbose logs")
    parser.add_argument(
        "--api-token", "-t", required=True, help="Slack API token")
    parsed = parser.parse_args()
    if parsed.verbose or parsed.debug:
        LOGGER.setLevel(logging.DEBUG)
    if parsed.debug:
        sys.excepthook = idb_excepthook

    rtm_client = slack.RTMClient(token=parsed.api_token)
    rtm_client.start()

if __name__ == '__main__':
    sys.exit(main(*sys.argv))


from rest_framework import status
from django.test import TestCase
from rest_framework.test import APIClient
from ..serializers import ClimateChangeInfo, ClimateChangeInfoSerializer
from ..views import *
import random

# initialize the APIClient app
client = APIClient()

class GetClimateChangeDataTest(TestCase):
    """ Test module for GET Weather API """

    def test_get_valid_single_response(self):
        limit = random.randint(1, 10)
        offset = random.randint(1, 10)
        Region = ['UK', 'England', 'Wales', 'Scotland', 'Northern_Ireland', 'England_and_Wales', 'England_N', 'England_S', 'Scotland_N', 'Scotland_E', 'Scotland_W', 'England_E_and_NE', 'England_NW_and_N_Wales', 'Midlands', 'East_Anglia', 'England_SW_and_S_Wales', 'England_SE_and_Central_S']
        Parameter = ['Tmax', 'Tmin', 'Tmean', 'Sunshine', 'Rainfall', 'Raindays1mm', 'AirFrost']
        region = random.choice(Region)
        parameter = random.choice(Parameter)
        response = client.get('/api/get-weather-data/', data = {'limit' : limit, 'offset' : offset, 'region': region, 'parameter': parameter}, format='json')
        response_new = parse_weather_data(region, parameter)
        data_points = response_new['data_points']
        climate_change_info_list = []
        for data in data_points:
            climate_change_object = ClimateChangeInfo(
                year = data[0],
                jan = data[1],
                feb = data[2],
                mar = data[3],
                apr = data[4],
                may = data[5],
                jun = data[6],
                jul = data[7],
                aug = data[8],
                sep = data[9],
                oct = data[10],
                nov = data[11],
                dec = data[12],
                win = data[13],
                spr = data[14],
                sum = data[15],
                aut = data[16],
                ann = data[17]
            )
            climate_change_info_list.append(climate_change_object)
        # the many param informs the serializer that it will be serializing more than a single climate change object.
        serializer = ClimateChangeInfoSerializer(climate_change_info_list, many=True)
        self.assertEqual(response.data['results'], serializer.data[offset:offset+limit])
        self.assertEqual(response.status_code, status.HTTP_200_OK)

    def test_get_invalid_single_response(self):
        limit = random.randint(1, 10)
        offset = random.randint(1, 10)
        response = client.get('/api/get-weather-data/', data = {'limit' : limit, 'offset' : offset, 'region': 'India', 'parameter': 'Rainfall'}, format='json')
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)

    def test_invalid_request_method(self):
        response = client.post('/api/get-weather-data/', data = {'limit' : 10, 'offset' : 10, 'region': 'England', 'parameter': 'Rainfall'}, format='json')
        self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)


import numpy as np
import os
import isceobj
import datetime
import logging

logger = logging.getLogger('isce.topsinsar.topo')

def runTopo(self):

    hasGPU= self.useGPU and self._insar.hasGPU()
    if hasGPU:
        runTopoGPU(self)
    else:
        runTopoCPU(self)

def runTopoCPU(self):
    from zerodop.topozero import createTopozero
    from isceobj.Planet.Planet import Planet

    swathList = self._insar.getValidSwathList(self.swaths)

    ####Catalog for logging
    catalog = isceobj.Catalog.createCatalog(self._insar.procDoc.name)

    ####Load in DEM
    demfilename = self.verifyDEM()
    catalog.addItem('Dem Used', demfilename, 'topo')

    boxes = []
    for swath in swathList:
        #####Load the reference product
        reference = self._insar.loadProduct( os.path.join(self._insar.referenceSlcProduct,  'IW{0}.xml'.format(swath)))

        numCommon  = self._insar.numberOfCommonBursts[swath-1]
        startIndex = self._insar.commonBurstStartReferenceIndex[swath-1]

        if numCommon > 0:
            catalog.addItem('Number of common bursts IW-{0}'.format(swath), self._insar.numberOfCommonBursts[swath-1], 'topo')

            ###Check if geometry directory already exists.
            dirname = os.path.join(self._insar.geometryDirname, 'IW{0}'.format(swath))
            os.makedirs(dirname, exist_ok=True)

            ###For each burst
            for index in range(numCommon):
                ind = index + startIndex
                burst = reference.bursts[ind]

                latname = os.path.join(dirname, 'lat_%02d.rdr'%(ind+1))
                lonname = os.path.join(dirname, 'lon_%02d.rdr'%(ind+1))
                hgtname = os.path.join(dirname, 'hgt_%02d.rdr'%(ind+1))
                losname = os.path.join(dirname, 'los_%02d.rdr'%(ind+1))

                demImage = isceobj.createDemImage()
                demImage.load(demfilename + '.xml')

                #####Run Topo
                planet = Planet(pname='Earth')
                topo = createTopozero()
                topo.slantRangePixelSpacing = burst.rangePixelSize
                topo.prf = 1.0/burst.azimuthTimeInterval
                topo.radarWavelength = burst.radarWavelength
                topo.orbit = burst.orbit
                topo.width = burst.numberOfSamples
                topo.length = burst.numberOfLines
                topo.wireInputPort(name='dem', object=demImage)
                topo.wireInputPort(name='planet', object=planet)
                topo.numberRangeLooks = 1
                topo.numberAzimuthLooks = 1
                topo.lookSide = -1
                topo.sensingStart = burst.sensingStart
                topo.rangeFirstSample = burst.startingRange
                topo.demInterpolationMethod='BIQUINTIC'
                topo.latFilename = latname
                topo.lonFilename = lonname
                topo.heightFilename = hgtname
                topo.losFilename = losname
                topo.topo()

                bbox = [topo.minimumLatitude, topo.maximumLatitude, topo.minimumLongitude, topo.maximumLongitude]
                boxes.append(bbox)

                catalog.addItem('Number of lines for burst {0} - IW-{1}'.format(index,swath), burst.numberOfLines, 'topo')
                catalog.addItem('Number of pixels for bursts {0} - IW-{1}'.format(index,swath), burst.numberOfSamples, 'topo')
                catalog.addItem('Bounding box for burst {0} - IW-{1}'.format(index,swath), bbox, 'topo')

        else:
            print('Skipping Processing for Swath {0}'.format(swath))

        topo = None

    boxes = np.array(boxes)
    bbox = [np.min(boxes[:,0]), np.max(boxes[:,1]), np.min(boxes[:,2]), np.max(boxes[:,3])]
    catalog.addItem('Overall bounding box', bbox, 'topo')

    catalog.printToLog(logger, "runTopo")
    self._insar.procDoc.addAllFromCatalog(catalog)

    return

def runTopoGPU(self):
    '''
    Try with GPU module.
    '''

    from isceobj.Planet.Planet import Planet
    from zerodop.GPUtopozero.GPUtopozero import PyTopozero
    from isceobj import Constants as CN
    from isceobj.Util.Poly2D import Poly2D
    from iscesys import DateTimeUtil as DTU

    swathList = self._insar.getValidSwathList(self.swaths)

    ####Catalog for logging
    catalog = isceobj.Catalog.createCatalog(self._insar.procDoc.name)

    ####Load in DEM
    demfilename = self.verifyDEM()
    catalog.addItem('Dem Used', demfilename, 'topo')

    frames = []
    swaths = []
    swathStarts = []

    for swath in swathList:
        #####Load the reference product
        reference = self._insar.loadProduct( os.path.join(self._insar.referenceSlcProduct,  'IW{0}.xml'.format(swath)))

        numCommon  = self._insar.numberOfCommonBursts[swath-1]
        startIndex = self._insar.commonBurstStartReferenceIndex[swath-1]

        if numCommon > 0:
            catalog.addItem('Number of common bursts IW-{0}'.format(swath), self._insar.numberOfCommonBursts[swath-1], 'topo')

            reference.bursts = reference.bursts[startIndex:startIndex+numCommon]
            reference.numberOfBursts = numCommon

            frames.append(reference)
            swaths.append(swath)
            swathStarts.append(startIndex)

    if len(frames) == 0:
        raise Exception('There is no common region between the two dates to process')

    topSwath = min(frames, key=lambda x: x.sensingStart)
    leftSwath = min(frames, key=lambda x: x.startingRange)
    bottomSwath = max(frames, key=lambda x: x.sensingStop)
    rightSwath = max(frames, key=lambda x: x.farRange)

    r0 = leftSwath.startingRange
    rmax = rightSwath.farRange
    dr = frames[0].bursts[0].rangePixelSize
    t0 = topSwath.sensingStart
    tmax = bottomSwath.sensingStop
    dt = frames[0].bursts[0].azimuthTimeInterval
    wvl = frames[0].bursts[0].radarWavelength
    width = int(np.round((rmax-r0)/dr) + 1)
    lgth = int(np.round((tmax-t0).total_seconds()/dt) + 1)

    polyDoppler = Poly2D(name='topsApp_dopplerPoly')
    polyDoppler.setWidth(width)
    polyDoppler.setLength(lgth)
    polyDoppler.setNormRange(1.0)
    polyDoppler.setNormAzimuth(1.0)
    polyDoppler.setMeanRange(0.0)
    polyDoppler.setMeanAzimuth(0.0)
    polyDoppler.initPoly(rangeOrder=0,azimuthOrder=0, coeffs=[[0.]])
    polyDoppler.createPoly2D()

    slantRangeImage = Poly2D()
    slantRangeImage.setWidth(width)
    slantRangeImage.setLength(lgth)
    slantRangeImage.setNormRange(1.0)
    slantRangeImage.setNormAzimuth(1.0)
    slantRangeImage.setMeanRange(0.)
    slantRangeImage.setMeanAzimuth(0.)
    slantRangeImage.initPoly(rangeOrder=1,azimuthOrder=0,coeffs=[[r0,dr]])
    slantRangeImage.createPoly2D()

    dirname = self._insar.geometryDirname
    os.makedirs(dirname, exist_ok=True)

    latImage = isceobj.createImage()
    latImage.initImage(os.path.join(dirname, 'lat.rdr'), 'write', width, 'DOUBLE')
    latImage.createImage()

    lonImage = isceobj.createImage()
    lonImage.initImage(os.path.join(dirname, 'lon.rdr'), 'write', width, 'DOUBLE')
    lonImage.createImage()

    losImage = isceobj.createImage()
    losImage.initImage(os.path.join(dirname, 'los.rdr'), 'write', width, 'FLOAT', bands=2, scheme='BIL')
    losImage.setCaster('write', 'DOUBLE')
    losImage.createImage()

    heightImage = isceobj.createImage()
    heightImage.initImage(os.path.join(dirname, 'hgt.rdr'),'write',width,'DOUBLE')
    heightImage.createImage()

    demImage = isceobj.createDemImage()
    demImage.load(demfilename + '.xml')
    demImage.setCaster('read', 'FLOAT')
    demImage.createImage()

    orb = self._insar.getMergedOrbit(frames)
    pegHdg = np.radians( orb.getENUHeading(t0))

    elp = Planet(pname='Earth').ellipsoid

    topo = PyTopozero()
    topo.set_firstlat(demImage.getFirstLatitude())
    topo.set_firstlon(demImage.getFirstLongitude())
    topo.set_deltalat(demImage.getDeltaLatitude())
    topo.set_deltalon(demImage.getDeltaLongitude())
    topo.set_major(elp.a)
    topo.set_eccentricitySquared(elp.e2)
    topo.set_rSpace(dr)
    topo.set_r0(r0)
    topo.set_pegHdg(pegHdg)
    topo.set_prf(1.0/dt)
    topo.set_t0(DTU.seconds_since_midnight(t0))
    topo.set_wvl(wvl)
    topo.set_thresh(.05)
    topo.set_demAccessor(demImage.getImagePointer())
    topo.set_dopAccessor(polyDoppler.getPointer())
    topo.set_slrngAccessor(slantRangeImage.getPointer())
    topo.set_latAccessor(latImage.getImagePointer())
    topo.set_lonAccessor(lonImage.getImagePointer())
    topo.set_losAccessor(losImage.getImagePointer())
    topo.set_heightAccessor(heightImage.getImagePointer())
    topo.set_incAccessor(0)
    topo.set_maskAccessor(0)
    topo.set_numIter(25)
    topo.set_idemWidth(demImage.getWidth())
    topo.set_idemLength(demImage.getLength())
    topo.set_ilrl(-1)
    topo.set_extraIter(10)
    topo.set_length(lgth)
    topo.set_width(width)
    topo.set_nRngLooks(1)
    topo.set_nAzLooks(1)
    topo.set_demMethod(5) # BIQUINTIC METHOD
    topo.set_orbitMethod(0) # HERMITE

    # Need to simplify orbit stuff later
    nvecs = len(orb._stateVectors)
    topo.set_orbitNvecs(nvecs)
    topo.set_orbitBasis(1) # Is this ever different?
    topo.createOrbit() # Initializes the empty orbit to the right allocated size
    count = 0
    for sv in orb._stateVectors:
        td = DTU.seconds_since_midnight(sv.getTime())
        pos = sv.getPosition()
        vel = sv.getVelocity()
        topo.set_orbitVector(count,td,pos[0],pos[1],pos[2],vel[0],vel[1],vel[2])
        count += 1

    topo.runTopo()

    latImage.addDescription('Pixel-by-pixel latitude in degrees.')
    latImage.finalizeImage()
    latImage.renderHdr()

    lonImage.addDescription('Pixel-by-pixel longitude in degrees.')
    lonImage.finalizeImage()
    lonImage.renderHdr()

    heightImage.addDescription('Pixel-by-pixel height in meters.')
    heightImage.finalizeImage()
    heightImage.renderHdr()

    descr = '''Two channel Line-Of-Sight geometry image (all angles in degrees). Represents vector drawn from target to platform.
            Channel 1: Incidence angle measured from vertical at target (always +ve).
            Channel 2: Azimuth angle measured from North in Anti-clockwise direction.'''
    losImage.setImageType('bil')
    losImage.addDescription(descr)
    losImage.finalizeImage()
    losImage.renderHdr()

    demImage.finalizeImage()

    if slantRangeImage:
        try:
            slantRangeImage.finalizeImage()
        except:
            pass

    ####Start creating VRTs to point to global topo output
    for swath, frame, istart in zip(swaths, frames, swathStarts):
        outname = os.path.join(dirname, 'IW{0}'.format(swath))

        os.makedirs(outname, exist_ok=True)

        for ind, burst in enumerate(frame.bursts):
            top = int(np.rint((burst.sensingStart - t0).total_seconds()/dt))
            bottom = top + burst.numberOfLines
            left = int(np.rint((burst.startingRange - r0)/dr))
            right = left + burst.numberOfSamples

            buildVRT( os.path.join(dirname, 'lat.rdr'),
                      os.path.join(outname, 'lat_%02d.rdr'%(ind+istart+1)),
                      [width, lgth],
                      [top,bottom, left, right],
                      bands=1,
                      dtype='DOUBLE')

            buildVRT( os.path.join(dirname, 'lon.rdr'),
                      os.path.join(outname, 'lon_%02d.rdr'%(ind+istart+1)),
                      [width, lgth],
                      [top,bottom, left, right],
                      bands=1,
                      dtype='DOUBLE')

            buildVRT( os.path.join(dirname, 'hgt.rdr'),
                      os.path.join(outname, 'hgt_%02d.rdr'%(ind+istart+1)),
                      [width, lgth],
                      [top,bottom, left, right],
                      bands=1,
                      dtype='DOUBLE')

            buildVRT( os.path.join(dirname, 'los.rdr'),
                      os.path.join(outname, 'los_%02d.rdr'%(ind+istart+1)),
                      [width, lgth],
                      [top,bottom, left, right],
                      bands=2,
                      dtype='FLOAT')

            catalog.addItem('Subset for IW{0}-B{1}'.format(swath, ind+1+istart), 'Lines: {0}-{1} out of {2}, Pixels: {3}-{4} out of {5}'.format(top, bottom, lgth, left, right, width), 'topo')

#            print('IW{0}-B{1}: {2} - {3}/ {4}, {5} - {6} /{7}'.format(swath, ind+1+istart, top, bottom, lgth, left, right, width))

    catalog.printToLog(logger, "runTopo")
    self._insar.procDoc.addAllFromCatalog(catalog)

    return

def buildVRT(srcname, dstname, dims, bbox, bands=1, dtype='FLOAT'):
    '''
    Write a VRT to point to the parent mosaicked file.
    '''

    header='<VRTDataset rasterXSize="{width}" rasterYSize="{lgth}">'
    band = '''    <VRTRasterBand dataType="{dtype}" band="{band}">
        <NoDataValue>0.0</NoDataValue>
        <SimpleSource>
            <SourceFilename relativeToVRT="1">{relpath}</SourceFilename>
            <SourceBand>{band}</SourceBand>
            <SourceProperties RasterXSize="{gwidth}" RasterYSize="{glgth}" DataType="{dtype}"/>
            <SrcRect xOff="{left}" yOff="{top}" xSize="{width}" ySize="{lgth}"/>
            <DstRect xOff="0" yOff="0" xSize="{width}" ySize="{lgth}"/>
        </SimpleSource>
    </VRTRasterBand>
'''
    tail = "</VRTDataset>"

    width = bbox[3] - bbox[2]
    lgth = bbox[1] - bbox[0]

    odtype = dtype
    if dtype.upper() == 'FLOAT':
        dtype = 'Float32'
    elif dtype.upper() == 'DOUBLE':
        dtype = 'Float64'
    elif dtype.upper() == 'BYTE':
        dtype = 'UInt8'
    else:
        raise Exception('Unsupported type {0}'.format(dtype))

    relpath = os.path.relpath(srcname + '.vrt', os.path.dirname(dstname))
    gwidth = dims[0]
    glgth = dims[1]
    left = bbox[2]
    top = bbox[0]

    img = isceobj.createImage()
    img.bands = bands
    img.scheme = 'BIL'
    img.setWidth(width)
    img.setLength(lgth)
    img.dataType = odtype
    img.filename = dstname
    img.setAccessMode('READ')
    img.renderHdr()

    with open(dstname + '.vrt', 'w') as fid:
        fid.write( header.format(width=width, lgth=lgth) + '\n')

        for bnd in range(bands):
            fid.write( band.format(width=width, lgth=lgth,
                                   gwidth=gwidth, glgth=glgth,
                                   left=left, top=top,
                                   relpath=relpath, dtype=dtype,
                                   band=bnd+1))

        fid.write(tail + '\n')


import sqlite3

conn = sqlite3.connect('C:/Users/User/Desktop/test1.db')
cursor = conn.cursor()

def table_creation():
    cursor.execute('''CREATE TABLE games(
        ID INT,
        name TEXT,
        genre TEXT,
        year INT,
        studio TEXT
        )
        ''')

table_creation()

conn.commit()

#cursor.execute('''SELECT * FROM students WHERE avg_score>9''')
'''
data = cursor.fetchall()

conn.commit()

print(data)'''


import pandas, xlsxwriter, progressbar, datetime

def export(storage):
    if storage != []:
        now = datetime.datetime.now()
        filename = "bikescrape-%d-%d-%d-%d%d%d" % (now.year, now.month, now.day, now.hour, now.minute, now.second)
        writer = pandas.ExcelWriter('%s.xlsx' % filename, engine='xlsxwriter')
        export = pandas.DataFrame(data=storage, columns=['Listing', 'Price', 'Link', 'Region'])
        export.to_excel(writer, sheet_name='Listings', index=False)
        worksheet = writer.sheets['Listings']
        worksheet.set_default_row(30)
        worksheet.set_column(0, 0, 45)
        worksheet.set_column(1, 1, 8)
        worksheet.set_column(2, 2, 80)
        worksheet.set_column(3, 3, 25)
        writer.book.formats[0].set_text_wrap(True)
        print("Exporting to %s.xlsx" % filename)
        writer.save()
        return ("%s.xlsx" %filename)
    else:
        print("Nothing to export :(")
        return None


import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics.pairwise import cosine_similarity

from main import user_item_rating
import modelEvaluator

class PopularityRecommender:
    MODEL_NAME = 'Popularity'

    def __init__(self, interactions_test_indexed_df, popularity_df, items_df=None):
        """
            This class is a Popularity Recommendation Model. It recommends based on the global popularity of items

            Args:
                interactions_test_indexed_df: the test sub-set of interactions_full_indexed_df
                popularity_df: the global item popularity dataframe
                items_df: a complete dataframe of the articles
        """

        self.interactions_test_indexed_df = interactions_test_indexed_df
        self.popularity_df = popularity_df
        self.items_df = items_df

    def get_model_name(self):
        """
            Returns:
                model_name: the name of the model
        """

        return self.MODEL_NAME

    def recommend_items(self, user_id, items_to_ignore=None, topn=10, verbose=False):
        """
            Args:
                user_id: the id of the user that the model is recommending items to
                items_to_ignore: the items that should be ignored by the recommender
                topn: defaults to 10. Defines amount of items to be recommended
                verbose: defaults to False
            Returns:
                recommendations_df: the topn recommended items
        """

        # Recommend the more popular items that the user hasn't seen yet.
        if items_to_ignore is None:
            items_to_ignore = []
        recommendations_df = self.popularity_df[~self.popularity_df['contentId'].isin(items_to_ignore)] \
            .sort_values('Rate', ascending=False) \
            .head(topn)

        if verbose:
            if self.items_df is None:
                raise Exception('"items_df" is required in verbose mode')

            recommendations_df = recommendations_df.merge(self.items_df, how='left',
                                                          left_on='contentId',
                                                          right_on='contentId')[
                ['Rate', 'contentId', 'title', 'url', 'lang']]

        return recommendations_df

def inspect_interactions(articles_df, interactions_train_indexed_df, interactions_test_indexed_df, person_id,
                         test_set=True):
    """
        Args:
            articles_df: a full dataframe of the articles
            interactions_train_indexed_df: the train sub-set of interactions_full_indexed_df
            interactions_test_indexed_df: the test sub-set of interactions_full_indexed_df
            person_id: the user's id
            test_set: defaults to True. Decides whether to use the train set or the test set for inspection
        Returns:
            interactions_df: the inspected interactions dataframe
    """
    if test_set:
        interactions_df = interactions_test_indexed_df
    else:
        interactions_df = interactions_train_indexed_df
    return interactions_df.loc[person_id].merge(articles_df, how='left',
                                                left_on='contentId',
                                                right_on='contentId') \
        .sort_values('Rate', ascending=False)[['Rate',
                                               'contentId',
                                               'title', 'url', 'lang']]

def inspect_interactions_user(user_id, interactions_df, users_enough_interactions, items_enough_rated):
    """
        Args:
            user_id: the user's id
            interactions_df: a complete dataframe of user interactions with the articles
            users_enough_interactions: a list of users that have interacted with at minimum three (3) articles
            items_enough_rated: a list of articles that have at least two (2) different users interacting with them
        Returns:
            item_scores_df: an item scores dataframe
    """
    # the amount of most similar users chosen for recommendation
    top = 5

    user_item_df, user_item_matrix = user_item_rating(interactions_df, users_enough_interactions, items_enough_rated)

    # The Sample User's content scores
    user_index = np.where(users_enough_interactions == user_id)[0][0]

    similar_users = []
    for i in range(len(users_enough_interactions)):
        if i != user_index:
            similar_users.append(
                [i, cosine_similarity([user_item_matrix[:, i]], [user_item_matrix[:, user_index]])[0][0]])

    similar_users = np.array(similar_users)
    ind = np.argsort(similar_users[:, -1])
    similar_users = similar_users[ind]

    # sorted in asc order, therefore most similar users are put in the end of the array
    similar_users = similar_users[-top:]

    # simply initialising the items array
    items = np.zeros(len(items_enough_rated))

    # for each item there will be a summed score based on the ratings of each similar user
    for similar_user in similar_users:
        s_u_index = int(similar_user[0])
        for i in range(len(items_enough_rated)):
            s = user_item_matrix[:, s_u_index]
            items[i] += s[i]

    item_scores_df = pd.DataFrame()
    item_scores_df['contentId'] = items_enough_rated
    item_scores_df['Rate'] = items

    item_scores_df = item_scores_df.sort_values(by='Rate',
                                                ascending=False)

    return item_scores_df

def item_collaborative_filtering(articles_enough_df, articles_df, interactions_full_df):
    """
        Performs item-based collaborative filtering. This model only takes global item ratings into account and thus is
        not personalised for the user

        Args:
            articles_enough_df: a list of articles that have at least two (2) different users interacting with them
            articles_df: a complete dataframe of the articles
            interactions_full_df: a complete dataframe of all user ratings for the articles

        Returns:
            global_metrics: the metrics of the evaluation
    """

    # The User Sample's ID
    user_id = -1479311724257856983

    print('\nPreparing data for Item-based Collaborative Filtering model...\n')

    interactions_train_df, interactions_test_df = train_test_split(interactions_full_df,
                                                                   stratify=interactions_full_df['personId'],
                                                                   test_size=0.20,
                                                                   random_state=42)
    # Indexing by personId to speed up the searches during evaluation
    interactions_full_indexed_df = interactions_full_df.set_index('personId')
    interactions_train_indexed_df = interactions_train_df.set_index('personId')
    interactions_test_indexed_df = interactions_test_df.set_index('personId')

    items = inspect_interactions(articles_df, interactions_train_indexed_df, interactions_test_indexed_df,
                                 user_id, test_set=True)

    item_popularity_df = interactions_full_indexed_df.groupby('contentId')['Rate'].sum().sort_values(
        ascending=False).reset_index()
    model_evaluator = modelEvaluator.ModelEvaluator(interactions_full_indexed_df, interactions_train_indexed_df,
                                                    interactions_test_indexed_df, articles_df, item_popularity_df)

    print("\n Here there are the ten most relevant items basing on the sum of ratings:\n")
    print(item_popularity_df.head(10))
    popularity_model = PopularityRecommender(interactions_test_indexed_df, item_popularity_df, articles_df)

    print('\nEvaluating Popularity recommendation model...\n')
    global_metrics, pop_detailed_results_df = model_evaluator.evaluate_model(popularity_model, items)

    print('\nGlobal metrics:\n%s' % global_metrics)
    print(pop_detailed_results_df.head(10))

    print(
        "\nFirst 20 relevant items for the user ID -1479311724257856983 \n(just a sample one, if you want to change "
        "user just call the function with an other ID): \n")

    print(items.head(20))

    return global_metrics

def user_collaborative_filtering(articles_df, interactions_full_df, interactions_df):
    """
        Performs user-based collaborative filtering. This model takes user-item ratings into account and is personalised

        Args:
            articles_df: a complete dataframe of the articles
            interactions_full_df: a complete dataframe of all user ratings for the articles
            interactions_df: a complete dataframe of all user interactions with the articles

        Returns:
            global_metrics: the metrics of the evaluation
    """
    # The User Sample's ID
    user_id = -1479311724257856983

    print('\nPreparing data for User-based Collaborative Filtering model...\n')

    interactions_train_df, interactions_test_df = train_test_split(interactions_full_df,
                                                                   stratify=interactions_full_df['personId'],
                                                                   test_size=0.20,
                                                                   random_state=42)
    # Indexing by personId to speed up the searches during evaluation
    interactions_full_indexed_df = interactions_full_df.set_index('personId')
    interactions_train_indexed_df = interactions_train_df.set_index('personId')
    interactions_test_indexed_df = interactions_test_df.set_index('personId')

    users_enough_interactions_train = interactions_train_df['personId'].unique()
    users_enough_interactions_test = interactions_test_df['personId'].unique()
    items_enough_rated_train = interactions_train_df['contentId'].unique()
    items_enough_rated_test = interactions_test_df['contentId'].unique()

    item_scores_df = inspect_interactions_user(user_id, interactions_df, users_enough_interactions_test,
                                               items_enough_rated_test)

    print(
        "\nFirst 10 most relevant items for the user ID -1479311724257856983 \n(just a sample one, if you want to "
        "change user just call the function with an other ID): \n")

    print(item_scores_df.head(10))

    model_evaluator = modelEvaluator.ModelEvaluator(interactions_full_indexed_df, interactions_train_indexed_df,
                                                    interactions_test_indexed_df, articles_df, item_scores_df)

    popularity_model = PopularityRecommender(interactions_test_indexed_df, item_scores_df, articles_df)

    print('\nEvaluating Recommendation model...\n')
    global_metrics, pop_detailed_results_df = model_evaluator.evaluate_model(popularity_model, item_scores_df)

    print('\nGlobal metrics:\n%s' % global_metrics)
    print(pop_detailed_results_df.head(10))

    return global_metrics


import logging

from ..akamai_utils.client import AkamaiApiClient

from nodestream.pipeline.extractors import Extractor

class AkamaiWAFExtractor(Extractor):
    def __init__(self, **akamai_client_kwargs) -> None:
        self.client = AkamaiApiClient(**akamai_client_kwargs)
        self.logger = logging.getLogger(self.__class__.__name__)

    async def extract_records(self):
        ag_map = {
            'POLICY':'Web Policy Violation',
            'WAT':'Web Attack Tool',
            'TOOL': 'Web Attack Tool',
            'PROTOCOL':'Web Protocol Attack',
            'SQL':'SQL Injection',
            'XSS':'Cross Site Scripting',
            'CMD':'Command Injection',
            'CMDI': 'Command Injection',
            'LFI':'Local File Inclusion',
            'RFI':'Remote File Inclusion',
            'PLATFORM':'Web Platform Attack',
            'OUTBOUND':'Total Outbound'
        }

        for config in self.client.list_appsec_configs():
            try:
                if 'productionVersion' in config.keys():
                    export = self.client.export_appsec_config(config_id = config['id'], config_version = config['productionVersion'])
                    # Construct output dict
                    output_config = {
                        'configId': config['id'],
                        'configName': config['name'],
                        'productionVersion': config['productionVersion'],
                        'policies': []
                    }
                    # Iterate through policies and add custom dict to output
                    for policy in export['securityPolicies']:
                        output_policy = {
                            'policyId': policy['id'],
                            'policyName': policy['name'],
                            'attackGroupActions': []
                        }
                        if 'webApplicationFirewall' in policy.keys():
                            for action in policy['webApplicationFirewall']['attackGroupActions']:
                                output_policy['attackGroupActions'].append({
                                        'group': action['group'],
                                        'groupName': ag_map[action['group']],
                                        'action': action['action']
                                    }
                                )
                            output_config['policies'].append(output_policy)

                    yield output_config
                else:
                    self.logger.warning(f"No production version exists for config {config['name']}: {err}")
            except Exception as err:
                self.logger.error(f"Failed to export appsec configuration {config['name']}: {err}")


from functools import wraps
from brevity.models import Post, Tag, Upvote
from brevity import db
from sqlalchemy import func, or_
import flask
import werkzeug
import werkzeug.wrappers

AdmindefinedTag = {
    "dbms" : "database",
    "database" : "dbms",
    "swe" : "software engineering",
    "software engineering" : "swe",
    "artificial intelligence" : "ai",
    "ai" : "artificial intelligence",
    "data structure" : "ds",
    "ds" : "data structure",
    "competitive programming" : "cp",
    "cp" : "competitive programming",
    "bst" : "binary search tree",
    "binary search tree" : "bst",
    "breadth first search" : "bfs",
    "bfs" : "breadth first search",
    "depth first search" : "dfs",
    "dfs" : "depth first search",
    "digital signal processing" : "dsp",
    "dsp" : "digital signal processing",
    "operating system" : "os",
    "os" : "operating system",
    "machine learning" : "ml",
    "ml" : "machine learning"

}
### search type tag(tag_type=0)
### otherwise(tag_type=1)
def search_result(searched_val,page,type):

    search = "%{}%".format(searched_val)

    tag_type=1

    sz = len(searched_val)
    if searched_val[0]=='[' and searched_val[sz-1]==']':
        tag_type=0
    if searched_val[0]=='[' and tag_type==0:
        searched_val = searched_val[1:]
    sz = len(searched_val)
    if searched_val[sz-1]==']' and tag_type==0:
        searched_val = searched_val[:sz-1]

    if tag_type==0:
        searched_val = searched_val.lower()
        AdminChosenTag = searched_val
        if(searched_val in AdmindefinedTag):
            AdminChosenTag = AdmindefinedTag[searched_val]

        AdminChosenTag = AdminChosenTag.lower()

        if type==0:
            posts = db.session.query(Post).join(Tag).filter(Post.id==Tag.post_id).filter(or_(func.lower(Tag.tag)==searched_val,func.lower(Tag.tag)==AdminChosenTag)).paginate(page=page, per_page=5)
        elif type==1:
            posts = db.session.query(Post).join(Tag).filter(Post.id==Tag.post_id).filter(or_(func.lower(Tag.tag)==searched_val,func.lower(Tag.tag)==AdminChosenTag)).order_by(Post.date_posted.desc()).paginate(page=page, per_page=5)
        else:
            posts = db.session.query(Post).join(Tag).join(Upvote).filter(Post.id==Tag.post_id).filter(Post.id==Upvote.post_id).filter(or_(func.lower(Tag.tag)==searched_val,func.lower(Tag.tag)==AdminChosenTag)).group_by(Post.id).order_by(func.count(Post.id).desc()).paginate(page=page, per_page=5)
    else:
        if type==0:
            posts = Post.query.filter(Post.title.like(search)).paginate(page=page, per_page=5)
        elif type==1:
            posts = Post.query.filter(Post.title.like(search)).order_by(Post.date_posted.desc()).paginate(page=page, per_page=5)
        else:
            posts = db.session.query(Post).join(Upvote).filter(Post.id==Upvote.post_id).filter(Post.title.like(search)).group_by(Post.id).order_by(func.count(Post.id).desc()).paginate(page=page, per_page=5)

    return posts

def response(*, mimetype: str = None, template_file: str = None):
    def response_inner(f):

        @wraps(f)
        def view_method(*args, **kwargs):
            response_val = f(*args, **kwargs)

            if isinstance(response_val, werkzeug.wrappers.Response):
                return response_val

            if isinstance(response_val, flask.Response):
                return response_val

            if isinstance(response_val, dict):
                model = dict(response_val)
            else:
                model = dict()

            if template_file and not isinstance(response_val, dict):
                raise Exception(
                    "Invalid return type {}, we expected a dict as the return value.".format(type(response_val)))

            if template_file:
                response_val = flask.render_template(template_file, **response_val)

            resp = flask.make_response(response_val)
            resp.model = model
            if mimetype:
                resp.mimetype = mimetype

            return resp

        return view_method

    return response_inner


##################################
### AUTHOR:
###     BENJAMIN WITTENBRINK
###     JOHN KOHLER
###     EVA BATELAAN
###     MICHELLE LAHRKAMP
###     THOMAS BRINK
###     MS&E 231 FINAL PROJECT
##################################

#%%
import tweepy
import argparse
import pandas as pd
import csv
import os
import time
import pickle
from datetime import datetime
import numpy as np

#%%
def load_politician_accounts(house, senate, prez):
    politicians_set = set()
    for filename in [house, senate, prez]:
        path = os.path.abspath(filename)
        with open(path) as f:
            account_list = csv.reader(f)
            for row in account_list:
                politicians_set.add(tuple(row))

    return politicians_set

#%%
def get_following_lists(client, users):
    for user in users:
        id = user[1]

        completed_files = os.listdir("data/following_lists")
        if np.any([str(id) in f for f in completed_files]):
            continue # we have already pulled this user's following list

        user = client.get_user(id = id, user_fields=["public_metrics"])
        if user.data is not None and user.data.public_metrics["following_count"] > 5000:
            continue # we will ignore this user's following

        paginator = tweepy.Paginator(
		    client.get_users_following,
		    id = id,
		    user_fields=['username'],
            max_results=1000
        )

        following = set()
        for rsp in paginator:
            time.sleep(60)
            if rsp.data is not None:
                for followee in rsp.data:
                    following.add((followee["username"],str(followee["id"]), followee["name"]))

        filename = "data/following_lists/" + id + "_following.pkl"
        with open(filename, "wb") as f:
            pickle.dump(following, f)

    return

#%%
def create_full_accounts_list(politicians):
    follower_lists_dir = os.listdir("data/following_lists")
    follower_lists = []
    for filename in follower_lists_dir:
        path = "data/following_lists/" + filename
        with open(path, "rb") as f:
            list = pickle.load(f)
        follower_lists.append(list)

    follower_counts = {}
    for list in follower_lists:
        for user in list:
            id = user[1]
            if id not in follower_counts.keys():
                follower_counts[id] = 1
            else:
                follower_counts[id] += 1

    politician_ids = set()
    for politician in politicians:
        id = politician[1]
        politician_ids.add(id)

    accounts_dict = {}
    for list in follower_lists:
        for user in list:
            id = user[1]
            if follower_counts[id] >= 15 or id in politician_ids:
                if id not in accounts_dict.keys():
                    accounts_dict[id] = user

    for politician in politicians:
        id = politician[1]
        if id not in accounts_dict.keys():
            print("lonely politician:" + politician[0])
            accounts_dict[id] = politician

    accounts_list = []
    for id in accounts_dict.keys():
        accounts_list.append(accounts_dict[id])

    filename = "data/accounts.pkl"
    with open(filename, "wb") as f:
        pickle.dump(accounts_list, f)

#%%
def pull_tweets(client, accounts):
    for account in accounts:
        id = account[1]

        completed_files = os.listdir("data/new_tweets")
        if np.any([str(id) in f for f in completed_files]):
            continue # we have already pulled this user's tweets

        start = datetime(year=2022,month=10,day=15,hour=0,minute=0,second=0)
        end = datetime(year=2022,month=12,day=1,hour=0,minute=0,second=0)

        paginator = tweepy.Paginator(
		    client.get_users_tweets,
		    id = id,
		    start_time = start,
            end_time = end,
            tweet_fields=[
			"author_id", "created_at", "public_metrics",
            "lang", "context_annotations"
            ],
            user_fields=["username"],
            max_results=100
        )

        tweets = []

        for rsp in paginator:
            time.sleep(1)
            if rsp.data is not None:
                for tweet in rsp.data:
                    tweets.append(tweet.data)

        # dump paginator
        filename = "data/new_tweets/" + str(id) + "_tweets.pkl"
        with open(filename, "wb") as f:
            pickle.dump(tweets, f)

    return

#%%
## MAIN
if __name__ == "__main__":
    parser = argparse.ArgumentParser(
		description="Get Following Matrix from Politician Twitter Accounts"
	)
    parser.add_argument("--keyfile", help="file with user credentials", required=True)

    flags = parser.parse_args()
    creds = {}
    for line in open(flags.keyfile, "r"):
        row = line.strip()
        if row:
            key, value = row.split()
            creds[key] = value

    politicians = load_politician_accounts("data/115th-Congress-House-seeds.csv", "data/115th-Congress-Senate-seeds.csv", "data/2016-presidential-candidates-seeds.csv")

    client = tweepy.Client(bearer_token=creds["bearer_token"])

    get_following_lists(client, politicians)

    create_full_accounts_list(politicians)
    with open("data/accounts.pkl", "rb") as f:
        full_accounts_list = pickle.load(f)

    get_following_lists(client, full_accounts_list)

# %%