"""
    This module contains code related to
    Think Python, 2nd Edition
    by Allen Downey
    http://thinkpython2.com

    This is to complete the exercises in Chapter 10: Lists in Think Python 2

    Note: Although this is saved in a .py file, code was run on an interpreter to get results
    Note: Using Python 3.9.0
"""
import bisect

def make_word_list():
    """
        Reads lines from a file and builds a list using append.

        return: list of strings
    """
    # initialize variables
    a_list = []

    # open file
    fin = open('words.txt')
    for line in fin:
        word = line.strip()
        a_list.append(word)

    # Close the file
    fin.close()
    return a_list

def in_bisect(word_list, word):
    """
        Checks whether a word is in a list using bisection search.
        Precondition: the words in the list are sorted

        word_list: list of strings
        word: string

        return: boolean; True if the word is in the list, False otherwise
    """
    if len(word_list) == 0:
        return False

    i = len(word_list) // 2
    if word_list[i] == word:
        return True

    if word_list[i] > word:
        # search the first half
        return in_bisect(word_list[:i], word)
    else:
        # search the second half
        return in_bisect(word_list[i+1:], word)

def reverse_pair(a_list, word):
    """
        Determine whether a reversed word appeas in the word list

        a_list: list
        word: string

        return: boolean; True if the word appears in the list, false otherwise
    """
    reversed_word = word[::-1]
    return in_bisect(a_list, reversed_word)

print("Welcome! We're going to determine whether a word list has reversed pairs!")

# initialize variable
word_list = make_word_list()

for word in word_list:
    if reverse_pair(word_list, word):
        print(word, word[::-1])


import argparse
import numpy as np
import pandas as pd

desc = """Concatenate the data in multiple .csv files.
  This is good for when many similar sims have been run that return the same
  kinds of data, and they are to be plotted together."""
parser = argparse.ArgumentParser(description=desc)
fh = "Provide a list of csv files to concatenate"
parser.add_argument("filenames", nargs="+", help=fh)
Ah = """Whether to average all values that report about the same epoch.
  This requires that the csv files have the columns [Epoch, %%RMSR]"""
  # Btw, the % needs to be escaped for some reason in the above line
parser.add_argument("-A", "--avg_epochs", action="store_true", help=Ah)
parser.add_argument("-o", "--out_file", default="OUT.csv",
  help="Name of output file. Default=OUT.csv")
parser.add_argument("-c", "--cutoff", type=float,
  help="Will ignore all values lower than cutoff. default=None")
parser.add_argument("--old", action="store_true",
  help="Is this an older file format? If so, use the key %%SSR, not %%RMSR")
args = parser.parse_args()

outfile = args.out_file
#i0 = lambda f: 1+f.rindex("/") if ("/" in f) else 0
#trimmed_fnames = [f[i0(f):f.rindex(".csv")] for f in args.filenames]
#outfile = "_-_".join(trimmed_fnames) + ".csv"
#outfile = "OUT.csv"
cutoff = args.cutoff
y_key = "%SSR" if args.old else "%RMSR"

rd_csv = lambda f: pd.read_csv(f, skipinitialspace=True)
df = pd.concat(map(rd_csv, args.filenames))
if cutoff is not None:
  df = df[df[y_key] > cutoff]
else:
  df = df[df["Code"] < 3]

#print(df)
if args.avg_epochs:
  # Average all values that refer to the same Epoch
  #cutoff = 39 # Ignore small values...
  Epochs = df["Epoch"].unique()
  df2 = pd.DataFrame(np.zeros( (Epochs.shape[0], 2) ))
  df2.columns = ["Epoch", y_key]
  df2["Epoch"] = Epochs
  for E in Epochs:
    filtered = df[y_key][df["Epoch"] == E]
    #filtered = filtered[filtered > cutoff]
    df2.loc[E, y_key] = np.mean(filtered.to_numpy())
  df = df2

df.to_csv(outfile, index=None)
print(f"Combined data saved to {outfile}")


import os
import pytest
from pathlib import Path
from tempfile import TemporaryDirectory
import pandas as pd
from feast import FeatureStore
from tests.generators import Generator, csv, parquet, start_date, end_date
from datetime import datetime, timezone
from yummy import select_all

def entity_df_all():
    return select_all(datetime(2021, 10, 2, 1, 0, 0, tzinfo=timezone.utc))

def e2e_basic(feature_store: FeatureStore, tmp_dir: TemporaryDirectory, backend: str, entity_df: pd.DataFrame):
    """
    This will test all backends with basic data stores (parquet and csv)
    """
    feature_store.config.offline_store.backend = backend

    csv_fv, csv_fv_name = csv(tmp_dir)
    parquet_fv, parquet_fv_name = parquet(tmp_dir)

    entity = Generator.entity()
    feature_store.apply([entity, csv_fv, parquet_fv])

    feature_vector = feature_store.get_historical_features(
        features=[
            f"{csv_fv_name}:f0",
            f"{parquet_fv_name}:f0",
        ], entity_df=entity_df, full_feature_names=True
    ).to_df()

    feature_store.materialize(start_date=start_date, end_date=end_date)

    fv = feature_vector[feature_vector.entity_id.astype('int') == 1].to_dict(orient="records")[0]
    csv_f0 = float(fv[f"{csv_fv_name}__f0"])
    parquet_f0 = float(fv[f"{parquet_fv_name}__f0"])

    assert(csv_f0 is not None)
    assert(parquet_f0 is not None)

    ofv = feature_store.get_online_features(
        features=[
            f"{csv_fv_name}:f0",
            f"{parquet_fv_name}:f0",
        ],
        entity_rows = [{"entity_id": 1}],
        full_feature_names=True,
    ).to_df().to_dict(orient="records")

    assert(abs(ofv[0]['fv_csv__f0'] - csv_f0) < 1e-6)
    assert(abs(ofv[0]['fv_parquet__f0'] - parquet_f0) < 1e-6)

@pytest.mark.polars
def test_e2e_basic_polars(feature_store: FeatureStore, tmp_dir: TemporaryDirectory):
    """
    This will test polars backend with basic data stores (parquet and csv)
    """
    entity_df = Generator.entity_df()
    e2e_basic(feature_store, tmp_dir, "polars", entity_df)

@pytest.mark.polars
def test_e2e_basic_select_all_polars(feature_store: FeatureStore, tmp_dir: TemporaryDirectory):
    """
    This will test polars backend with basic data stores (parquet and csv)
    """
    e2e_basic(feature_store, tmp_dir, "polars", entity_df_all())

@pytest.mark.dask
def test_e2e_basic_dask(feature_store: FeatureStore, tmp_dir: TemporaryDirectory):
    """
    This will test dask backend with basic data stores (parquet and csv)
    """
    entity_df = Generator.entity_df()
    e2e_basic(feature_store, tmp_dir, "dask", entity_df)

@pytest.mark.dask
def test_e2e_basic_select_all_dask(feature_store: FeatureStore, tmp_dir: TemporaryDirectory):
    """
    This will test dask backend with basic data stores (parquet and csv)
    """
    e2e_basic(feature_store, tmp_dir, "dask", entity_df_all())

@pytest.mark.ray
def test_e2e_basic_ray(feature_store: FeatureStore, tmp_dir: TemporaryDirectory):
    """
    This will test ray backend with basic data stores (parquet and csv)
    """
    entity_df = Generator.entity_df()
    e2e_basic(feature_store, tmp_dir, "ray", entity_df)

@pytest.mark.ray
def test_e2e_basic_select_all_ray(feature_store: FeatureStore, tmp_dir: TemporaryDirectory):
    """
    This will test ray backend with basic data stores (parquet and csv)
    """
    e2e_basic(feature_store, tmp_dir, "ray", entity_df_all())

@pytest.mark.parametrize("backend", ["polars", "dask", "ray"])
@pytest.mark.nospark
def test_e2e_basic_nospark(feature_store: FeatureStore, tmp_dir: TemporaryDirectory, backend: str):
    """
    This will test all backends with basic data stores (parquet and csv)
    """
    entity_df = Generator.entity_df()
    e2e_basic(feature_store, tmp_dir, backend, entity_df)

@pytest.mark.parametrize("backend", ["spark", "polars", "dask", "ray"])
@pytest.mark.spark
def test_e2e_basic_spark(feature_store: FeatureStore, tmp_dir: TemporaryDirectory, backend: str):
    """
    This will test all backends with basic data stores (parquet and csv)
    """
    entity_df = Generator.entity_df()
    e2e_basic(feature_store, tmp_dir, backend, entity_df)

@pytest.mark.parametrize("backend", ["spark", "polars", "dask", "ray"])
@pytest.mark.spark
def test_e2e_basic_select_all_spark(feature_store: FeatureStore, tmp_dir: TemporaryDirectory, backend: str):
    """
    This will test all backends with basic data stores (parquet and csv)
    """
    e2e_basic(feature_store, tmp_dir, backend, entity_df_all())


"""
Administra la lectura de datos, y el modo de guardarlos
"""
from .utils import getCvlacsDirs
import bs4
import ssl
from urllib.request import urlopen
from bs4 import BeautifulSoup
import csv
import re
# Ajusta los certificados utilizados para realizar las conexiones
ssl._create_default_https_context = ssl._create_unverified_context

class Admin(object):

    def __init__(self,filename):

        self.cvlacDirs=getCvlacsDirs(filename)
        self.libros=[]

    def Run(self):
        """
        Realiza la lectura
        """
        if len(self.cvlacDirs)<2:
            print("No hay información para procesar")
            return
        for i in range(1,len(self.cvlacDirs)):
            self.LecturaCvlac(self.cvlacDirs[i][-1])

    def LecturaCvlac(self, urlCvlac):
        """
        Realiza la lectura de una url de CvLAC
        """
        uClient = urlopen(urlCvlac)
        page_html = uClient.read()
        uClient.close()
        page_soup = BeautifulSoup(page_html,"html.parser")
        # Cada sección de información se maneja como una tabla
        secciones = page_soup.findAll("table")
        nombreInvestigador = ""
        for i, sec in enumerate( secciones[0:] ):
            nombreSec = sec.h3
            if nombreSec != None:
                nombreSec = nombreSec.text
            else:
                nombreSec = ""
            if nombreSec != "":
                print(f"seccion {i}:", nombreSec)
            # Extrae el nombre del investigador
            if nombreInvestigador == "" and nombreSec == "" :
                tdNombre = self.ExtraerTag(sec,['tr','td'])

                filas = sec.findAll("tr", limit=12)
                for fila in filas:
                    if fila.td.text == "Nombre":
                        nombreInvestigador = fila.findAll("td")[1].text.strip()
                        print("  Nombre identificado:", nombreInvestigador)
                        break

            elif nombreSec != None and nombreSec == "Estrategias pedagógicas para el fomento a la CTI":
                self.ProcesaSecEstrFomentoCTI(sec, nombreInvestigador)

    def ProcesaSecEstrFomentoCTI(self, sec,investigador):
        """
        Procesa la sección de Capítulos de Libros
        """
        # Contenedor con la información de cada libro junto con el tipo
        contInfo = sec.findAll("blockquote")

        estrategia=[]
        for i in range(0, len(contInfo)):
            info_libro = contInfo[i].text

            #Nombre Libro
            linesInfo = [ l.strip() for l in info_libro.split("\n")]
            linesInfo = [ l for l in linesInfo if len(l)>0]
            #print(linesInfo)

            #Contiene la información del tipo de producto
            tipo = "Por definir"

            NombreProducto=linesInfo[0]
            NombreProducto= NombreProducto[24:]

            listaAutores=[]

            #Año de inicio
            AnoInicio = linesInfo[1]
            AnoInicio = AnoInicio[9:-1]

            #Año de finalización
            AnoFin = linesInfo[2]
            AnoFin = AnoFin[13:-1]
            if AnoFin==" - ":
                AnoFin=""

            estrategia = [investigador, NombreProducto, tipo, AnoInicio, AnoFin]
            self.libros.append(estrategia)

    #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    def SaveEstrFomentoCTI(self,fname):
        """
        Guarda los articulos en un archivo csv
        """
        encabezado=["Investigador","Nombre de la estrategia", "Tipo", "Año de inicio", "Año de finalización"]
        datos=[encabezado]+self.libros
        with open( fname,'w', encoding="UTF-8") as f:
            writer = csv.writer(f, delimiter=',',quotechar='"')
            for row in datos:
                writer.writerow(row)

    #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    def ExtraerTag(self,soup,ruta):
        """
        Extrae un un tag siguiendo la ruta de nodos en orden descendente, si
        reverse está activado la ruta indica como ascender en el arbol
        """
        tagResultado = soup
        for tagName in ruta:
            tagResultado = tagResultado.find(tagName)
            #print("  tag:",tagName)
            if tagResultado == None:
                break
        return tagResultado


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Jun  2 19:32:13 2018

@author: Fan JIANG, Chali SHEN
"""

import numpy as np
import scipy.stats as sts
import scipy.optimize as opt
import matplotlib.pyplot as plt
import time

# The function we need to use
def d_plus(sigma,t,x,y):
    return 1./(sigma*np.sqrt(t))*np.log(x/y)+0.5*sigma*np.sqrt(t)

def d_minus(sigma,t,x,y):
    return 1./(sigma*np.sqrt(t))*np.log(x/y)-0.5*sigma*np.sqrt(t)

def Call(sigma,t,S,K):
    return S*sts.norm.cdf(d_plus(sigma,t,S,K)) - K*sts.norm.cdf(d_minus(sigma,t,S,K))

def Put(sigma,t,S,K):
    return K*sts.norm.cdf(d_plus(sigma,t,K,S)) - S*sts.norm.cdf(d_minus(sigma,t,K,S))

def Value(sigma,alpha,beta,t,S,K):
    return np.sum(alpha*Call(sigma,t,S,K) + beta*Put(sigma,t,S,K),axis = 0)

# parameters
sigma = 0.2 #volatility
days = 10 # time interval
h = days/360. # 10 days, assuming 360 days per year
del_h = 1./360.
T = 1. # maturity: 1 year
t = 0. # present time: 0 year
I = 10 # number of stocks
alpha = (np.array([10.]*5+[10.]*5)).reshape((I,1))
beta = (np.array([5.]*5+[5.]*5)).reshape((I,1))
S_0 = (np.ones(I)*10).reshape((I,1))
K = (np.ones(I)*15).reshape((I,1))
rho = 0.4 # corelation of B_0 and B_i
lamb = 0.5 # parameter for the score function G = exp(lamb * (L_n - L_{n-1}))

num_event = 10**4
num_iter = 500

level = 0.99999  # level of Var
target = 1. - level

lamb_possion = 5. # parameter of N_t: N_t~Poisson(lamb_possion* h)
#amplitude = -0.1

start = time.time()

print("generating poisson process")
N_poisson = np.random.poisson( lamb_possion *h, size = (num_event,num_iter))
days = max(np.amax(N_poisson),days)
del_h = h/days
Proc_poisson = np.zeros((num_event, days,num_iter))
for i in range(num_event):
    for j in range(num_iter):
        if N_poisson[i,j] > 0:
            index = np.random.randint(0,days,size = N_poisson[i,j])
            #Proc_poisson[i,index,j] = amplitude
            # the amplitude of each chock
            Proc_poisson[i,index,j] = np.random.normal(-2,1,size = len(index))*np.sqrt(del_h)
print("poisson process generated")

B0 = np.random.randn(num_event, days,num_iter)*np.sqrt(del_h)
B_i = np.random.randn(I,num_event, days,num_iter)*np.sqrt(del_h)
print("number of sub-iterations: "+str(num_iter))
print("num_event per iteration: " + str(num_event))

final_Loss = np.zeros((num_iter,num_event,2))
constants = np.zeros(num_iter) # the second term in the equation for calculating expectation
initial_value = Value(sigma,alpha,beta,T-t,S_0,K)

for i in range(num_iter):
    S_t = np.zeros((I, num_event,2))
    S_t[:,:,0] = S_0
    Values = np.zeros((num_event,2))
    Values[:,0] = Value(sigma,alpha,beta,T-t,S_t[:,:,0],K)
    Loss = np.zeros((num_event,days+1))
    #print("*"*50)
    if i%50==0:
        print("Iteration " + str(i))
    #print("")
    constant_normalisation = 1.
    # selection- mutation algorithm
    for j in range(1,days):
        W = B_i[:,:,j-1,i] *np.sqrt(1-rho) + (B0[:,j-1,i]+ Proc_poisson[:,j-1,i]) * np.sqrt(rho)
        S_t[:,:,1] = S_t[:,:,0] + S_t[:,:,0]*sigma*W
        Values[:,1] = Value(sigma,alpha,beta,T-j*del_h,S_t[:,:,1],K)
        Loss[:,j] = Loss[:,j-1]-(Values[:,1] - Values[:,0])
        G = np.exp(lamb*(Loss[:,j] - Loss[:,j-1])) # score function
        constant_normalisation *= np.mean(G)
        #selection
        index = np.random.choice(range(num_event),p = G/np.sum(G),size = num_event)
        B0[:,j-1,i] = B0[index,j-1,i]
        Proc_poisson[:,j-1,i] = Proc_poisson[index,j-1,i]
        Loss[:,j] = Loss[index,j]
        Values[:,0] = Values[index,1]
        S_t[:,:,0] = S_t[:,index,1]

    # last mutation
    W = B_i[:,:,days-1,i] *np.sqrt(1-rho) + (B0[:,days-1,i]+  Proc_poisson[:,days-1,i]  ) * np.sqrt(rho)
    S_t[:,:,1] = S_t[:,:,0] + S_t[:,:,0]*sigma*W
    Values[:,1] = Value(sigma,alpha,beta,T-days*del_h,S_t[:,:,1],K)
    Loss[:,days] = Loss[:,days-1]-(Values[:,1] - Values[:,0])
    #print("Loss: "+str(Loss[:,j]))
    final_Loss[i,:,1] = Loss[:,days]
    final_Loss[i,:,0] = Loss[:,days-1]
    constants[i] = constant_normalisation

print("Initial Value: " + str(initial_value))
"""
select the quantile
sort the final loss, and the weight associated
find the value of loss such that accumulated weight is greater than alpha
"""
Loss_last = final_Loss[:,:,1]
Loss_secondLast = final_Loss[:,:,0]
index_sort = Loss_last.argsort(axis = 1)
for i in range(num_iter):
    Loss_last[i,::-1] = Loss_last[i,index_sort[i]]
    Loss_secondLast[i,::-1] = Loss_secondLast[i,index_sort[i]]

weight = 1./np.exp(lamb*Loss_secondLast)
weight_accumulate = np.cumsum(weight,axis = 1)
VaR_iter = [] # record the VaR for each iteration
for i in range(num_iter):
    tmp = Loss_last[i,weight_accumulate[i] >= num_event*target/constants[i]]
    if len(tmp) !=0:
        VaR_iter.append(tmp[0])

VaR_iter= np.array(VaR_iter)

# VaR
VaR_mean =  np.mean(VaR_iter[:])
VaR_std = np.std(VaR_iter[:])
print("VaR pour "+str(level*100)+"%: "+str(VaR_mean))
print("l'intervalle de confiance de 95% avec "+str(num_iter)+" interations de VaR: ["+
     str(VaR_mean - 1.96*VaR_std/np.sqrt(num_iter))+", "+str(VaR_mean + 1.96*VaR_std/np.sqrt(num_iter))+"]")
print("l'erreur relative: "+str(100*2*1.96*VaR_std/np.sqrt(num_iter)/VaR_mean)+"%")

# Conditional VaR
terms1 = np.mean((final_Loss[:,:,1] > VaR_mean)/np.exp(lamb*final_Loss[:,:,0]),axis =1)
probs = terms1 * constants
Conditional_Var_term1 = np.mean((final_Loss[:,:,1] > VaR_mean)*final_Loss[:,:,1]/np.exp(lamb*final_Loss[:,:,0]),axis =1)*constants
Conditional_Var = []
for i in range(len(probs)):
    if probs[i] != 0:
        Conditional_Var.append(Conditional_Var_term1[i]/probs[i])

Conditional_Var = np.array(Conditional_Var)

# probabilities
mean = np.mean(probs)
std = np.std(probs)
print("")
print("P(L> VaR): " + str(mean))
print("P(L <= VaR) = "+ str((1- mean)*100.)+"%")
print("l'intervalle de confiance de 95% avec "+str(num_iter)+" interations: ["+
     str(mean - 1.96*std/np.sqrt(num_iter))+", "+str(mean + 1.96*std/np.sqrt(num_iter))+"]")
print("l'erreur relative: "+str(100*2*1.96*std/np.sqrt(num_iter)/mean)+"%")

mean_condi = np.mean(Conditional_Var)
std_condi = np.std(Conditional_Var)
print("")
print("Conditional Var E[L|L>=VaR] avec P(L>VaR)="+str(mean)+": "+ str(mean_condi))
print("l'intervalle de confiance de 95% avec "+str(num_iter)+" interations: ["+
     str(mean_condi - 1.96*std_condi/np.sqrt(num_iter))+", "+str(mean_condi + 1.96*std_condi/np.sqrt(num_iter))+"]")
print("l'erreur relative: "+str(100*2*1.96*std_condi/np.sqrt(num_iter)/mean_condi)+"%")

end = time.time()

# distribution conditionnelle
interval = np.linspace(VaR_mean, min(np.amax(final_Loss[:,:,1]),2*VaR_mean))
P = []
P_mean = np.zeros(len(interval))
P_std = np.zeros(len(interval))
integral = 0.
for k in range(len(interval)-1):
    tmp = []
    for i in range(len(probs)):
        if probs[i] != 0:
            tmp.append(np.mean((final_Loss[i,:,1]>interval[k])*(final_Loss[i,:,1]<=interval[k+1])/np.exp(lamb*final_Loss[i,:,0]))*constants[i]/probs[i]/(interval[k+1]-interval[k]))
    P.append(np.array(tmp))
    P_mean[k] = np.mean(P[k])
    P_std[k] = np.std(P[k])
    integral +=P_mean[k]*(interval[k+1]-interval[k])
print("integral:" +str(integral))

plt.figure(dpi=150)
plt.plot(interval,P_mean)
plt.xlabel("Perte au dela de la VaR")
plt.ylabel("Proba conditionelle")
plt.title("Distribution conditionelle au dela de la VaR(mesure P)")

# show the trajectories of B0 and poisson process for extreme losses
B0 = B0 + Proc_poisson
losses = final_Loss[:,:,1]
m, n  = losses.shape
losses = losses.reshape(m*n)
indice = np.where(losses > VaR_mean)
losses = losses[indice]
B = B0.reshape((B0.shape[0]*B0.shape[2],days))
B = B[indice,:]
B = B.reshape((B.shape[-1],B.shape[-2]))
xx = range(1,days+1)
B_extreme = B[:,losses.argsort()]
B_extreme = np.cumsum(B_extreme,axis = 0)

plt.figure(dpi=150)
num_mouvment = 10
plt.plot(xx,B_extreme[:,-num_mouvment:])
plt.hlines(0.0,0,days,color = 'r')
plt.xlabel("jour")
plt.ylabel("B0")
plt.title(str(num_mouvment) + "B0 + Processus poisson associe a la perte extreme quand rho = "+str(rho))

#show the poisson process for extreme losses
pp = Proc_poisson.reshape((Proc_poisson.shape[0]*Proc_poisson.shape[2],days))
pp = pp[indice,:]
pp = pp.reshape((pp.shape[-1],pp.shape[-2]))
pp_extreme = pp[:,losses.argsort()]
pp_extreme = np.cumsum(pp_extreme,axis = 0)

plt.figure(dpi=150)
plt.plot(xx,pp_extreme[:,-num_mouvment:])
plt.hlines(0.0,0,days,color = 'r')
plt.xlabel("jour")
plt.ylabel("Poisson")
plt.title(str(num_mouvment) + " Processus poisson associe a la perte extreme quand rho = "+str(rho))

print("time: "+ str(end-start)+"s")


# %%
import pandas as pd

from textblob import TextBlob
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer

from NPF.teachers_and_covid import start
from NPF.library import process_text
import spacy
from tqdm import tqdm
from nltk.stem import PorterStemmer
import nltk
import numpy as np

nlp = spacy.load("en_core_web_sm")
import string
from nltk.tokenize import RegexpTokenizer

# %%
df = pd.read_csv(start.CLEAN_DIR + "tweets_relevant.csv")
df = df.rename(columns={"text": "tweet_text"})
df = df[["unique_id", "tweet_text"]]
df["unique_id"] = df.unique_id.astype(int)
df = df.set_index("unique_id")
# df = df.sample(200)

annotations = pd.read_csv(
    start.CLEAN_DIR + "annotations_characters.csv", index_col="unique_id"
)
annotations = annotations[
    [
        "split",
        "character",
        "character_final",
        "tweet_hero",
        "tweet_villain",
        "tweet_victim",
    ]
]
annotations = annotations.rename(
    columns={
        "character": "tweet_character",
        "character_final": "tweet_character_final",
        "split": "tweet_split",
    }
)
df = df.merge(
    annotations, how="left", left_index=True, right_index=True, indicator=True
)
df["annotated"] = np.where(df._merge == "both", 1, 0)
df = df.drop("_merge", axis=1)

# %% Make lower, remove punctuation, stem
def make_lower(text):
    return text.lower()

df["tweet_text_clean"] = df.tweet_text.apply(make_lower)
df.sample()

def remove_punctuation(text):
    return text.translate(str.maketrans("", "", string.punctuation))

df["tweet_text_clean"] = df.tweet_text_clean.apply(remove_punctuation)
df.sample()

stemmer = PorterStemmer()

def stem_string(text):
    token_list = []
    for token in text.split(" "):
        token_list.append(stemmer.stem(token))
    return " ".join(token_list)

df["tweet_text_clean"] = df.tweet_text_clean.apply(stem_string)
df.sample()

# %% Doc-Term Matrix
term_matrix = process_text.vectorize_text(
    df=df, text_col="tweet_text_clean", remove_stopwords=False, lemma=False, min_df=35
)
term_matrix.columns = ["term_" + str(col) for col in term_matrix.columns]
term_matrix.to_csv(start.CLEAN_DIR + "matrix.csv", index=True)

matrix_annotations = term_matrix[term_matrix.index.isin(annotations.index)]
matrix_annotations.to_csv(start.CLEAN_DIR + "matrix_annotations.csv", index=True)
# %%
lsa_matrix, word_weights = process_text.create_lsa_dfs(
    matrix=term_matrix, n_components=100
)
lsa_matrix = lsa_matrix.add_prefix("lsa_")
lsa_matrix.to_csv(start.CLEAN_DIR + "lsa.csv")

lsa_annotations = lsa_matrix[lsa_matrix.index.isin(annotations.index)]
lsa_annotations.to_csv(start.CLEAN_DIR + "lsa_annotations.csv", index=True)

# %% LIWC
liwc = pd.read_csv(start.CLEAN_DIR + "liwc_original.csv")
liwc = liwc.add_prefix("liwc_")
liwc = liwc.rename(columns={"liwc_unique_id": "unique_id"})
liwc = liwc.set_index("unique_id")
liwc.to_csv(start.CLEAN_DIR + "liwc.csv", index=True)

liwc_annotations = liwc[liwc.index.isin(annotations.index)]
liwc_annotations.to_csv(start.CLEAN_DIR + "liwc_annotations.csv", index=True)

# %%
feature_df = df.merge(liwc, left_index=True, right_index=True, how="left")
feature_df = feature_df.merge(lsa_matrix, left_index=True, right_index=True)
feature_df = feature_df.merge(term_matrix, left_index=True, right_index=True)

feature_df.to_csv(start.MAIN_DIR + "data/clean/features.csv")

features_annotations = feature_df[feature_df.index.isin(annotations.index)]
features_annotations = features_annotations[
    features_annotations.tweet_split != "validation"
]
features_annotations.to_csv(
    start.MAIN_DIR + "data/clean/features_annotations.csv", index=True
)

# %%


"""Form for entering the transaction data."""

from general import ledgeradd
from general.functions import acc_list_to_multiline
from general.functions import multiline_to_acc_list
from general.functions import move_list_entry
from general.ledgerparse import Transaction
import npyscreen

class PostingList(npyscreen.MultiLineEditable):
    """PostingList holding teh posts."""

    def __init__(self, *arg, **kwargs):
        """Initialize the class."""
        super(PostingList, self).__init__(*arg, **kwargs)

        # set up key shortcuts
        self.add_handlers({
            '+': self.move_up,
            '-': self.move_down
        })

        # set up additional multiline options
        self.scroll_exit = True

    def move_up(self, keypress=None):
        """Move selected offer up in the list."""
        lis = self.values

        # cancel if list is < 2
        if len(lis) < 2:
            return False

        # move selected item up
        new_index = move_list_entry(
            lis=lis,
            index=self.cursor_line,
            direction=1
        )

        self.cursor_line = new_index

    def move_down(self, keypress=None):
        """Move selected offer down in the list."""
        lis = self.values

        # cancel if list is < 2
        if len(lis) < 2:
            return False

        # move selected item up
        new_index = move_list_entry(
            lis=lis,
            index=self.cursor_line,
            direction=-1
        )

        self.cursor_line = new_index

class TitleMultiLineEdit(npyscreen.TitleText):
    """Titled MultiLineEdit"""

    _entry_type = PostingList

class TransactionForm(npyscreen.FormMultiPageActionWithMenus):
    """Transaction form for entering the transaction data."""

    def __init__(self, *arg, **kwargs):
        """Initialize the class."""
        super(TransactionForm, self).__init__(*arg, **kwargs)

        # set up key shortcuts
        self.add_handlers({
            '^O': self.on_ok,
            '^Q': self.on_cancel,
            '^F': self.clear_widget,
            '^A': self.select_account,
            '^L': self.show_presets
        })

    def clear_widget(self, keypress=None):
        """Clear widget."""
        self.get_widget(self.editw).value = ''

    def select_account(self, keypress=None):
        """
        Switch to account widget.

        Buggy! npyscreen switches to the widget correctly directly after
        first usage. After that it chooses the wrong widget according
        to following pattern:
            - a widget above account[0] is selected:
                > ONE widget above account[0] is beeing selected
            - a widget below account[0] is selected:
                > ONE widget below account[0] is beeing selected
        """
        # disable old editing if it's not the account[0]
        self.get_widget(self.editw).entry_widget.editing = False
        self.set_editing(self.postings)

    def new_trans(self):
        """Make new transaction."""
        self.parentApp.tmpTrans_new = True
        self.beforeEditing()

    def afa_feature(self):
        """Start the afa feature."""
        # cancel if there is no afa table in the settings
        if len(self.parentApp.S.get_afa_table()) == 0:
            npyscreen.notify_confirm(
                'No afa table exists. Add entries in the settings.',
                form_color='WARNING'
            )
            return False

        # save the entered transaction
        self.values_to_tmp(message=False)

        trans = ledgeradd.afa_check_trans_in_journal(
            settings=self.parentApp.S,
            transaction=self.parentApp.tmpTransC
        )

        # got no transaction, but error message
        if type(trans) is str:
            npyscreen.notify_confirm(
                trans,
                form_color='WARNING'
            )
            return False

        # seems ok, go on to the first afa form
        posts = ledgeradd.afa_get_postings(transaction=trans)

        # cancle if no posts were found
        if len(posts) == 0:
            npyscreen.notify_confirm(
                'No postings found for afa feature, canceling ...',
                form_color='WARNING'
            )

        # otherwise go on
        self.parentApp.tmpTransC = trans

        # fill the chooseforms
        self.parentApp.getForm('AfaPostingChoose').list.values = posts
        self.parentApp.getForm('AfaTypeChoose').list.values = (
            self.parentApp.S.get_afa_table()
        )

        # switch to the posting chooser first
        self.parentApp.setNextForm('AfaPostingChoose')
        self.parentApp.switchFormNow()

    def show_history(self):
        """Sow the history."""
        npyscreen.notify_confirm(
            self.parentApp.History
        )

    def show_presets(self, keypress=None):
        """Show presets."""
        self.values_to_tmp(message=False)
        self.parentApp.setNextForm('Presets')
        self.parentApp.switchFormNow()

    def add_preset(self):
        """Add current trans to presets."""
        self.values_to_tmp(message=False)

        # get name for preset
        name = npyscreen.notify_input(
            'Name for preset'
        )

        if not name:
            return

        # get info for preset
        info = npyscreen.notify_input(
            'Infotext'
        )

        if info is False:
            return

        # try to add it to the presets
        added = self.parentApp.P.add_trans(
            name=name,
            info=info,
            transaction=self.parentApp.tmpTrans
        )

        if not added:
            npyscreen.notify_confirm(
                'Could not add preset. Name probably already exists.',
                form_color='WARNING'
            )

    def show_settings(self):
        """Switch to settings form."""
        self.values_to_tmp(message=False)
        self.parentApp.setNextForm('Settings')
        self.parentApp.switchFormNow()

    def exit(self):
        """Exit the programm."""
        self.parentApp.setNextForm(None)
        self.parentApp.switchFormNow()

    def create(self):
        """Create the form."""
        # create the menu
        self.m = self.new_menu(name='Menu')
        self.m.addItem(text='New', onSelect=self.new_trans, shortcut='n')
        self.m.addItem(text='Afa feature', onSelect=self.afa_feature, shortcut='a')
        self.m.addItem(text='History', onSelect=self.show_history, shortcut='h')
        self.m.addItem(text='Show presets', onSelect=self.show_presets, shortcut='p')
        self.m.addItem(text='Add to presets', onSelect=self.add_preset, shortcut='P')
        self.m.addItem(text='Settings', onSelect=self.show_settings, shortcut='s')
        self.m.addItem(text='Exit', onSelect=self.exit, shortcut='e')

        self.date = self.add_widget_intelligent(
            npyscreen.TitleDateCombo,
            name='Date:',
            begin_entry_at=22
        )
        self.state = self.add_widget_intelligent(
            npyscreen.TitleSelectOne,
            name='State:',
            begin_entry_at=22,
            values=['*', '!'],
            value=[0],
            slow_scroll=True,
            scroll_exit=True,
            max_height=2
        )
        self.code = self.add_widget_intelligent(
            npyscreen.TitleText,
            name='Code:',
            begin_entry_at=22
        )
        self.payee = self.add_widget_intelligent(
            npyscreen.TitleText,
            name='Payee',
            begin_entry_at=22
        )
        self.comments = self.add_widget_intelligent(
            TitleMultiLineEdit,
            name='Comments:',
            begin_entry_at=22,
            max_height=2
        )

        self.postings = self.add_widget_intelligent(
            TitleMultiLineEdit,
            name='Postings:',
            begin_entry_at=22,
            max_height=8
        )

        self.force_add = self.add_widget_intelligent(
            npyscreen.TitleMultiSelect,
            name='Force adding:',
            begin_entry_at=22,
            values=['enabled'],
            max_height=2,
            scroll_exit=True
        )

    def beforeEditing(self):
        """Get values from temp / settings if new."""
        # get title for form according to filename
        self.name = self.parentApp.S.gen_ledger_filename(absolute=True)

        # generate transaction from settings defaults, if tmpTrans is new
        if self.parentApp.tmpTrans_new:
            self.parentApp.gen_tmptrans()

        self.date.value = self.parentApp.tmpTrans.get_date()
        self.date.entry_widget.dateFmt = self.parentApp.S.date_fmt
        self.state.value = [self.state.values.index(self.parentApp.tmpTrans.get_state())]
        self.code.value = self.parentApp.tmpTrans.code
        self.payee.value = self.parentApp.tmpTrans.payee
        self.comments.entry_widget.values = self.parentApp.tmpTrans.get_comments()
        self.postings.entry_widget.values = acc_list_to_multiline(
            self.parentApp.tmpTrans.postings_to_lists()
        )
        self.force_add.value = (
            [0] if self.parentApp.tmpTrans.get_force_add() else []
        )

    def values_to_tmp(self, message=True):
        """Store values to temp."""
        self.parentApp.tmpTransC = Transaction()

        self.parentApp.tmpTrans.set_date(self.date.value)
        self.parentApp.tmpTransC.set_date(self.date.value)

        self.parentApp.tmpTrans.set_aux_date(self.date.value)
        self.parentApp.tmpTransC.set_aux_date(self.date.value)

        self.parentApp.tmpTrans.set_state(
            self.state.values[self.state.value[0]]
        )
        self.parentApp.tmpTransC.set_state(
            self.state.values[self.state.value[0]]
        )

        self.parentApp.tmpTrans.code = self.code.value
        self.parentApp.tmpTransC.code = self.code.value

        self.parentApp.tmpTrans.payee = self.payee.value
        self.parentApp.tmpTransC.payee = self.payee.value

        self.parentApp.tmpTrans.set_comments(
            self.comments.entry_widget.values
        )
        self.parentApp.tmpTransC.set_comments(
            [
                ledgeradd.replace(
                    text=x,
                    trans=self.parentApp.tmpTrans
                )
                for x in self.comments.entry_widget.values
            ]
        )

        # force add
        self.parentApp.tmpTrans.set_force_add(
            True if self.force_add.value == [0] else False
        )
        self.parentApp.tmpTransC.set_force_add(
            True if self.force_add.value == [0] else False
        )

        # add accounts and also add them with replaced values to the copy

        self.parentApp.tmpTrans.lists_to_postings(
            multiline_to_acc_list(
                multi=self.postings.entry_widget.values,
                dec_sep=self.parentApp.S.dec_separator
            ),
            commodity=self.parentApp.S.def_commodity
        )

        self.parentApp.tmpTransC.lists_to_postings(
            multiline_to_acc_list(
                multi=[
                    ledgeradd.replace(text=x, trans=self.parentApp.tmpTrans)
                    for x in self.postings.entry_widget.values
                ],
                dec_sep=self.parentApp.S.dec_separator
            ),
            commodity=self.parentApp.S.def_commodity
        )

        # check if accounts are valid for ledger

        # a code is given - only needed for modify feature
        if self.parentApp.tmpTrans.check()['code_exists']:
            return True

        # otherwise check things
        check_passed = True
        if self.parentApp.tmpTrans.check()['need_more_accounts']:
            if message:
                npyscreen.notify_confirm(
                    'Need at least two accounts for a valid ledger transaction.',
                    form_color='WARNING'
                )
            check_passed = False
        if self.parentApp.tmpTrans.check()['cannot_balance']:
            if message:
                npyscreen.notify_confirm(
                    'Cannot balance accounts!',
                    form_color='WARNING'
                )
            check_passed = False

        return check_passed

    def on_ok(self, keypress=None):
        """Press ok."""
        if self.values_to_tmp():

            # switch to transaction check form
            self.parentApp.setNextForm('TransactionCheck')
            self.parentApp.switchFormNow()

    def on_cancel(self, keypress=None):
        """Press cancel - exit programm."""
        self.exit()


from nonebot.default_config import *

#配置监听的端口
HOST = '127.0.0.1'
PORT = 8080

SUPERUSERS = {12345678} #配置超级用户
COMMAND_START = {''} #这里配置的是命令的起始字符

#配置机器人昵称
NICKNAME = {'小互君'}

###配置腾讯API
TX_CHAT_APPID = '2030141492' #腾讯AI的APPID
TX_CHAT_APPKEY = 'HEXah8nTIh4wO0g'#腾讯AI的APPKEY


import scrapy
import re
import os
from bs4 import BeautifulSoup
import hashlib
from .utils.postgres_connection import insert_hash_url, get_postgresql_connection
from .utils.content_parser import clean_content

class PageSpider(scrapy.Spider):
    name = "pages"
    start_urls = ["https://es.wikipedia.org/wiki/Nube_privada_virtual"]
    current_dir = os.getcwd()
    save_dir = os.path.join(current_dir, "files")  # Reemplaza con tu ruta
    connection = get_postgresql_connection()

    def parse(self, response):
        soup = BeautifulSoup(response.text, "html.parser")
        current_page_url = response.url
        links = soup.find_all("a")
        soup_title = soup.find("title")
        doc_title = soup_title.get_text() if soup_title else "Sin título"

        hash = hashlib.sha256(current_page_url.encode("UTF-8")).hexdigest()
        file_path = os.path.join(self.save_dir, hash)
        content_file = clean_content(soup, hash)
        with open(file_path, "w", encoding="utf-8") as file:
            insert_hash_url(hash, current_page_url, doc_title, self.connection)
            file.write(content_file)

        for link in links:
            href = link.get("href")
            if href:
                url_valida = self.es_url_valida(href)
                if url_valida and url_valida != current_page_url:
                    yield {
                        "from_": current_page_url,
                        "to_": url_valida,
                    }
                    yield scrapy.Request(url_valida, callback=self.parse)

    def es_url_valida(self, url):
        patrones_invalidos = [
            r"^/wiki/Portal",
            r"^/wiki/Wikipedia",
            r"^/wiki/Ayuda",
            r"^/wiki/Especial",
            r"^/wiki/Archivo",
            r"^/wiki/Categor",
        ]

        for patron in patrones_invalidos:
            if re.match(patron, url):
                return None

        patron = r"^/wiki/[^\s]+$"
        if re.match(patron, url):
            return f"https://es.wikipedia.org{url}"


from flask import Flask, Response, request, session
from twilio import twiml

import time
import sqlite3

app = Flask(__name__)
app.config.from_object(__name__)
# A secret key is required in order to store session data
secret_key = 'xxxsecretkeyxxx'

@app.route("/")
def hello():
    return "Hello"

@app.route("/twilio", methods=["POST"])
def respond_sms():
    # Grab unix time
    epoch_time = int(time.time())
    # Store the message from the request sent by twilio
    i_msg = request.form.get("Body")
    # Store the number the message was sent from
    from_number = request.values.get("From")

    # The counter keeps track of the number of messages from this number in the
    # past 4 hours. If none, default to 0
    counter = session.get('counter', 0)
    counter += 1
    session['counter'] = counter
    # Retrieve the stored state from twilio's servers. Again, default to 0
    state = session.get('state', 0)
    # Record the message in the sqlite db
    sql_write(counter, epoch_time, from_number, i_msg)

    # The default message is ERROR. There should not be a point where there is
    # fall through and this happens
    msg = "ERROR"

    # If the person texts "RESET_ZER0", reset the state to zero. This is
    # just for debugging purposes
    if "RESET_ZER0" in i_msg:
        state = 0

    if "Call" in i_msg:
        msg = "Okay! A human will call you shortly. Thank you for checking in."
        state = 200

    if state == 0:
        msg = "Hello. I see that you are currently not checked into the airport X for flight #YYY. Are you currently at the airport? Please reply with Y or N"
        state = 1

    # This is the state of responding to the inital text
    elif state == 1:
        if "Y" in i_msg:
            msg = "You have been checked in!"
            state = 200
        elif "N" in i_msg:
            msg = "Uh oh! Will you be making the flight?"
            state = 2
        else:
            msg = "I did not understand that, sorry! Please reply with either Y if you will be making the flight, or N if you will not"

    # This is the state of Not checked in
    elif state == 2:
        if "Y" in i_msg:
            msg = "Thanks! If you are not checked in within the hour, a human will be in contact with you."
            state = 200
        elif "N" in i_msg:
            msg = "I'm sorry to hear that. A human will call you shortly in order to resolve any issue"
            state = 400
        else:
            msg = "I did not understand that, sorry! Please reply with either Y if you will be making the flight, or N if you will not"

    # This is the "failure state". This means we need to involve humans
    elif state == 400:
        msg = "There has been an issue with our system. A human will be in contact with you in order to resolve any issue."
        state = 0

    # Prepare and then send a twilio SMS respond
    resp = twiml.Response()
    resp.sms(msg)

    # This is the success state. Send a thank you text and switch back to the
    # beginning state
    if state == 200:
        msg = "Thank you for flying for Southwest! If you have an unresolved issue, please contact (ZZZ)-XXX-YYYY"
        state = 0
        resp.sms(msg)

    # Store the state and time in the session cookie
    session['state'] = state
    session['time'] = epoch_time

    return str(resp)

# Write the message to the db
def sql_write(m_id, m_time, m_from, m_text, new=False):
    conn = sqlite3.connect('test.db')
    c = conn.cursor()
    if new:
        c.execute('''CREATE TABLE messages
                     (num real, time real, frm text, msg text)''')
    c.execute('''INSERT INTO messages VALUES (?, ?, ?, ?)''',
                (m_id, m_time, m_from, m_text))
    conn.commit()
    conn.close()

if __name__ == "__main__":
    app.secret_key = secret_key
    app.run(debug=True)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import numpy as np

def activation_summary(var):
    tf.summary.histogram(var.op.name+'/activation', var)

def variable_summary(var):
    tf.summary.histogram(var.op.name, var)

def batch_normal(input_ ,is_training=False, name="batch_norm"):
    return tf.layers.batch_normalization(input_, momentum=0.9, epsilon=1e-4, center=True,\
                                         scale=True, training=is_training, fused=True, name=name)

def conv2d(input_,
           output_dim,
           k_h=5, k_w=5,
           s_h=1, s_w=1,
           stddev=0.02,
           padding='SAME',
           name="conv2d"):
    with tf.variable_scope(name):
        w = tf.get_variable('weight', [k_h, k_w, input_.get_shape()[-1].value, output_dim],
                            initializer=tf.truncated_normal_initializer(stddev=stddev))
        conv = tf.nn.conv2d(input_, w, strides=[1, s_h, s_w, 1], padding=padding)

        biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0))
        conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape())
        # variable_summary(w)
        # variable_summary(biases)
        return conv

def conv3d(input_, output_dim,
           kernel=[4,4,4],
           strides=[1,1,1],
           stddev=0.02,
           padding='SAME',
           name="conv3d"):
    with tf.variable_scope(name):
        k_d, k_h, k_w = kernel
        s_d, s_h, s_w = strides
        w = tf.get_variable('weight', [k_d, k_h, k_w, input_.get_shape()[-1].value, output_dim],
                            initializer=tf.truncated_normal_initializer(stddev=stddev))
        conv = tf.nn.conv3d(input_, w, strides=[1, s_d, s_h, s_w, 1], padding=padding)

        biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0))
        conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape())
        # variable_summary(w)
        # variable_summary(biases)
        return conv

def deconv3d(input_, output_shape,
             kernel,
             strides=[2,2,2,1],
             stddev=0.02,
             padding='SAME',
             name="deconv3d"):

    with tf.variable_scope(name):
        # filter : [height, width, output_channels, in_channels]
        k_d, k_h, k_w = kernel
        w = tf.get_variable('weight', [k_d, k_h, k_w, output_shape[-1], input_.get_shape()[-1].value],
                            initializer=tf.random_normal_initializer(stddev=stddev))

        o_d, o_h, o_w, o_ch = output_shape
        s_d, s_h, s_w, s_ch = strides
        deconv = tf.nn.conv3d_transpose(input_, w, output_shape=[input_.get_shape()[0].value, o_d, o_h, o_w, o_ch],
                                        strides=[1, s_d, s_h, s_w, s_ch], padding=padding, name=name)

        biases = tf.get_variable('biases', [o_ch], initializer=tf.constant_initializer(0.0))
        deconv = tf.reshape(tf.nn.bias_add(deconv, biases), deconv.get_shape())
        # variable_summary(w)
        # variable_summary(biases)
        return deconv

def fully_connect(input_, output_size, stddev=0.02, bias_start=0.0, name=None):
  shape = input_.get_shape().as_list()

  with tf.variable_scope(name or "Linear"):

    matrix = tf.get_variable("Matrix", [shape[1], output_size], tf.float32,
                 tf.random_normal_initializer(stddev=stddev))
    bias = tf.get_variable("bias", [output_size],
      initializer=tf.constant_initializer(bias_start))
    return tf.matmul(input_, matrix) + bias

def max_pool2d(inputs,
               kernel_size,
               stride=[2, 2],
               padding='VALID',
               name='maxpool'):
  """ 2D max pooling.
  Args:
    inputs: 4-D tensor BxHxWxC
    kernel_size: a list of 2 ints
    stride: a list of 2 ints

  Returns:
    Variable tensor
  """
  with tf.variable_scope(name) as sc:
    kernel_h, kernel_w = kernel_size
    stride_h, stride_w = stride
    outputs = tf.nn.max_pool(inputs,
                             ksize=[1, kernel_h, kernel_w, 1],
                             strides=[1, stride_h, stride_w, 1],
                             padding=padding,
                             name=name)
    return outputs

def dropout(inputs,
            is_training,
            name,
            keep_prob=0.5,
            noise_shape=None):
  """ Dropout layer.

  Args:
    inputs: tensor
    is_training: boolean tf.Variable
    scope: string
    keep_prob: float in [0,1]
    noise_shape: list of ints

  Returns:
    tensor variable
  """
  with tf.variable_scope(name) as sc:
    outputs = tf.cond(tf.cast(is_training, tf.bool),
                      lambda: tf.nn.dropout(inputs, keep_prob, noise_shape),
                      lambda: inputs)
    return outputs

def instance_norm(x):

    epsilon = 1e-9

    mean, var = tf.nn.moments(x, [1, 2], keep_dims=True)

    return tf.div(tf.subtract(x, mean), tf.sqrt(tf.add(var, epsilon)))


from django.shortcuts import render,redirect, get_object_or_404
from django.http  import HttpResponse
from .models import TrackForms,NewForm
from django.contrib.auth.decorators import login_required
from .forms import NewNewFormForm
# Create your views here.
def welcome(request):
    return render(request, 'all-scoots/index.html')

def search_results(request):

    if 'referenceID' in request.GET and request.GET["referenceID"]:
        search_term = request.GET.get("referenceID")
        searched_ref = NewForm.search_by_referenceID(search_term)
        message = f"{search_term}"

        return render(request, 'all-scoots/search.html',{"message":message,"referenceID":searched_ref})

    else:
        message = "You haven't searched for any term"
        return render(request, 'all-scoots/search.html',{"message":message})
def profile(request):
    return render(request,'all-scoots/index.html')

def logout(request):
    return render(request, 'all-scoots/index.html')

@login_required(login_url='/accounts/login/')
def cargo_list(request):
    scoty = NewForm.objects.all()
    print(scoty)
    return render(request,'all-scoots/cargo.html',{"scoty":scoty})

@login_required(login_url='/accounts/login/')
def new_cargo(request):
    current_user = request.user
    if request.method == 'POST':
        form = NewNewFormForm(request.POST, request.FILES)
        if form.is_valid():
            newnewformform = form.save(commit=False)
            newnewformform .save()
    else:
        form = NewNewFormForm()
    return render(request, 'all-scoots/new_cargo.html', {"form": form})

def update_cargo(request, pk):
    instance = get_object_or_404(NewForm, pk=pk)
    form = NewNewFormForm(request.POST or None, instance=instance)
    if form.is_valid():
        form.save()
        return redirect('/')
    return render(request, 'all-scoots/update_cargo.html', {'form': form})


import os

# 设定文件夹
main_dir = 'D:\\DATA1\\MRIi\\2'

# 下级目录
for dirname in os.listdir(main_dir):
    subdir = os.path.join(main_dir, dirname)

    # 下级目录文件夹
    for filename in os.listdir(subdir):
        # 文件路径
        filepath = os.path.join(subdir, filename)

        # 重命名加上文件夹名
        new_filename = dirname + '_' + filename
        new_filepath = os.path.join(subdir, new_filename)
        os.rename(filepath, new_filepath)

        # 移动文件
        final_filepath = os.path.join(main_dir, new_filename)
        os.rename(new_filepath, final_filepath)


# -*- coding: utf-8 -*-
"""
Created on Sun Nov 26 16:06:15 2017

@author: akash
"""

#followers_count

import sys
import json

#set the path to the output file
tweets_data_path = 'D:/Third Semester/Statistics Software/Hadoop/Project/MTA/mta_3rdMarch2018.txt'

#initialize an array and open the output file for reading

tweets_file = open(tweets_data_path, "r")
tweets_data = []
#process each line in output file
for line in tweets_file:

    try:
        tweet = json.loads(line)
        (tweet['user']['verified'])
        (tweet['user']['name']), (tweet['user']['followers_count'])
        tweets_data.append(tweet)
    except:
        continue

import pandas as pd

elon_followers = pd.DataFrame()
elon_followers['name'] = map(lambda tweet: tweet['user']['name'], tweets_data)
elon_followers['followers_count'] = map(lambda tweet: tweet['user']['followers_count'], tweets_data)
elon_followers['verified'] = map(lambda tweet: tweet['user']['verified'], tweets_data)

elon_followers = elon_followers.sort_values(['followers_count'], ascending=[False])
print(elon_followers.loc[elon_followers['verified'] == True])
#print(elon_followers)


"""
给定一个整数数组 nums和一个整数目标值 target，请你在该数组中找出 和为目标值 target 的那两个整数，并返回它们的数组下标。

你可以假设每种输入只会对应一个答案。但是，数组中同一个元素在答案里不能重复出现。

你可以按任意顺序返回答案。

示例 1：

输入：nums = [2,7,11,15], target = 9
输出：[0,1]
解释：因为 nums[0] + nums[1] == 9 ，返回 [0, 1] 。
示例 2：

输入：nums = [3,2,4], target = 6
输出：[1,2]
示例 3：

输入：nums = [3,3], target = 6
输出：[0,1]

提示：

2 <= nums.length <= 104
-109 <= nums[i] <= 109
-109 <= target <= 109
只会存在一个有效答案
进阶：你可以想出一个时间复杂度小于 O(n2) 的算法吗？

作者：力扣 (LeetCode)
链接：https://leetcode.cn/leetbook/read/top-interview-questions-easy/x2jrse/
来源：力扣（LeetCode）
著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。
"""
class Solution(object):
    def twoSum(self, nums, target):
        """
        :type nums: List[int]
        :type target: int
        :rtype: List[int]
        """
        # 利用一个map解决
        this_dict = {}
        for index, num in enumerate(nums):
            if (target-num) not in this_dict:
                this_dict[num] = index
            else:
                return [index, this_dict[target-num]]


import sys
import os
from twython import Twython
from picamera import PiCamera
from time import sleep

CONSUMER_KEY = 'consumer key here'
CONSUMER_SECRET = 'consumer secret here'
ACCESS_KEY = 'access key here'
ACCESS_SECRET = 'access secret here'

#other cameras will require different code
camera = PiCamera()
camera.start_preview()
sleep(5)
camera.capture('/home/pi/image.jpg')
camera.stop_preview()

api = Twython(CONSUMER_KEY,CONSUMER_SECRET,ACCESS_KEY,ACCESS_SECRET)
photo = open('image.jpg','rb')
image_ids = api.upload_media(media=photo)
api.update_status(status='', media_ids=image_ids['media_id'])


import matplotlib.pyplot as plt
import matplotlib.cm as cm
import pickle
import numpy as np

def save_image(data, cm, fn):
    sizes = np.shape(data)
    height = float(sizes[0])
    width = float(sizes[1])

    fig = plt.figure()
    fig.set_size_inches(width/height, 1, forward=False)
    ax = plt.Axes(fig, [0., 0., 1., 1.])
    ax.set_axis_off()
    fig.add_axes(ax)

    ax.imshow(data, cmap=cm)
    plt.savefig(fn, dpi=height)
    plt.close()

with open("julia_data","rb") as f:
    julia_set = pickle.load(f)
save_image(julia_set,cm.hot,'./julia_set.png')


# importing necessary libraries
from torch import nn
import numpy as np
import joblib
import torch

# defining the model
def clp_network(trial=False, n_layers=-1, units=[]):

    custom_network = not trial and len(units) == n_layers

    if not custom_network: n_layers = trial.suggest_int("n_layers", 2, 5)
    layers = []

    in_features = 6

    for i in range(n_layers):

        if custom_network: out_features = units[i]
        else: out_features = trial.suggest_int("n_units_l{}".format(i), 4, 10)

        layers.append(nn.Linear(in_features, out_features))
        layers.append(nn.LeakyReLU())

        in_features = out_features

    layers.append(nn.Linear(in_features, 2))
    layers.append(nn.LeakyReLU())

    return nn.Sequential(*layers)

# loading the model
print('Please enter which model you want to use:')
print('\t1. Manual Model\n\t2. Automatic Model\n\t\t DEFAULT = Automatic Model')
mode = int(input())
if mode==1: model_dict_path = 'Models\\customer_loyalty_prediction.pt'
else: model_dict_path = 'Models\\customer_loyalty_prediction_autp.pt'
model_dict = torch.load(model_dict_path)

# extracting network architecture data
architecture = model_dict['architecture']
n_layers = architecture['n_layers']
units = []
for i in range(n_layers):
    units.append(architecture["n_units_l{}".format(i)])

# setting up the model
model = clp_network(False, n_layers, units)
model.load_state_dict(model_dict['model_state_dict'])
model.eval()

# loading the scaler model
data_scaler = joblib.load('Models\\data_scaler.gz')

# getting input from user
print('Please enter the following values:')
print('Debt1:', end='\t')
debt1 = float(input())
print('Credit:', end='\t')
credit = float(input())
print('Age:', end='\t')
age = float(input())
print('Work Experience:', end='\t')
exp = float(input())
print('Education:', end='\t')
edu = float(input())
print('Time of Residency:', end='\t')
red = float(input())

# processing the data
data = np.array([debt1, credit, age, edu, exp, red])
data = data.reshape(1, -1)
data = torch.tensor(data_scaler.transform(data))

# feeding the data to model
with torch.no_grad():
    output = model(data.float())

# outputing the predicted value
output = output.argmax(dim=1).sum().item()
print(output)


import json
import pickle
import datasets
from datasets.tasks import QuestionAnsweringExtractive
from numpy import int64

logger = datasets.logging.get_logger(__name__)

_CITATION = """\
@inproceedings{rurebus,
  Address = {Moscow, Russia},
  Author = {Ivanin, Vitaly and Artemova, Ekaterina and Batura, Tatiana and Ivanov, Vladimir and Sarkisyan, Veronika and Tutubalina, Elena and Smurov, Ivan},
  Title = {RuREBus-2020 Shared Task: Russian Relation Extraction for Business},
  Booktitle = {Computational  Linguistics  and  Intellectual  Technologies:  Proceedings of the International Conference “Dialog” [Komp’iuternaia Lingvistika  i  Intellektual’nye  Tehnologii:  Trudy  Mezhdunarodnoj  Konferentsii  “Dialog”]},
  Year = {2020}
}
"""

_DESCRIPTION = """\
RuREBus dataset is tokenized for NER task with sbert_large_nlu_ru  model\
"""

_URL = ""
_URLS = {
    "train": "/content/train_instances.json", # путь до train в директории
    "test": "/content/test_instances.json", # путь до test в директории
}

class RuREBus(datasets.GeneratorBasedBuilder):
    """RuREBus: RuREBus labeled for bert_large_nlu_ru model"""

    def _info(self):
        return datasets.DatasetInfo(
            description=_DESCRIPTION,
            features=datasets.Features(
                        {   "id": datasets.Value("string"),
                            "ner_tags": datasets.Sequence(datasets.Value("int32")),
                            "tokens": datasets.Sequence(datasets.Value("string")),
                        }
            ),

            # No default supervised_keys (as we have to pass both question
            # and context as input).
            supervised_keys=None,
            homepage="https://github.com/dialogue-evaluation/RuREBus",
            citation=_CITATION,

        )

    def _split_generators(self, dl_manager):
        print(_URLS)
        downloaded_files = dl_manager.download_and_extract(_URLS)

        return [
            datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepath": downloaded_files["train"]}),
            datasets.SplitGenerator(name=datasets.Split.TEST, gen_kwargs={"filepath": downloaded_files["test"]}),
        ]

    def _generate_examples(self, filepath):
        """This function returns the examples in the raw (text) form."""
        logger.info("generating examples from = %s", filepath)

        with open(filepath, encoding='utf-8') as f:
            print(filepath)
            dct = json.load(f)
            id_ = 0

            for ner_tags, tokens in zip(dct['label'], dct['sentence']):

                yield id_, {
                            "id": str(id_),
                            "ner_tags": ner_tags,
                            "tokens": tokens,
                              }

                id_ += 1


import tkinter as tk
import random
def moveNum(nTag):              #移动被鼠标点击的数字，参数是在Canvas中字符对象的tag
    global X0,Y0,w,gameOver     #空位所在位置的坐标(X0,Y0)
    x,y=w.coords(nTag)          #得到被点击的数字所在位置的坐标
    if abs(x-X0)+abs(y-Y0)==60 and gameOver==False:  #判断被单击数字是否和空白处相邻
        w.move(nTag,X0-x,Y0-y)  #如果和空白处相邻,移动数字到空白处,第2,3参数是移动增量
        X0,Y0=x,y                           #记录新空白处位置坐标
        numbers=list('087654321')
        for y in range(90,270,60):           #y为Canvas的y坐标,90,150,210
            for x in range(90,270,60):       #x为Canvas的x坐标,90,150,210
                m=numbers.pop()
                if m!= '0':                 #如m!= '0'，8个数字还未检查完
#tag='1'字符对象显示1，在(90,90)位置正确，tag='2'字符对象显示2，在(150,90)处正确，等等
                    if [x,y]!=w.coords("A"+m):  #如相应字符对象位置不正确，玩家未赢
                        return
                else:                       #如m=='0'，8个数字所在位置正确，玩家赢了
                    gameOver=True
                    label['text']="你赢了"
root = tk.Tk()                                #初始化窗口
root.title('数字华容道_用Canvas实现')          #窗口标题
root.geometry("300x260+200+20")#窗口宽300,高=2600,窗口左上点离屏幕左边界200,离屏幕上边界距离20。
root.resizable(width=False,height=False) #设置窗口是否可变，这里宽不可变，高不可变，默认为True
w = tk.Canvas(root, width = 300, height = 260, background = "#D2B48C")        #建立Canvas对象
w.pack()
gameOver=False
numbers=list('123456780')
random.shuffle(numbers)             #使列表数字和空白随机排列
label=tk.Label(w,text='单击数字移动方块',fg='#000000',font=("Arial",15), background = "#D2B48C")
label.place(x=20,y=10,width=250,height=40)
X0=Y0=0                                #空位所在位置的坐标(X0,Y0)
for x in range(60,300,60):          #画棋盘4条竖线,x为Canvas的x坐标
    w.create_line(x,60,x,240,fill = "#FFE4B5",width = 3)
for y in range(60,300,60):          #画棋盘4条横线,y为Canvas的y坐标
    w.create_line(60,y,240,y,fill = "#FFE4B5",width = 3)
for y in range(90,270,60):           #y为Canvas的y坐标,90,150,210
    for x in range(90,270,60):       #x为Canvas的x坐标,90,150,210
        n=numbers.pop()
        if n!='0':#如n!='0',建立8个数字对象,注意tag有2个值,可通过"R"操作所有数字对象,如删除所有数字
            w.create_text(x,y,text=n,fill="#000000",tag=("A"+n,"R"),font=("Arial",40))#tag=A1,显示1
            def leftClick(event,nTag="A"+n): #每个按钮都生成一个事件函数，参数默认值是该数字的tag
                moveNum(nTag)            #所有事件函数都调用同一函数
            w.tag_bind("A"+n,'<Button-1>',leftClick)        #绑定左键单击事件
        else:                           #n=0,代表空白，记住无数字的空位的Canvas坐标(x,y)
            X0,Y0=x,y
root.mainloop()


import FWCore.ParameterSet.Config as cms

from CommonTools.ParticleFlow.Isolation.tools_cfi import isoDepositReplace

def load_muonPFiso_sequence(proc, seq_name, algo, coneR, src, src_charged_hadron='', src_neutral_hadron='', src_photon='', src_charged_pileup='',
                            veto_charged_hadron='Threshold(0.0)', veto_neutral_hadron='Threshold(0.5)', veto_photon='Threshold(0.5)', veto_charged_pileup='Threshold(0.5)' ):

    doCH, doNH, doPh, doPU = False, False, False, False
    if src_charged_hadron != '': doCH = True
    if src_neutral_hadron != '': doNH = True
    if src_photon         != '': doPh = True
    if src_charged_pileup != '': doPU = True

    iso_seq = cms.Sequence()

    if doCH:
        setattr(proc, 'muPFIsoDepositCH'+algo, isoDepositReplace(src, src_charged_hadron))
        iso_seq += getattr(proc, 'muPFIsoDepositCH'+algo)

    if doNH:
        setattr(proc, 'muPFIsoDepositNH'+algo, isoDepositReplace(src, src_neutral_hadron))
        iso_seq += getattr(proc, 'muPFIsoDepositNH'+algo)

    if doPh:
        setattr(proc, 'muPFIsoDepositPh'+algo, isoDepositReplace(src, src_photon))
        iso_seq += getattr(proc, 'muPFIsoDepositPh'+algo)

    if doPU:
        setattr(proc, 'muPFIsoDepositPU'+algo, isoDepositReplace(src, src_charged_pileup))
        iso_seq += getattr(proc, 'muPFIsoDepositPU'+algo)

    iso_vals_seq = cms.Sequence()

    if doCH:
        setattr(proc, 'muPFIsoValueCH'+algo,
          cms.EDProducer('CandIsolatorFromDeposits',
            deposits = cms.VPSet(
              cms.PSet(
                src = cms.InputTag('muPFIsoDepositCH'+algo),
                deltaR = cms.double(coneR),
                weight = cms.string('1'),
                vetos = cms.vstring('0.0001',veto_charged_hadron),
                skipDefaultVeto = cms.bool(True),
                mode = cms.string('sum')
              )
            )
          )
        )
        iso_vals_seq += getattr(proc, 'muPFIsoValueCH'+algo)

    if doNH:
        setattr(proc, 'muPFIsoValueNH'+algo,
          cms.EDProducer('CandIsolatorFromDeposits',
            deposits = cms.VPSet(
              cms.PSet(
                src = cms.InputTag('muPFIsoDepositNH'+algo),
                deltaR = cms.double(coneR),
                weight = cms.string('1'),
                vetos = cms.vstring('0.01', veto_neutral_hadron ),
                skipDefaultVeto = cms.bool(True),
                mode = cms.string('sum')
              )
            )
          )
        )
        iso_vals_seq += getattr(proc, 'muPFIsoValueNH'+algo)

    if doPh:
        setattr(proc, 'muPFIsoValuePh'+algo,
          cms.EDProducer('CandIsolatorFromDeposits',
            deposits = cms.VPSet(
              cms.PSet(
                src = cms.InputTag('muPFIsoDepositPh'+algo),
                deltaR = cms.double(coneR),
                weight = cms.string('1'),
                vetos = cms.vstring('0.01', veto_photon),
                skipDefaultVeto = cms.bool(True),
                mode = cms.string('sum')
              )
            )
          )
        )
        iso_vals_seq += getattr(proc, 'muPFIsoValuePh'+algo)

    if doPU:
        setattr(proc, 'muPFIsoValuePU'+algo,
          cms.EDProducer('CandIsolatorFromDeposits',
            deposits = cms.VPSet(
              cms.PSet(
                src = cms.InputTag('muPFIsoDepositPU'+algo),
                deltaR = cms.double(coneR),
                weight = cms.string('1'),
                vetos = cms.vstring('0.01', veto_charged_pileup ),
                skipDefaultVeto = cms.bool(True),
                mode = cms.string('sum')
              )
            )
          )
        )
        iso_vals_seq += getattr(proc, 'muPFIsoValuePU'+algo)

    iso_seq *= iso_vals_seq

    setattr(proc, seq_name, iso_seq)

def load_pfParticle_sequence(proc, withLeptons):

    from CommonTools.PileupAlgos.Puppi_cff import puppi

    if withLeptons:

        puppiR05 = puppi.clone()
        setattr(proc, "puppiR05", puppiR05)

        pfAllPhotonsPuppi = cms.EDFilter("CandPtrSelector", src = cms.InputTag("puppiR05"), cut = cms.string("pdgId == 22"))
        pfAllNeutralHadronsPuppi = cms.EDFilter("CandPtrSelector", src = cms.InputTag("puppiR05"), cut = cms.string("pdgId == 111 || pdgId == 130 || pdgId == 310 || pdgId == 2112"))
        pfAllChargedHadronsPuppi = cms.EDFilter("CandPtrSelector", src = cms.InputTag("puppiR05"), cut = cms.string("pdgId == 211 || pdgId == -211 || pdgId == 321 || pdgId == -321 || pdgId == 999211 || pdgId == 2212 || pdgId == -2212"))
        setattr(proc, "pfAllPhotonsPuppi", pfAllPhotonsPuppi)
        setattr(proc, "pfAllNeutralHadronsPuppi", pfAllNeutralHadronsPuppi)
        setattr(proc, "pfAllChargedHadronsPuppi", pfAllChargedHadronsPuppi)

        pfParticlesForPUPPI = cms.Sequence()
        pfParticlesForPUPPI += getattr(proc, "puppiR05")
        pfParticlesForPUPPI += getattr(proc, "pfAllPhotonsPuppi")
        pfParticlesForPUPPI += getattr(proc, "pfAllNeutralHadronsPuppi")
        pfParticlesForPUPPI += getattr(proc, "pfAllChargedHadronsPuppi")

        return pfParticlesForPUPPI

    else :

        packedPFCandidatesNoLep = cms.EDFilter("CandPtrSelector", src = cms.InputTag("particleFlow"), cut = cms.string("abs(pdgId) != 13 && abs(pdgId) != 11 "))
        setattr(proc, "packedPFCandidatesNoLep", packedPFCandidatesNoLep)

        puppiR05NoLep = puppi.clone(
                                               candName = 'packedPFCandidatesNoLep'
                                               )
        setattr(proc, "puppiR05NoLep", puppiR05NoLep)

        pfAllPhotonsPuppiNoLep = cms.EDFilter("CandPtrSelector", src = cms.InputTag("puppiR05NoLep"), cut = cms.string("pdgId == 22"))
        pfAllNeutralHadronsPuppiNoLep = cms.EDFilter("CandPtrSelector", src = cms.InputTag("puppiR05NoLep"), cut = cms.string("pdgId == 111 || pdgId == 130 || pdgId == 310 || pdgId == 2112"))
        pfAllChargedHadronsPuppiNoLep = cms.EDFilter("CandPtrSelector", src = cms.InputTag("puppiR05NoLep"), cut = cms.string("pdgId == 211 || pdgId == -211 || pdgId == 321 || pdgId == -321 || pdgId == 999211 || pdgId == 2212 || pdgId == -2212"))
        setattr(proc, "pfAllPhotonsPuppiNoLep", pfAllPhotonsPuppiNoLep)
        setattr(proc, "pfAllNeutralHadronsPuppiNoLep", pfAllNeutralHadronsPuppiNoLep)
        setattr(proc, "pfAllChargedHadronsPuppiNoLep", pfAllChargedHadronsPuppiNoLep)

        pfParticlesForPUPPINoLep = cms.Sequence()
        pfParticlesForPUPPINoLep += getattr(proc, "packedPFCandidatesNoLep")
        pfParticlesForPUPPINoLep += getattr(proc, "puppiR05NoLep")
        pfParticlesForPUPPINoLep += getattr(proc, "pfAllPhotonsPuppiNoLep")
        pfParticlesForPUPPINoLep += getattr(proc, "pfAllNeutralHadronsPuppiNoLep")
        pfParticlesForPUPPINoLep += getattr(proc, "pfAllChargedHadronsPuppiNoLep")

        return pfParticlesForPUPPINoLep

def load_fullPFpuppiIsolation(proc):
    proc.pfParticlesPUPPILeptons = load_pfParticle_sequence(proc, True)

    muon_src, cone_size = 'probeMuons', 0.4
    load_muonPFiso_sequence(proc, 'MuonPFIsoSequencePUPPI', algo = 'R04PUPPI',
                            src = muon_src,
                            src_charged_hadron = 'pfAllChargedHadronsPuppi',
                            src_neutral_hadron = 'pfAllNeutralHadronsPuppi',
                            src_photon         = 'pfAllPhotonsPuppi',
                            veto_charged_hadron='Threshold(0.0)',
                            veto_neutral_hadron='Threshold(0.0)',
                            veto_photon='Threshold(0.0)',
                            coneR = cone_size
                            )

    proc.pfParticlesPUPPINoLeptons = load_pfParticle_sequence(proc, False)
    load_muonPFiso_sequence(proc, 'MuonPFIsoSequencePUPPINoLep', algo = 'R04PUPPINoLep',
                            src = muon_src,
                            src_charged_hadron = 'pfAllChargedHadronsPuppiNoLep',
                            src_neutral_hadron = 'pfAllNeutralHadronsPuppiNoLep',
                            src_photon         = 'pfAllPhotonsPuppiNoLep',
                            veto_charged_hadron='Threshold(0.0)',
                            veto_neutral_hadron='Threshold(0.0)',
                            veto_photon='Threshold(0.0)',
                            coneR = cone_size
                            )

    proc.loadPUPPIisoInValueMaps = cms.EDProducer("AnyNumbersToValueMaps",
                                                  collection = cms.InputTag("probeMuons"),
                                                  associations = cms.VInputTag(cms.InputTag("muPFIsoValueCHR04PUPPI"), cms.InputTag("muPFIsoValueNHR04PUPPI"), cms.InputTag("muPFIsoValuePhR04PUPPI"),cms.InputTag("muPFIsoValueCHR04PUPPINoLep"), cms.InputTag("muPFIsoValueNHR04PUPPINoLep"), cms.InputTag("muPFIsoValuePhR04PUPPINoLep")),
    )
    proc.fullPuppIsolationSequence = cms.Sequence(proc.pfParticlesPUPPILeptons+proc.MuonPFIsoSequencePUPPI+proc.pfParticlesPUPPINoLeptons+proc.MuonPFIsoSequencePUPPINoLep+proc.loadPUPPIisoInValueMaps )

    return proc.fullPuppIsolationSequence


import sys
import pandas as pd

def solver(*args):
    freqs = [0]
    acc = 0
    duplicate = False
    iteration = 0
    while not duplicate:
        iteration += 1
        for arg in args:
            acc += int(arg)
            freqs.append(acc)
        if  iteration % 144  is 0:
            print(f"Iteration: {iteration}")
            tmp = pd.DataFrame(freqs)

            dupes = tmp[tmp.duplicated(keep='first')]

            if len(dupes) > 0:
                duplicate = True
                print(f'dupes: {len(dupes)}')
                acc = dupes.iloc[0,0]
                print(f'all freq {len(freqs)}')
            else:
                print("nothing")
                dupes = []
    return acc

print(solver(*sys.argv[1:]))


import numpy as np
import tensorflow as tf

def one_hot(v):
    """
    Return a one-hot numpy array of the given tensor of size 8
    :param v: array to be transformed
    :return: one-hot numpy array of given shape
    """
    v = tf.one_hot(v, 8)
    nump = v.numpy()
    nump = np.reshape(nump, (1, 64, 64, 64, 8))
    return nump


#!/usr/bin/python3
'''
A perfect number is a number for which the sum of its proper divisors is
exactly equal to the number.  For example, the sum of the proper divisors of 28
would be 1 + 2 + 4 + 7 + 14 = 28, which means that 28 is a perfect number.

A number n is called deficient if the sum of its proper divisors is less than
n and it is called abundant if this sum exceeds n.

As 12 is the smallest abundant number, 1 + 2 + 3 + 4 + 6 = 16, the smallest
number that can be written as the sum of two abundant numbers is 24.  By
mathematical analysis, it can be shown that all integers greater than 28123 can
be written as the sum of two abundant numbers.  However, this upper limit
cannot be reduced any further by analysis even though it is known that the
greatest number that cannot be expressed as the sum of two abundant numbers is
less than this limit.

Find the sum of all the positive integers which cannot be written as the sum of
two abundant numbers.
'''

from helpers import print_memory_usage_report, print_time_elapsed
from time import time
import tracemalloc

from math import floor, sqrt
from typing import List

# Every number >= this number can be written as the sum of two abundant integers
MIN_GUARANTEE_SUM_OF_TWO = 28124

def _find_sum_of_proper_divisors(num: int) -> int:
    '''Find the sum of a number's proper divisors.'''
    divisors = {1}
    for divisor in range(2, floor(sqrt(num)) + 1):
        if num % divisor == 0:
            divisors.add(divisor)
            divisors.add(num / divisor)

    return sum(divisors)

def _find_all_abundant_numbers() -> List[int]:
    '''Get a list of all relevant abundant numbers.'''
    return [
        num
        for num in range(1, MIN_GUARANTEE_SUM_OF_TWO)
        if _find_sum_of_proper_divisors(num) > num
    ]

def _can_be_written_as_sum_of_list_pair(
    num: int,
    num_list: List[int]
) -> bool:
    '''Can a given number be written as the sum of two numbers from a list?

    Args:
        num: The number to check
        num_list: The *sorted* list of candidate addends
    '''
    low, high = 0, len(num_list) - 1
    while low <= high:
        pair_sum = num_list[low] + num_list[high]
        if pair_sum == num:
            return True
        elif pair_sum > num:
            high -= 1
        else:
            low += 1

    return False

def main():
    # Keep track of time elapsed and memory used
    start_time = time()
    tracemalloc.start()

    # ********** Solution begins here ********** #

    sum = 0
    abundant_nums = _find_all_abundant_numbers()
    for num in range(1, MIN_GUARANTEE_SUM_OF_TWO):
        if not _can_be_written_as_sum_of_list_pair(num, abundant_nums):
            sum += num

    print(f'Sum of integers non-writable as sums of abundants:\n\n\t{sum}\n')

    # ********** Solution ends here ********** #

    # Stop tracking time and memory
    snapshot = tracemalloc.take_snapshot()
    end_time = time()
    tracemalloc.stop()

    # Print time elapsed and memory used
    print_time_elapsed(start_time, end_time)
    print_memory_usage_report(snapshot)

if __name__ == '__main__':
    main()


import os

# import pandas as pd
# import numpy as np

# import sqlalchemy
# from sqlalchemy.ext.automap import automap_base
# from sqlalchemy.orm import Session
# from sqlalchemy import create_engine

from flask import Flask, jsonify, render_template
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)

@app.route("/")
def Home():
    """Return the homepage."""
    return render_template("index.html")

@app.route("/World Map")
def WorldMap():
     """Movie Counting by Countries"""
     return render_template("map1.html")

@app.route("/Profits")
def Profits():
    """Profits Analysis"""
    return render_template("profit.html")

if __name__ == "__main__":
    app.run(debug=True)


class Solution:
    def cutOffTree(self, forest: List[List[int]]) -> int:
        '''
        0 => 不能走
        1 => 可以走
        >1=> 有樹

        2500 cells => N

        [
            [54581641, 64080174, 1, 69107959],
            [86374198, 61363882, 68783324, 79706116],
            [1, 92178815, 89819108, 94701471],
            [83920491, 1, 1, 1],
            [89078499, 1, 1, 60813308]]

        找此點->下一個目標的座標 的最短距離
        # BFS => N * N => 6.25 * 10^6
        def shortest_path(i1,j1,i2,j2):
            return 最短距離
        '''
        # n*n + 2n + nlogn = 6.263 * 10^6
        height, width = len(forest), len(forest[0])

        def shortest_path(i1,j1,i2,j2):
            que = collections.deque()
            que.append((i1, j1)) # x, y
            done_set = set()
            done_set.add((i1, j1))

            step = 0
            while que:
                que2 = collections.deque()

                for x, y in que:
                    if x == i2 and y == j2:
                        return step

                    for i, j in [x-1,y], [x+1,y], [x,y-1], [x,y+1]:
                        if not (0 <= i < height and 0 <= j < width):
                            continue
                        if forest[i][j] == 0:
                            continue
                        if (i, j) in done_set:
                            continue
                        que2.append((i, j))
                        done_set.add((i, j))

                que = que2
                step += 1
            return -1

        # O(n)
        height_arr = []
        for i in range(height):
            for j in range(width):
                height_arr.append((forest[i][j], i, j))
        # O(nlogn)
        height_arr.sort()

        # 從第一棵樹的idx開始
        # O(n)
        idx = 0
        while idx < len(height_arr):
            if height_arr[idx][0] > 1:
                break
            idx += 1

        # O(n * n)
        total_steps = 0
        cur_x = cur_y = 0
        while idx < len(height_arr):
            result = shortest_path(
                cur_x,
                cur_y,
                height_arr[idx][1],
                height_arr[idx][2]
            )

            if result == -1:
                print(forest)
                print(cur_x, cur_y, height_arr[idx][1], height_arr[idx][2])
                return -1

            total_steps += result
            cur_x, cur_y = height_arr[idx][1], height_arr[idx][2]
            forest[cur_x][cur_y] = 1
            idx += 1
        return total_steps


#!/bin/python3

import math
import os
import random
import re
import sys

def Swap(w, i, j):
    tmp = w[i]
    w[i] = w[j]
    w[j] = tmp

# Complete the biggerIsGreater function below.
def biggerIsGreater(w):
    ww = list(w)
    i = len(ww)-2
    while i >= 0 and ww[i] >= ww[i+1]:
        i -=1

    if i == -1:
        return 'no answer'

    j = len(ww)-1
    while ww[i] >= ww[j]:
        j -= 1

    # Swap
    ww[i], ww[j] = ww[j], ww[i]

    # Reorder
    ww[i+1:] = ww[len(ww): i: -1]

    return ''.join(ww)

if __name__ == '__main__':
    fptr = open(os.environ['OUTPUT_PATH'], 'w')

    T = int(input())

    for T_itr in range(T):
        w = input()

        result = biggerIsGreater(w)

        fptr.write(result + '\n')

    fptr.close()


import pandas as pd
from itertools import islice
import csv

Sourcedataframe = pd.read_csv("blankAgentList.csv", header=None)
count = 1
count2 = 0

for index, row in Sourcedataframe.iterrows(): #going through each row line by line

    #this for loop counts the amount of times it has gone through the csv file. If it has gone through it more than three times, it resets the counter back to 1.
    for column in Sourcedataframe:
        if count > 3:
            count = 1

            #if program is on it's first count, it opens the 'Sourcedataframe', reads/writes every third row to a new csv file named 'agentList1.csv'.
        if count == 1:
            with open('blankAgentList.csv') as infile:

              with open('agentList1.csv', 'w') as outfile:
                reader = csv.DictReader(infile)
                writer = csv.DictWriter(outfile, fieldnames=reader.fieldnames)
                writer.writeheader()
                for row in reader:
                    count2 += 1
                    if not count2 % 3:
                        writer.writerow(row)

        elif count == 2:
            with open('blankAgentList.csv') as infile:

              with open('agentList2.csv', 'w') as outfile:
                reader = csv.DictReader(infile)
                writer = csv.DictWriter(outfile, fieldnames=reader.fieldnames)
                #Sourcedataframe.iloc[2, 0]
                writer.writeheader()
                for row in reader:
                    count2 += 1
                    if not count2 % 3:
                        writer.writerow(row)

        elif count == 3:
            with open('blankAgentList.csv') as infile:

              with open('agentList3.csv', 'w') as outfile:
                reader = csv.DictReader(infile)
                writer = csv.DictWriter(outfile, fieldnames=reader.fieldnames)
                writer.writeheader()
                for row in reader:
                    count2 += 1
                    if not count2 % 3:
                        writer.writerow(row)

        count = count + 1 #counts how many times it has ran through the main for loop.


import falcon_kit.mains.fasta_filter as mod
import pytest
from falcon_kit import FastaReader
import functools
import helpers
import pytest
import os
import sys
import io
import json

def test_help():
    try:
        mod.main(['prog', '--help'])
    except SystemExit:
        pass

#############################
### Define the test data. ###
#############################
fasta_tests = {}
expected_tests = {}
expected_general_median_tests = {}
expected_internal_median_tests = {}
expected_longest_tests = {}

fasta_tests[1] = ''
expected_tests[1] = ''

fasta_tests[2] = """\
>synthetic/1/0_500
GATTACA
>synthetic/2/0_500
GATTACA
>synthetic/3/0_500
GATTACA
>synthetic/4/0_500
GATTACA
>synthetic/5/0_500
GATTACA
"""
expected_tests[2] = fasta_tests[2]

fasta_tests[3] = """\
>synthetic/1/0_500
GATTACAGATTACA
>synthetic/1/0_500
GATTACAGATTACAGATTACAGATTACA
>synthetic/1/0_500
GATTACAGATTACAGATTACA
>synthetic/1/500_1000
GATTACAGATTACAGATTACAGATTACA
>synthetic/1/0_500
GATTACA
"""
expected_tests[3] = """\
>synthetic/1/0_500
GATTACAGATTACAGATTACA
"""

expected_longest_tests[3] = """\
>synthetic/1/0_500
GATTACAGATTACAGATTACAGATTACA
"""

fasta_tests[4] = """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/1/0_5
ACGTA
>synthetic/1/5_7
AC
>synthetic/1/7_10
ACG
>synthetic/1/10_14
ACGT
>synthetic/1/14_15
A
>synthetic/4/0_1
A
>synthetic/5/0_2
AC
>synthetic/5/2_5
ACG
>synthetic/5/5_9
ACGT
>synthetic/5/9_14
ACGTA
"""
expected_tests[4] = """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/1/7_10
ACG
>synthetic/4/0_1
A
>synthetic/5/5_9
ACGT
"""

fasta_tests[5] = """\
@synthetic/1/0_500
A
@synthetic/2/0_500
A
@synthetic/3/0_500
A
@synthetic/4/0_500
A
@synthetic/5/0_500'
A
"""
# That is invalid fasta.
expected_tests[5] = None    # The module under test should throw on the above input

fasta_tests[6] = """\
>synthetic/5/2_5
ACG
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/1/0_5
ACGTA
>synthetic/1/10_14
ACGT
>synthetic/1/14_15
A
>synthetic/4/0_1
A
>synthetic/5/0_2
AC
>synthetic/5/5_9
ACGT
>synthetic/5/9_14
ACGTA
>synthetic/1/5_7
AC
>synthetic/1/7_10
ACG
"""
expected_tests[6] = """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/4/0_1
A
>synthetic/5/5_9
ACGT
>synthetic/1/7_10
ACG
"""

# Test ccs input
fasta_tests[8] = """\
>synthetic/1/ccs
GATTACA
"""
expected_tests[8] = """\
>synthetic/1/0_7
GATTACA
"""

# Specific for the internal-median command.
fasta_tests[7] = """\
>synthetic/1/0_3
AAA
>synthetic/1/3_7
ACAC
>synthetic/1/7_10
AAA
>synthetic/2/0_3
TTT
>synthetic/2/3_8
TTTGG
>synthetic/3/0_5
TTTGG
>synthetic/3/5_8
TTT
>synthetic/4/0_5
CCCCC
"""
expected_general_median_tests[7] = """\
>synthetic/1/7_10
AAA
>synthetic/2/3_8
TTTGG
>synthetic/3/0_5
TTTGG
>synthetic/4/0_5
CCCCC
"""
expected_internal_median_tests[7] = """\
>synthetic/1/3_7
ACAC
>synthetic/2/3_8
TTTGG
>synthetic/3/0_5
TTTGG
>synthetic/4/0_5
CCCCC
"""

whitelist_tests_general_median = [
    {   'fasta': fasta_tests[4],
        'whitelist': set(),
        'expected': expected_tests[4]
    },
    {   'fasta': fasta_tests[4],
        'whitelist': None,
        'expected': expected_tests[4]
    },
    {   'fasta': fasta_tests[4],
        'whitelist': set(['synthetic/2', 'synthetic/4', 'synthetic/3']),
        'expected': """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/4/0_1
A
"""
    },
    {   'fasta': fasta_tests[4],
        'whitelist': set(['synthetic/2', 'synthetic/3', 'synthetic/1', 'synthetic/4', 'synthetic/5']),
        'expected': expected_tests[4]
    },
]

whitelist_tests_longest = [
    {   'fasta': fasta_tests[4],
        'whitelist': set(),
        'expected': """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/1/0_5
ACGTA
>synthetic/4/0_1
A
>synthetic/5/9_14
ACGTA
"""
    },
    {   'fasta': fasta_tests[4],
        'whitelist': None,
        'expected': """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/1/0_5
ACGTA
>synthetic/4/0_1
A
>synthetic/5/9_14
ACGTA
"""
    },
    {   'fasta': fasta_tests[4],
        'whitelist': set(['synthetic/2', 'synthetic/4', 'synthetic/3']),
        'expected': """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/4/0_1
A
"""
    },
    {   'fasta': fasta_tests[4],
        'whitelist': set(['synthetic/2', 'synthetic/3', 'synthetic/1', 'synthetic/4', 'synthetic/5']),
        'expected': """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/1/0_5
ACGTA
>synthetic/4/0_1
A
>synthetic/5/9_14
ACGTA
"""
    },
]

whitelist_tests_pass = [
    {   'fasta': fasta_tests[4],
        'whitelist': set(),
        'expected': fasta_tests[4]
    },
    {   'fasta': fasta_tests[4],
        'whitelist': None,
        'expected': fasta_tests[4]
    },
    {   'fasta': fasta_tests[4],
        'whitelist': set(['synthetic/2', 'synthetic/1', 'synthetic/3']),
        'expected': """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/1/0_5
ACGTA
>synthetic/1/5_7
AC
>synthetic/1/7_10
ACG
>synthetic/1/10_14
ACGT
>synthetic/1/14_15
A
"""
    },
    {   'fasta': fasta_tests[4],
        'whitelist': set(['synthetic/2', 'synthetic/3', 'synthetic/1', 'synthetic/4', 'synthetic/5']),
        'expected': fasta_tests[4]
    },
]

#########################
### Helper functions. ###
#########################
def check_run(func, fasta, expected, whitelist_set=None):
    if expected is None:
        return check_run_raises(func, fasta, whitelist_set)
    fp_in = io.StringIO(fasta)
    fp_out = io.StringIO()
    func(fp_in, fp_out, whitelist_set)
    if expected != fp_out.getvalue():
        print((fp_out.getvalue()))
    assert expected == fp_out.getvalue()

def check_run_raises(func, fasta, whitelist_set=None):
    fp_in = io.StringIO(fasta)
    fp_out = io.StringIO()
    with pytest.raises(Exception) as e:
        func(fp_in, fp_out, whitelist_set)
    assert not isinstance(e.value, AssertionError) # Test assertions elsewhere.

def check_main_from_file(cmd, fasta, expected, in_fa_file, capsys):
    in_fa_file.write(fasta)
    argv = ['prog', cmd, str(in_fa_file)]
    mod.main(argv)
    out, err = capsys.readouterr()
    if expected != out:
        print(out)
    assert(out == expected)

def check_main_from_file_with_whitelist(cmd, fasta, expected, in_whitelist_file, in_fa_file, capsys):
    in_fa_file.write(fasta)
    argv = ['prog', '--zmw-whitelist-fn', in_whitelist_file, cmd, str(in_fa_file)]
    mod.main(argv)
    out, err = capsys.readouterr()
    if expected != out:
        print(out)
    assert(out == expected)

def check_main_from_stdin(cmd, fasta, expected, capsys):
    mod.sys.stdin = io.StringIO(fasta)
    argv = ['prog', cmd, '-']
    mod.main(argv)
    out, err = capsys.readouterr()
    if expected != out:
        print(out)
    assert(out == expected)

def check_main_from_stdin_with_whitelist(cmd, fasta, expected, in_whitelist_file, capsys):
    mod.sys.stdin = io.StringIO(fasta)
    argv = ['prog', '--zmw-whitelist-fn',  in_whitelist_file, cmd, '-']
    mod.main(argv)
    out, err = capsys.readouterr()
    if expected != out:
        print(out)
    assert(out == expected)

#############################
## Test internal functions ##
#############################
def test_yield_record_and_tokenized_headers():
    # We could delete this test, since we have whitelist tests also.
    # Or we could expand this.
    fp_in = io.StringIO(fasta_tests[1])
    records = FastaReader.yield_fasta_record(fp_in, log=lambda x: None)
    w = set()
    assert [] == list(mod.yield_record_and_tokenized_headers(w, records))

def test_load_zmw_whitelist(tmpdir, caplog):
    fn = ''
    expected = set()
    assert mod.load_zmw_whitelist(fn) == expected
    assert not caplog.records

    path = tmpdir.join('whitelist_empty.json')
    path.write('') # empty
    fn = str(path)
    expected = set()
    assert mod.load_zmw_whitelist(fn) == set()
    assert 'Assuming empty whitelist' in caplog.text
    caplog.clear()

    path = tmpdir.join('whitelist_empty_list.json')
    path.write('[]') # empty
    fn = str(path)
    expected = set()
    assert mod.load_zmw_whitelist(fn) == expected
    assert not caplog.records

    redundant = ['aa/1', 'bb/2', 'aa/1']
    path = tmpdir.join('whitelist_redundant_list.json')
    path.write(json.dumps(redundant))
    fn = str(path)
    expected = set(['aa/1', 'bb/2'])
    assert mod.load_zmw_whitelist(fn) == expected
    assert not caplog.records

#############################
### Test the PASS filter. ###
#############################
@pytest.mark.parametrize('case', [
    1, 2, 3, 4,
])
def test_run_pass_filter(case):
    provided = expected = fasta_tests[case]
    check_run(mod.run_pass_filter, provided, expected)

def test_run_pass_filter_5():
    check_run_raises(mod.run_pass_filter, fasta_tests[5])

def test_run_pass_filter_8_ccs():
    provided = fasta_tests[8]
    expected = expected_tests[8]
    check_run(mod.run_pass_filter, provided, expected)

########################################
### Test the streamed median filter. ###
########################################
@pytest.mark.parametrize('case', [
    1, 2, 3, 4, 5,
])
def test_run_streamed_median_filter(case):
    provided = fasta_tests[case]
    expected = expected_tests[case]
    check_run(mod.run_streamed_median_filter, provided, expected)

def test_run_streamed_median_7g():
    # test general
    curried = functools.partial(mod.run_streamed_median_filter, zmw_filter_func=mod.median_zmw_subread)
    check_run(curried, fasta_tests[7], expected_general_median_tests[7])

def test_run_streamed_median_7i():
    # test internal
    curried = functools.partial(mod.run_streamed_median_filter, zmw_filter_func=mod.internal_median_zmw_subread)
    check_run(curried, fasta_tests[7], expected_internal_median_tests[7])

def test_run_streamed_longest_3():
    check_run(mod.run_streamed_longest_filter, fasta_tests[3], expected_longest_tests[3])

#######################################
### Test the general median filter. ###
#######################################
@pytest.mark.parametrize('case', [
    1, 2, 3, 4, 5, 6
])
def test_run_median_filter(case):
    provided = fasta_tests[case]
    expected = expected_tests[case]
    check_run(mod.run_median_filter, provided, expected)

def test_run_median_filter_piped_input():
    """
    The run_median_filter asserts if the input file doesn't exist.
    This is to prevent specifying streams we can't rewind.
    """
    with pytest.raises(AssertionError) as e:
        mod.run_median_filter(sys.stdin, '', '')
    assert 'Cannot rewind' in str(e.value)

def test_run_median_filter_7():
    check_run(mod.run_median_filter, fasta_tests[7], expected_general_median_tests[7])

########################################
### Test the internal median filter. ###
########################################
@pytest.mark.parametrize('case', [
    1, 2, 3, 4, 5, 6
])
def test_run_internal_median_filter(case):
    provided = fasta_tests[case]
    expected = expected_tests[case]
    check_run(mod.run_internal_median_filter, provided, expected)

def test_run_internal_median_filter_piped_input():
    """
    The run_median_filter asserts if the input file doesn't exist.
    This is to prevent specifying streams we can't rewind.
    """
    check_run_raises(mod.run_internal_median_filter, fasta_tests[5])
    with pytest.raises(AssertionError) as e:
        mod.run_internal_median_filter(sys.stdin, '', '')
    assert 'Cannot rewind' in str(e.value)

def test_run_internal_median_filter_7():
    check_run(mod.run_internal_median_filter, fasta_tests[7], expected_internal_median_tests[7])

###############################
### Test whitelisting ZMWs. ###
###############################
@pytest.mark.parametrize('test_data', whitelist_tests_general_median)
def test_whitelist_run_internal_median_filter(test_data):
    test_fasta = test_data['fasta']
    test_whitelist = test_data['whitelist']
    test_expected = test_data['expected']
    check_run(mod.run_internal_median_filter, test_fasta, test_expected, test_whitelist)

@pytest.mark.parametrize('test_data', whitelist_tests_general_median)
def test_whitelist_run_median_filter(test_data):
    test_fasta = test_data['fasta']
    test_whitelist = test_data['whitelist']
    test_expected = test_data['expected']
    check_run(mod.run_median_filter, test_fasta, test_expected, test_whitelist)

@pytest.mark.parametrize('test_data', whitelist_tests_general_median)
def test_whitelist_run_streamed_median_filter(test_data):
    test_fasta = test_data['fasta']
    test_whitelist = test_data['whitelist']
    test_expected = test_data['expected']
    check_run(mod.run_streamed_median_filter, test_fasta, test_expected, test_whitelist)

@pytest.mark.parametrize('test_data', whitelist_tests_longest)
def test_whitelist_run_streamed_longest_filter(test_data):
    test_fasta = test_data['fasta']
    test_whitelist = test_data['whitelist']
    test_expected = test_data['expected']
    check_run(mod.run_streamed_longest_filter, test_fasta, test_expected, test_whitelist)

@pytest.mark.parametrize('test_data', whitelist_tests_pass)
def test_whitelist_run_pass_filter(test_data):
    test_fasta = test_data['fasta']
    test_whitelist = test_data['whitelist']
    test_expected = test_data['expected']
    check_run(mod.run_pass_filter, test_fasta, test_expected, test_whitelist)

###############################
### Test the main commands. ###
###############################
def test_main_cmd_pass_1(capsys):
    check_main_from_stdin('pass', fasta_tests[4], fasta_tests[4], capsys)

def test_main_cmd_pass_2(tmpdir, capsys):
    in_fa_file = tmpdir.join('in.fa')
    check_main_from_file('pass', fasta_tests[4], fasta_tests[4], in_fa_file, capsys)

def test_main_cmd_streamed_median_1(capsys):
    check_main_from_stdin('streamed-median', fasta_tests[4], expected_tests[4], capsys)

def test_main_cmd_streamed_median_2(tmpdir, capsys):
    in_fa_file = tmpdir.join('in.fa')
    check_main_from_file('streamed-median', fasta_tests[4], expected_tests[4], in_fa_file, capsys)

def test_main_cmd_median_1(tmpdir, capsys):
    in_fa_file = tmpdir.join('in.fa')
    check_main_from_file('median', fasta_tests[4], expected_tests[4], in_fa_file, capsys)

def test_main_cmd_internal_median_1(tmpdir, capsys):
    in_fa_file = tmpdir.join('in.fa')
    check_main_from_file('internal-median', fasta_tests[4], expected_tests[4], in_fa_file, capsys)

def test_main_cmd_streamed_internal_median_1(tmpdir, capsys):
    check_main_from_stdin('streamed-internal-median', fasta_tests[4], expected_tests[4], capsys)

def test_main_cmd_streamed_internal_median_2(tmpdir, capsys):
    in_fa_file = tmpdir.join('in.fa')
    check_main_from_file('streamed-internal-median', fasta_tests[4], expected_tests[4], in_fa_file, capsys)

#####################################################
### Test ZMW whitelisting from the main commands. ###
#####################################################
def test_whitelist_main_cmd_streamed_longest_from_stdin(tmpdir, capsys):
    for test_id, test_data in enumerate(whitelist_tests_longest):
        test_fasta = test_data['fasta']
        test_whitelist = test_data['whitelist']
        test_expected = test_data['expected']

        test_whitelist_fn = tmpdir.join('zmw.whitelist.{}.json'.format(test_id))
        if test_whitelist == None:
            # In case None, just omit the path.
            test_whitelist_fn = ''
        else:
            # In case an empty list, write an empty file.
            test_whitelist_fn.write(json.dumps(list(test_whitelist)))

        check_main_from_stdin_with_whitelist('streamed-longest', test_fasta, test_expected, str(test_whitelist_fn), capsys)

#######################
### Unit-test functions
def test_tokenize_header():
    with pytest.raises(ValueError) as e:
        mod.tokenize_header('bad_header')
    assert 'bad_header' in str(e.value)


from numpy import *

arr1= array([1,2,3,4,5])
arr2= array([6,7,8,9,10])
print(sin(arr1))
print(sum(arr1))

arr3= arr1 + arr2
print(arr3)

arr1=arr1+5
print(arr1)

print(concatenate([arr1,arr2]))

for i in range(0,len(arr1)):
    arr1[i]=arr1[i]+5
print(arr1)

arr= empty(5)
for i in range(0,len(arr1)):
    arr[i]=arr1[i]+arr2[i]
print(arr)

#Max without in-built function

arr=array([1,5,7,2,4,67,27,23,21])
max=0
#or max=arr[0]
for i in arr:
    if i>max:
        max=i

print(max)

from array import *
arr = array('i',[1,2,3,4,5])
arr1 = array('i',[])

for i in range(len(arr)):
    arr1.append(arr[i]+5)
print(arr1)