from odoo import fields, models

class AccountMoveLine(models.Model):

    _inherit = "account.move.line"

    discount_due_date = fields.Date(
        related="move_id.discount_due_date",
        readonly=True,
    )
    discount_amount = fields.Monetary(
        related="move_id.discount_amount",
        readonly=True,
    )

    def _prepare_payment_line_vals(self, payment_order):
        self.ensure_one()
        values = super(AccountMoveLine, self)._prepare_payment_line_vals(payment_order)

        move = self.move_id
        if move and move.discount_due_date and move.has_discount:
            pay_with_discount = move._can_pay_invoice_with_discount()
            values["pay_with_discount"] = pay_with_discount
            if pay_with_discount:
                values["amount_currency"] = move.residual_with_discount
        return values


import dask.dataframe as dd
from dask.distributed import Client
from dask_cloudprovider.aws import EC2Cluster
import pandas as pd
import numpy as np
import timeit
import configparser
import os
import contextlib
from dotenv import load_dotenv

load_dotenv()

def get_aws_credentials():
    parser = configparser.RawConfigParser()
    parser.read(os.path.expanduser('~/.aws/config'))
    config = parser.items('default')
    parser.read(os.path.expanduser('~/.aws/credentials'))
    credentials = parser.items('mining')
    all_credentials = {key.upper(): value for key, value in [*config, *credentials]}
    with contextlib.suppress(KeyError):
        all_credentials["AWS_REGION"] = all_credentials.pop("REGION")
    all_credentials['EXTRA_PIP_PACKAGES'] = 's3fs pyarrow plotly'
    return all_credentials

cluster = EC2Cluster(region='us-east-1', n_workers=1, security=False, env_vars=get_aws_credentials())
client = Client(cluster)
# client = Client("174.129.112.139:8786")

random_state = 42
number = 1
repeat = 10

# aws_access_key_id = os.environ['aws_access_key_id']
# aws_secret_access_key = os.environ['aws_secret_access_key']
# aws_session_token = os.environ['aws_session_token']
# kwargs = dict(keystring=aws_access_key_id,
#               secretstring=aws_secret_access_key,
#               tokenstring=aws_session_token)
df: dd.DataFrame = dd.read_parquet('s3://merged-tweets/full-sample/full.parquet')
df['geo'] = df['geo'].astype(str)
df['coordinates'] = df['coordinates'].astype(str)
count = len(df.index)
functions = {
    'read': lambda: dd.read_parquet(
        's3://merged-tweets/full-sample/full.parquet'
    ),
    '10k': lambda: df.sample(frac=10000 / count, random_state=random_state).to_parquet(
        's3://merged-tweets/testing-sample/test_10k.parquet'
    ),
    '100k': lambda: df.sample(frac=100000 / count, random_state=random_state).to_parquet(
        's3://merged-tweets/testing-sample/test_100k.parquet'
    ),
    '500k': lambda: df.sample(frac=500000 / count, random_state=random_state).to_parquet(
        's3://merged-tweets/testing-sample/test_500k.parquet'
    ),
    'full': lambda: df.to_parquet(
        's3://merged-tweets/testing-sample/test_full.parquet'
    ),
}

df_results = pd.DataFrame()
for function in functions:
    time = timeit.repeat(functions[function], number=number, repeat=repeat)
    df_results.loc[0, f'{function}_avg'] = np.mean(time)
    df_results.loc[0, f'{function}_std'] = np.std(time)
df_results.to_csv('s3://merged-tweets/results/dask_etl_1_worker.csv', index=False)

cluster.close()


import gudhi
import csv
import pandas as pd
import numpy as np

#target_matrix = pd.read_csv('test_full_distance_matrix.csv',header=None) #should use gudhi's load data function
target_matrix = gudhi.read_lower_triangular_matrix_from_csv_file('test_full_distance_matrix.csv') #input distance matrix csv format and with comma
#print(target_matrix)
rips_complex = gudhi.RipsComplex(distance_matrix=target_matrix,max_edge_length=1)

simplex_tree = rips_complex.create_simplex_tree(max_dimension=4)
result_str = 'Rips complex is of dimension ' + repr(simplex_tree.dimension()) + ' - ' + \
    repr(simplex_tree.num_simplices()) + ' simplices - ' + \
    repr(simplex_tree.num_vertices()) + ' vertices.'
print(result_str)
fmt = '%s -> %.2f'
for filtered_value in simplex_tree.get_filtration():
    print(fmt % tuple(filtered_value))
diag = simplex_tree.persistence(homology_coeff_field=2, min_persistence=0)
print(diag)
print(np.shape(diag)) # when change the option - max_edge_length in function RipsConplex, the dimension of diagram would not change
# print(type(diag)) #: list
diag_array = np.array(diag)
combination_birth_t = list()
combination_death_t = list()
for i in range(len(diag)):
    combination_birth_t.append(diag_array[:,1][i][0])
    combination_death_t.append(diag_array[:, 1][i][1])

combination_birth_death_t = combination_birth_t + combination_death_t
combination_birth_death_t = np.unique(combination_birth_death_t)

tmp = np.zeros(shape=(len(diag),3))
tmp[:,0] = diag_array[:,0]
for j in range(len(diag)):
    tmp[j,1] = combination_birth_t[j]
    tmp[j,2] = combination_death_t[j]

column_names = ['Time','Betti_0','Betti_1','Betti_2','Betti_3'] # here, we work on 4-dimensions
betti = np.zeros(shape=(len(combination_birth_death_t),5),dtype=float)
betti[:,0] = np.array(combination_birth_death_t)

for ii in range(len(tmp)):
    targetBettiCol = tmp[ii,0] + 1
    targetBettiCol = int(targetBettiCol)
    birthID = np.where(betti[:,0] == tmp[ii,1])
    birthID = int(np.array(birthID))
    deathID = np.where(betti[:,0] == tmp[ii,2])
    deathID = int(np.array(deathID))

    if tmp[ii,2]==float('Inf'):
        new_deathID = deathID+1
        betti[birthID:new_deathID,targetBettiCol] = betti[birthID:new_deathID,targetBettiCol]+1
    else:
        betti[birthID:deathID, targetBettiCol] = betti[birthID:deathID, targetBettiCol] + 1

print(betti) # complete and consistent dim is (258,5)

betti_output = pd.DataFrame(betti,columns=column_names)
print(betti_output)

# Since GUDHI do not provide the wasserstein distance calculation, therefore we need to transform to TDA package in R #
# Firstly, save tmp file i.e., output diagram as .csv file. For example: #
np.savetxt('diag_output_matrix.csv', tmp, delimiter=',')
#--#

# bottleneck distance (not Wasserstein distance - it could be accessed through TDA package in R) #
#gudhi.bottleneck_distance(diag1, diag2, 0.1)

'''
# calculate the betti number command #
print(simplex_tree.betti_numbers())
print(simplex_tree.persistent_betti_numbers(0, 1)) #from_value and to_value; I should use this function
'''

'''
print(simplex_tree.dimension())
print(simplex_tree.num_simplices())
print(simplex_tree.num_vertices())
print(simplex_tree.persistent_betti_numbers(0.1,0.3))
print(int(np.array(np.where(betti[:,0]==tmp[0,1]))))
'''
plt = gudhi.plot_persistence_diagram(diag,legend=True)
plt.show()

# http://gudhi.gforge.inria.fr/python/latest/rips_complex_user.html - guideline
# http://gudhi.gforge.inria.fr/python/latest/simplex_tree_ref.html - guideline


# -*- coding: utf-8 -*-
"""
Created on Tue Apr 29 10:55:48 2014

@author: user
"""

import urllib
import re

def extractproductid(line):

    productid = re.search('data-product_id="P[0-9]+',line)

    if(productid):

        return productid

rootUrl  = "http://www.sephora.com/contentStore/mediaContentTemplate.jsp?mediaId=14300062&icid2=HomePage_QuickLink_Bestsellers_04.15.14_Link";
htmlFile = urllib.urlopen(rootUrl)

outFile = "C:/Users/user/Documents/NYU/FCS/finalProject/sephora_reviewurl.txt";

fout=open(outFile, "w")

for line in htmlFile:
    if "data-product_id=\"P" in line:
        productid = extractproductid(line)
        if(productid != None):
            onlyproductid = productid.group().split("\"")
           #print"inside:","http://reviews.sephora.com/8723Illuminate/"+onlyproductid[1]+"/reviews.htm?format=noscript"
            fout.write("http://reviews.sephora.com/8723Illuminate/"+onlyproductid[1]+"/reviews.htm?format=noscript"+"\n");
fout.close()


import environment
import math
import learner2
import pickle
import numpy as np
from utils import *
import sys

"""
Points to Note:

1. Keep the ordering of manufacturers constant at places that are permanent, for example in Environment.producers

"""

def get_inputs():
    n = int(input("Enter the number of manufacturers"))
    #alpha = float(input("Enter threshold quality"))
    print("Now, for each manufacturer, enter its quality(0-1), cost and capacity")
    Q,K,C = [],[],[]
    for i in range(n):
        r,s,t = input().split(",")
        Q.append(float(r))
        K.append(float(s))
        C.append(float(t))
    return n,Q,K,C

def main():
    n,Q,K,C = get_inputs()
    nsim = int(sys.argv[1])
    nrounds = int(sys.argv[2])
    error_margin_quality = float(sys.argv[3])
    optimistic_margin = error_margin_quality
    # burnout = int(float(3*math.log(nsim))/float(2*(error_margin_quality**2)))
    burnout = int(1.0/float(error_margin_quality**2))
    print("Burnout : ",burnout)
    R,alpha = 100,0.7
    for i in range(n):
        if(abs(R*Q[i]-C[i]) < 1):
            print(i,Q[i],C[i])
    # exit(0)
    suffix = "_"+str(n)+"_"+str((1/error_margin_quality))+"_"+str(nsim)
    print(n,Q,K,C)
    env = environment.Environment(n,Q,C,K,R,alpha)
    # learner = learner1.LearnerAgent(env)

    opt_allocation,excess_qual = env.get_optimal(0.0)
    env.reset()
    opt_allocation_margin,excess_qual_margin = env.get_optimal(error_margin_quality)
    print("excess quality : ",excess_qual)
    print(opt_allocation,opt_allocation_margin,sep="\n")
    opt_reward = env.get_expected_reward(opt_allocation)
    opt_reward_margin = env.get_expected_reward(opt_allocation_margin)
    # print(opt_reward)

    # check = 1
    # for i in range(n):
    #     manu1 = env.producers[i]
    #     manu2 =

    max_revenue = env.get_max_revenue()
    print(opt_reward,opt_reward_margin,max_revenue)
    # exit(0)
    total_regret = np.array([[0 for k in range(nsim)],[0 for k in range(nsim)]])
    regrets = np.array([0 for k in range(nsim)])
    correctness = np.array([0 for k in range(nsim)])
    average_quals = np.array([0 for k in range(nsim)],dtype=np.float)
    observed_indeces = [1,2,3,4]

    for j in range(nrounds):
        learner = learner2.LearnerAgent(env,burnout,optimistic_margin)
        # learner = learner2.LearnerAgent(env,burnout,0.00)
        cum_regret = 1
        once_correct = 0

        for i in range(nsim):
            print("Iteration no %d Round no %d" %(i,j))
            # allocation = learner.run()
            allocation_test = learner.run(i+burnout)
            learner.reset()
            allocation = learner.run(i)
            # print(allocation)
            # print("Allocation\n",allocation)
            # print(opt_allocation)
            qualities = env.get_qualities(allocation)
            # print("Rewards:\n",qualities)
            # if(qualities[5][1] == 0):
            #     observed_quals.append(0)
            # else:
            #     print("Observed Quality : ",float(qualities[observed_index][2])/float(qualities[observed_index][1]))
            #     print("Actual Quality :",env.producers[observed_index].q)

            correct_test,abs_correct_test,avg_qual_test = env.check_const(allocation_test, 0.00)
            correct,abs_correct,avg_qual = env.check_const(allocation, 0.00)
            once_correct = max(once_correct,abs_correct)
            correctness[i] += correct_test
            print("Correctness: ",correct_test," Once correct: ",once_correct)
            average_quals[i] += avg_qual

            if(i > burnout):
                regret = 0
                if not correct:
                    print("Not Even marginally correct")
                    regret = max_revenue
                elif not abs_correct:
                    print("Marginally correct")
                    regret = env.get_regret(allocation,0.00)
                else:
                    print("Totally correct")
                    regret = env.get_regret(allocation,opt_reward)
                # regret = max(0,regret)
                cum_regret += float(regret)
                regrets[i] += regret
                total_regret[0][i] = math.log(cum_regret)
                total_regret[1][i] += float(cum_regret)
            # print("Regret: ",regret)
        	# print(allocation)

            learner.update(qualities,env.T,i+2)
            learner.reset()
            if(i == burnout):
                print(Q)
                learner.print_estimates()
            #record outputs
        # learner.completeReset()

    saveVar(regrets,'./Variables/regret_'+suffix)
    saveVar(total_regret[0],'./Variables/totregret_'+suffix)
    saveVar(total_regret[1],'./Variables/lnregret_'+suffix)
    saveVar(correctness,'./Variables/correct_'+suffix)
    saveVar(average_quals,'./Variables/avg_quals_'+suffix)
    for observed_index in observed_indeces:
        saveVar(learner.prods[observed_index].q_est_history,'./Variables/q_history_'+str(observed_index)+"_"+suffix)

    myPlot(regrets/float(nrounds),'Regret Analysis - Regret in revenue','./Plots/Regret_'+suffix)
    myPlot(total_regret[0]/float(nrounds),'Total Regret Till that moment','./Plots/TotalRegretlogT1_'+suffix)
    # myPlotxy(total_regret[0],total_regret[1]/nrounds,'Cumulative Regret vs logT','./Plots/TotalRegretlogT1_'+suffix)
    myPlotlog(total_regret[1]/nrounds,'Cumulative Regret vs logT','./Plots/TotalRegretlogT2_'+suffix)
    myPlot(total_regret[1]/nrounds,'Total Regret Till that moment','./Plots/TotalRegret_'+suffix)
    myPlot(correctness/nrounds,'Correctness','./Plots/Correctness'+suffix)
    myPlot(average_quals/float(nrounds),'Average Qualities','./Plots/AverageQuals'+suffix)
    for observed_index in observed_indeces:
        myPlot2(nsim,learner.prods[observed_index].q_history,env.producers[observed_index].q,'Variation of quality','./Plots/q_track_'+str(observed_index))
        myPlot2(nsim,learner.prods[observed_index].q_est_history,env.producers[observed_index].q,'Variation of Mean quality','./Plots/q_mean_track_'+str(observed_index))

if __name__== "__main__":
    main()


from django.db import models
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _

class BlackLists(models.Model):
    user = models.ForeignKey(User, related_name='blacklist', verbose_name=_('user'))
    shielding = models.ForeignKey(User, related_name='shielding', verbose_name=_('shielding'))
    created_at = models.DateTimeField(auto_now_add=True)

    class Meta:
        db_table = 'blacklists'
        verbose_name = _('BlackLists')


def plot_fig03(data_df, dnb_idx):

  shift_dic = {3:0.64, 4:0.50, 5:0.42}

  stat_df = pd.DataFrame(index=['condition',
                                'week',
                                'average standard deviation $I_s$',
                                'average correlation strength $I_r$'])
  for i, (col, df) in enumerate(data_df.groupby(axis=1, level=[0,1])):

    shift   = shift_dic[df.shape[1]]
    std_val = df.loc[dnb_idx].std(axis=1).mean()
    corr_df = df.loc[dnb_idx].T.corr().abs()
    np.fill_diagonal(corr_df.values, None)
    corr_val = corr_df.mean().mean() - shift
    stat_df[i] = [col[0], col[1], std_val, corr_val]

  stat_df = stat_df.T
  stat_df = stat_df[stat_df['condition'] != 'TSNO']
  stat_df.condition = stat_df.condition.str.replace('TSOD-BTS','BTS').\
                      str.replace('TSOD','untreated')

  with sns.plotting_context('talk'):

    w, h = (5,4)
    kws = dict(data=stat_df,
               x='week',
               hue='condition',
               legend=False,
               hue_order = ['untreated', 'BTS'],
               palette   = ['xkcd:windows blue', 'xkcd:rose'],
               aspect=w/h,
               size=h)
    g1 = sns.factorplot(y=stat_df.columns[2], **kws)
    g2 = sns.factorplot(y=stat_df.columns[3], **kws)
    for i, g in enumerate([g1, g2]):
      g.ax.legend(frameon=False, loc='upper left')
      if in_ipython():
        g.fig.show()
        g.fig.savefig('tmp{}.png'.format(i))

if __name__ == '__main__':
  plot_fig03(data_df, dnb_idx)


import sys
import os
from matplotlib import pyplot as plt
import numpy as np
import math
import pandas as pd

sys.path.append(os.path.dirname(os.path.abspath(__file__))
                + "/../ModelPredictiveTrajectoryGenerator/")

try:
    import model_predictive_trajectory_generator as planner
    import motion_model
except:
    raise

table_path = os.path.dirname(os.path.abspath(__file__)) + "/lookuptable.csv"

show_animation = True

def search_nearest_one_from_lookuptable(tx, ty, tyaw, lookup_table):
    mind  = float("inf")
    minid = -1

    for (i, table) in enumerate(lookup_table):

        dx   = tx - table[0]
        dy   = ty - table[1]
        dyaw = tyaw - table[2]
        d    = math.sqrt(dx ** 2 + dy ** 2 + dyaw ** 2)

        if d <= mind:
            minid = i
            mind  = d

    return lookup_table[minid]

def get_lookup_table():
    data = pd.read_csv(table_path)

    return np.array(data)

def generate_path(target_states, k0):
    # x, y, yaw, s, km, kf

    lookup_table = get_lookup_table
    result = []

    for state in target_states:
        bestp = search_nearest_one_from_lookuptable(
            state[0], state[1], state[2], lookup_table)

        target = motion_model.State(x=state[0], y=state[1], yaw=state[2])
        init_p = np.array(
            [math.sqrt(state[0] ** 2 + state[1] ** 2), bestp[4], bestp[5]]).reshape(3, 1)

        x, y, yaw, p = planner.optimize_trajectory(target, k0, init_p)

        if x is not None:
            print("find goof path")
            result.append(
                [x[-1], y[-1], yaw[-1], float(p[0]), float(p[1]), float(p[2])])
    print("finish path generation")
    return result

def calculate_uniform_polar_states(nxy, nh, d, a_min, a_max, p_min, p_max):

    '''
    calculate uniform state

    :param nxy   : number of position sampling
    :param nh    : number of heading sampling
    :param d     : distance of terminal state
    :param a_min : position sampling min angle
    :param a_max : position sampling max angle
    :param p_min : heading sampling min angle
    :param p_max : heading sampling max angle
    :return      : states list

    '''
    angle_samples = [i / (nxy - 1) for i in range(nxy)]
    states = sample_states(angle_samples, a_min, a_max, d, p_min, p_max, nh)

    return states

def sample_states(angle_samples, a_min, a_max, d, p_min, p_max, nh):
    states = []
    for i in range(angle_samples):
        a = a_min + (a_max - a_min) * i

        for j in range(nh):
            xf = d * math.cos(a)
            yf = d * math.sin(a)

            if nh == 1:
                yawf = (p_max - p_min) / 2.0 + a
            else:
                yawf = p_min + (p_max - p_min) * j / (nh - 1) + a

            states.append([xf, yf, yawf])

    return states

def uniform_terminal_state_sampling_test1():

    k0     = 0.0
    nxy    = 5
    nh     = 3
    d      = 20
    a_min  = - np.deg2rad(45.0)
    a_max  =   np.deg2rad(45.0)
    p_min  = - np.deg2rad(45.0)
    p_max  =   np.deg2rad(45.0)
    states =   calculate_uniform_polar_states (nxy, nh, d, a_min, a_max, p_min, p_max)
    result =   generate_path(states, k0)

    for table in result:
        xc, yc, yawc = motion_model.generate_trajectory(
            table[3], table[4], table[5], k0)
        if show_animation:
            plt.plot(xc, yc, "-r")
    if show_animation:
        plt.grid(True)
        plt.axis("equal")
        plt.show()
    print("Done")

def main():
    uniform_terminal_state_sampling_test1()

if __name__ == '__main__':
    main()


import os

import numpy as np
import torch

from PIL import Image
from diffusers import ControlNetModel, StableDiffusionControlNetInpaintPipeline, DDIMScheduler
from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_controlnet import MultiControlNetModel
from transformers import BlipProcessor, BlipForQuestionAnswering

def postprocess(output: StableDiffusionPipelineOutput) -> list[Image.Image]:
    images = output.images
    assert isinstance(images[0], Image.Image), "images should be a list of PIL.Image.Image"
    nsfw = output.nsfw_content_detected
    images = [images[i] for i in range(len(images)) if not nsfw[i]]
    return images

def resize_tile_condition_image(input_image: Image, resolution: int, mode="long_side"):
    assert mode in ["long_side", "short_side"], "mode should be long_side or short_side"
    input_image = input_image.convert("RGB")
    W, H = input_image.size
    k = float(resolution) / (min(H, W) if mode == "short_side" else max(H, W))
    H *= k
    W *= k
    H = int(round(H / 64.0)) * 64
    W = int(round(W / 64.0)) * 64
    img = input_image.resize((W, H), resample=Image.LANCZOS)
    return img

def make_inpaint_condition(image, image_mask):
    image = np.array(image.convert("RGB")).astype(np.float32) / 255.0
    image_mask = np.array(image_mask.convert("L")).astype(np.float32) / 255.0

    assert image.shape[0:1] == image_mask.shape[0:1], "image and image_mask must have the same image size"
    image[image_mask > 0.5] = -1.0  # set as masked pixel
    image = np.expand_dims(image, 0).transpose(0, 3, 1, 2)
    image = torch.from_numpy(image)
    return image

class BLIP:
    def __init__(self, model_name="./checkpoints/blip-vqa-capfilt-large", device='cuda'):
        self.device = device
        self.processor = BlipProcessor.from_pretrained(model_name)
        self.model = BlipForQuestionAnswering.from_pretrained(model_name).to(device)

    def vqa(self, image_path: str, question: str) -> str:
        raw_image = Image.open(image_path).convert('RGB')
        inputs = self.processor(raw_image, question, return_tensors="pt").to(self.device)
        out = self.model.generate(**inputs)
        answer = self.processor.decode(out[0], skip_special_tokens=True)
        return answer

class InpaintControlNet:
    controlnet_models = {
        'softedge': 'control_v11p_sd15_softedge',
        'openpose': 'control_v11p_sd15_openpose',
        'lineart': 'control_v11p_sd15_lineart'
    }

    def __init__(self,
                 base_model="./checkpoints/realisticVisionV40VAE-inpainting",
                 controlnet_folder: str = "./checkpoints",
                 controlnet: list[str] = None,
                 device='cuda'):

        if controlnet is None:
            controlnet = ['lineart']

        assert all([k in self.controlnet_models for k in controlnet]), \
            f"controlnet should be a subset of {self.controlnet_models.keys()}"

        self.device = device
        self.controlnet = {}
        for k in controlnet:
            model_path = os.path.join(controlnet_folder, self.controlnet_models[k])
            self.controlnet[k] = ControlNetModel.from_pretrained(model_path, torch_dtype=torch.float16).to(self.device)

        self.pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained(
            base_model, controlnet=list(self.controlnet.values()), torch_dtype=torch.float16
        ).to(self.device)

        self.pipe.scheduler = DDIMScheduler.from_config(self.pipe.scheduler.config)
        self.pipe.enable_xformers_memory_efficient_attention()
        self.pipe.enable_model_cpu_offload()

    def __call__(self, prompt: str, image_path: str, mask_image_path: str,
                 control_image: list[str] = None,
                 controlnet: list[str] = None, **kwargs) -> list[str]:
        image = Image.open(image_path).convert("RGB")
        mask_image = Image.open(mask_image_path).convert("L")
        if control_image is not None:
            control_image = [Image.open(path).convert("RGB") for path in control_image]
        if not isinstance(controlnet, list) and not isinstance(controlnet, str):
            controlnet = ['lineart']
            control_image = [mask_image]
            kwargs['controlnet_conditioning_scale'] = 0.0
        else:
            assert len(controlnet) > 0, "controlnet should be a subset of {self.controlnet.keys()}"
            if isinstance(controlnet, str):
                controlnet = [controlnet]
            if isinstance(control_image, str):
                control_image = [control_image]
            assert len(controlnet) == len(control_image), "'controlnet' and 'control_image' should have the same length"
            assert all([k in self.controlnet for k in controlnet]), \
                f"controlnet should be a subset of {self.controlnet.keys()}"

        if len(controlnet) > 1:
            self.pipe.controlnet = MultiControlNetModel([self.controlnet[k] for k in controlnet]).to(self.device)
        elif len(controlnet) == 1:
            self.pipe.controlnet = self.controlnet[controlnet[0]].to(self.device)

        if 'width' not in kwargs and 'height' not in kwargs:
            kwargs.update({'width': image.width, 'height': image.height})

        with torch.inference_mode():
            results = postprocess(
                self.pipe(prompt=prompt, image=image, mask_image=mask_image,
                          control_image=control_image, **kwargs)
            )
            # save results to the same path of image_path
            result_paths = []
            for i, img in enumerate(results):
                img.save(image_path.replace('.', f'-result-{i}.'))
                result_paths.append(image_path.replace('.', f'-result-{i}.'))
        return result_paths


import numpy as np
import os
from PIL import ImageFont, ImageDraw, Image
from moviepy.audio.io.AudioFileClip import AudioFileClip
from moviepy.video.io.VideoFileClip import VideoFileClip
from moviepy.audio.AudioClip import AudioClip
from moviepy.video.VideoClip import ImageClip
from moviepy.editor import concatenate_videoclips, concatenate_audioclips, CompositeVideoClip, CompositeAudioClip, afx
from skimage.filters import gaussian

class Editor(object):
  def __init__(
    self,
    target_resolution: tuple,
    fps: int,
    font: str,
    text_size: int,
    text_color: str,
    text_stroke_width: int,
    text_stroke_color: str
  ):
    """
    Initialize Editor.

    Args:
        target_resolution (tuple): (width, height) of the editted video.
        fps (int): fps of the video.
        font (str): path to the font file.
        text_size (int): size of the subtitle.
        text_color (str): color of the subtitle.
        text_stroke_width (int): width of the stroke of the subtitle.
        text_stroke_color (str): color of the stroke of the subtitle.
    """
    self.target_resolution = target_resolution
    self.fps = fps
    self.text_size = text_size
    self.text_color = text_color
    self.font = ImageFont.truetype(font, size=text_size)
    self.max_text_cnt = target_resolution[0] // text_size
    self.text_stroke_width = text_stroke_width
    self.text_stroke_color = text_stroke_color

  def __is_image(self, image_name: str):
    """
    Determine whether given name is image or video.

    Args:
        image_name (str): file name

    Return:
        return (bool): True if image, False otherwise (of course, video.)
    """
    return image_name.endswith(("bmp", "jpg", "jpeg", "gif", "png", "tif", "wmf", "emf"))

  def __fit_image_or_video_in_screen(self, image: ImageClip | VideoFileClip):
    """
    Fit given image or video into target resolution.
    Ratio will be kept during resizing, and rest of the part would be blurred image.

    Args:
        image (ImageClip | VideoFileClip): image or video

    Return:
        return (VideoFileClip): fitted image or video.
    """
    # resize given image while keeping ratio
    if image.size[0] / image.size[1] > self.target_resolution[0] / self.target_resolution[1]:
      height = image.size[1] * self.target_resolution[0] / image.size[0]
      height = max(height, self.target_resolution[0])
    else:
      height = self.target_resolution[1]
    image = image.resize(height=height)

    # set image position: center
    image = image.set_position(("center", "center"))

    # set duration and position of the background
    background = image.resize(self.target_resolution)
    background = background.fl_image(lambda frame: gaussian(frame.astype(float), sigma=8))
    background = background.set_duration(image.duration)

    # composite with background image.
    return CompositeVideoClip([background, image])

  def __add_text_to_video(self, image: VideoFileClip, text: str):
    """
    Returns subtitle-adding function for moviepy.fl_image()

    Args:
        image (VideoFileClip): video to add subtitle
        text (str): subtitle

    Return:
        return (VideoFileClip): video with subtitle
    """
    def pipeline(subtitle):
      def fun(frame):
        frame = Image.fromarray(frame)
        draw = ImageDraw.Draw(frame)
        for idx, line in enumerate(subtitle):
          x = self.target_resolution[0] / 2
          # y = self.target_resolution[1] - self.text_size * (3 + 1.5 * (len(subtitle) - 1 - idx))
          # y = self.target_resolution[1] / 2 + self.text_size * (idx * 1.5 - 0.75)
          y = self.target_resolution[1] * 0.73 + self.text_size * idx * 1.4
          draw.text((x, y), line, self.text_color, self.font, "mm", stroke_width=self.text_stroke_width, stroke_fill=self.text_stroke_color)
        return np.array(frame)
      return fun

    # split text to put in the screen
    subtitles = []
    subtitle = ""
    for token in text.split():
      if len(subtitle) + len(token) > self.max_text_cnt:
        # enter: make new line
        subtitles.append(subtitle)
        subtitle = token
      else:
        # add token to current line
        subtitle += f" {token}"
        subtitle = subtitle.strip()
      if token.endswith((".", "!", "?", "\"")):
        # end of the line: make new empty line
        subtitles.append(subtitle)
        subtitle = ""
    if subtitle:
      subtitles.append(subtitle)

    # triple subtitles
    subtitles = list(zip(subtitles[::2], subtitles[1::2] + [""]))

    # add subtitles for each clips
    sub_clips = []
    get_subtitle_len = lambda l: sum(len(s) + (4 if s.endswith((".", "!", "?", "\"")) else 0) for s in l)
    total_len = sum(map(get_subtitle_len, subtitles))
    start = 0
    end = 0
    for idx, subtitle in enumerate(subtitles):
      if idx == len(subtitles) - 1:
        end = image.duration
      else:
        end = min(start + image.duration * (get_subtitle_len(subtitle) / total_len), image.duration)
      sub_clip = image.subclip(start, end)
      sub_clip = sub_clip.fl_image(pipeline(subtitle))
      sub_clips.append(sub_clip)
      start = end

    return concatenate_videoclips(sub_clips)

  def __edit_image_or_video_with_audio(self, line: dict, video_name: str):
    """
    Add a video to an audio, then return a video clip.

    Args:
        line (dict): must have audio_name, image_name, and duration.
        video_name (str): name of the video.

    Return:
        return (VideoClip): video clip.
    """
    # validation
    assert "audio_name" in line
    assert "image_name" in line
    assert "content" in line

    # audio clip
    audio = AudioFileClip(line["audio_name"])
    audio = audio.subclip(0, audio.duration - 0.05)
    audio = concatenate_audioclips([audio, AudioClip(lambda _: 0, duration=0.05)])

    # get image clip
    if self.__is_image(line["image_name"]):
      # if image, set duration as audio
      image = ImageClip(line["image_name"])
      image = image.set_duration(audio.duration)
    else:
      # if video, remove audio and set duration as audio (loop or cut)
      image = VideoFileClip(line["image_name"])
      image = image.without_audio()
      if image.duration < audio.duration:
        image = image.loop(duration=audio.duration)
      else:
        image = image.set_duration(audio.duration)

    # fit video into the screen
    image = self.__fit_image_or_video_in_screen(image)

    # add subtitles
    image = self.__add_text_to_video(image, line["content"])

    # add audio
    video = image.set_audio(audio)
    video = video.set_fps(self.fps)

    return video

  def __add_bgm_to_video(self, video: VideoFileClip, bgm: str, bgm_vol: float):
    # bgm
    bgm = AudioFileClip(bgm)
    bgm = bgm.volumex(bgm_vol)
    if bgm.duration < video.duration:
      bgm = afx.audio_loop(bgm, duration=video.duration)
    else:
      bgm = bgm.set_duration(video.duration)

    # compose bgm with audio of the video
    audio = CompositeAudioClip([video.audio, bgm])

    return video.set_audio(audio)

  def edit_video(self, scenes: list, dirpath: str, bgm: str, bgm_vol: float):
    """
    Edit and save the final video.

    Args:
        scenes (list): returned value by Image/Video Generator.
                       each line must have audio_name, video_name, and duration.
        dirpath (str): path of saved clips and video.
        bgm (str): path to the background music.
        bgm_vol (float): volume of the background music.

    Return:
        return (str): returns video_name if no error occurred.
    """
    # generate video clips for each lines
    video_clips = []
    for scene_idx, scene in enumerate(scenes):
      for line_idx, line in enumerate(scene):
        video_clips.append(self.__edit_image_or_video_with_audio(line=line, video_name=os.path.join(dirpath, f"clip-{scene_idx}-{line_idx}")))

    # make final video
    video = concatenate_videoclips(video_clips)
    video = self.__add_bgm_to_video(video, bgm, bgm_vol)
    video.write_videofile(os.path.join(dirpath, "video.mp4"))

    return os.path.join(dirpath, "video.mp4")


from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.http import HttpResponseRedirect, Http404
from django.core.paginator import Paginator
from django.contrib.auth.decorators import login_required, permission_required
from django.shortcuts import render_to_response, get_object_or_404
from models import *

@login_required
def meus_trajetos(request):
    trajetos = Trajeto.objects.filter(usuario=request.user)
    return render_to_response('meus_trajetos.html', locals(), context_instance = RequestContext(request),)

@login_required
def editar_trajeto(request, trajeto_id,classe):
    trajeto = Trajeto.objects.get(id=trajeto_id)
    nome = trajeto.nome
    distancia = trajeto.distancia
    coordenadas = Coordenadas.objects.filter(trajeto=trajeto.id)

    if request.method == 'POST':
        novas_coordenadas = request.POST.getlist('waypoints')
        if(len(novas_coordenadas) > 0):
            trajeto.nome = request.POST.get('nome')
            trajeto.distancia = request.POST.get('distancia')[:-2].replace(',','.')
            coordenadas.delete()

            trajeto.save()

            for coordenada in novas_coordenadas:
                tupla_coordenada = coordenada[1:-1].split(',')
                temp_latitude = tupla_coordenada[0]
                temp_longitude = tupla_coordenada[1]
                nova_coordenada = Coordenadas(latitude=temp_latitude, longitude= temp_longitude, trajeto=trajeto)
                nova_coordenada.save()

    return render_to_response('edit_trajeto.html',locals(),context_instance=RequestContext(request))

@login_required
def trajeto(request):
    if request.method == 'POST':
        nome_trajeto = request.POST.get('nome')
        '''Lembrar de depois tratar o erro para distancia 0
        Fazer funcao parser dps :B'''
        distancia_total = request.POST.get('distancia')
        distancia_total = distancia_total[:-2].replace(',','.')
        coordenadas_todas = request.POST.getlist('waypoints')

        novo_trajeto = Trajeto(nome=nome_trajeto, distancia=distancia_total,usuario= request.user)
        novo_trajeto.save()

        for coordenada in coordenadas_todas:
            tupla_coordenada = coordenada[1:-1].split(',')
            temp_latitude = tupla_coordenada[0]
            temp_longitude = tupla_coordenada[1]
            nova_coordenada = Coordenadas(latitude=temp_latitude, longitude= temp_longitude, trajeto=novo_trajeto)
            nova_coordenada.save()

    return render_to_response(
        'set_trajetos.html',
        locals(),
        context_instance=RequestContext(request),
        )


from PIL import Image
from io import BytesIO

class ImageProcess(object):
    """
    图片处理类
    将Requests中获取到的二进制图片转换为Pillow可以读取的JPEG图片格式
    并对图像进行二值化处理
    可使用的成员函数
    byte2jpeg(byte)——将二进制格式图片byte转化为Pillow可读取格式
    image_binarize(image)——将Pillow格式图片image进行二值化
    process(source)——将request返回的二进制图片格式进行二值化
    """""

    def byte2jpeg(self, byte_content) -> Image:
        """
        将request包获取的二进制图像格式转换成Pillow可以读取的格式
        :param byte_content: Byte类型的图片数据
        :return: Pillow可以读取的jpeg图片格式
        """""
        return Image.open(BytesIO(byte_content))

    def image_binarize(self, image) -> Image:
        """
        将验证码图像进行二值化处理
        :param image: 原始图像
        :return: 二值化处理后的图像
        """""

        # 将图片转化为灰度图像
        image = image.convert("L")
        # 按像素点将图像转换为矩阵
        image_pixel_matrix = image.load()
        # 获取图像的宽度与高度，并进行遍历
        # 图片的宽度与高度默认都为66和22
        width, height = image.size
        for h in range(height):
            for w in range(width):
                # 消除首尾行列的黑框
                if h == 0 or h == 21 or w == 0 or w == 61:
                    image_pixel_matrix[w, h] = 255
                else:
                    # 通过反复尝试，阈值为110-140时对于目前的验证码效果最好
                    # 105±5的范围内为最优情况，过小会丢失图像，过大会保留干扰图像
                    if image_pixel_matrix[w, h] < 103:
                        image_pixel_matrix[w, h] = 0
                    else:
                        image_pixel_matrix[w, h] = 255
        return image

    def process(self, source) -> Image:
        """
        直接对Request返回的二进制图片格式进行二值化处理
        :param source: Requests返回的二进制格式图片
        :return: 二值化处理后的JPEG格式图片
        """""
        image = self.byte2jpeg(source)
        result = self.image_binarize(image)
        return result


def get_write_data():
    import csv
    with open("telefdirect1.csv", mode="w", encoding='utf-8') as file:
        file_writer = csv.writer(file, delimiter = "|", lineterminator="\r")
        file_writer.writerow(["ID", "Name", "Surname", 'Number', 'Comment'])

def get_new_write(data):
    import csv
    with open("telefdirect1.csv", "a", encoding='utf-8') as file:
        file_writer = csv.writer(file, delimiter = "|", lineterminator="\n")
        file_writer.writerow(list(data))

def check_row1():
    import os
    isempty=os.stat('telefdirect1.csv').st_size
    if isempty == 0:
        get_write_data()


import random
"""
    ##### `2016`年保研上机题
    ** 要求： **
    *请从服务器将两个数据文件 `input.txt` 和 `words.txt`   下载到本地电脑的 `D`  盘根文件夹。
    *在  `D`  盘根文件夹的  `words.txt`  中存储了不超过  `30000`条的英文单词，每个单词占一行。单词的最大长度为`20`，
    且单词内部没有空格，文件中无重复单词。
    *在D盘根文件夹的 `input.txt` 中存储了一个“丢失”了空格和标点符号的英文文章。每行不超过 `128`个字符，请编写程序把
    该文章中第一行和最后一行显示在屏幕上。
     *编写程序将 `words.txt`中的最后三行显示在屏幕上； *编写程序利用 `words.txt`    中的单词作为词典，采用正向最大
     匹配切分单词算法对   `input.txt`    中的文本进行单词切分。切分时单词区分大小写， 切分分割标记采用空格，并将切分
     后的结果写入到`out.txt` 中。
    所谓正向最大匹配切分就是从左向右扫描待切分字符串，尽量取长词。 下面举一个简单例子：现有待切分字符串`ABCDEFGHIJ`，
    设词典中最大单词长度为`5`。那么按照算法首先取出`ABCDE`判断是否是单词， 如果是则切分到一个单词，否则舍弃最后一个字母接着判断，
    也就是判断`ABCD`是否是单词，依此类推，当只有一个字母时可以直接认定为是单词。 在成功切分出一个单词后对待切分字符串余下的部分
    再次执行上述过程。 *编写程序实现步骤`2`、`3` 描述的要求，并通过如下所示的主函数对进行验证，注意：除了指定添加的代码之外，不得修改
    `main` 函数其余部分。对  `main` 函数每修改一处，总分扣 `3` 分，最多扣    `10` 分。
    *本次考试考核
    `C` 语言程序设计，因此不可以使用`C + +`的` STL
    `的任何功能，如果需要添加下面样例之外的程序头文件，请举手得到监考老师批准。
    ** 程序： **
    '"""

def make_file():
    with open('input.txt', 'w')as fp:
        pass

def read_file(url1, url2):
    #读取两个文件的内容
    with open(url1, 'r')as fp:
        words = fp.read().split('\n')
    with open(url2, 'r')as fq:
        txt = fq.read().split('\n')
    print(txt[0])
    print(txt[-1])                 #输出文章第一行与最后一行
    for i in words[-3:]:
        print(i)                   #输出词典后三位
    return words, txt

def split_file(words, txt):
    out = []
    for string in txt:              #对每一行缺字符的字符串进行拆分
        while len(string) > 0:
            for i in range(5, 0, -1):
                if string[:i] in words:
                    out.append(string[:i])
                    string = string[i:]
                    break
                if i == 1:
                    out.append(string[0])
                    string = string[1:]
    print(' '.join(out))

def main():
    words, txt = read_file('1.txt', '2.txt')
    split_file(words, txt)

if __name__ == '__main__':
    main()


class Solution:
	def find(self, word, refer):
		# stack = [(word, level)]
		stack = [(word, 1)]
		# keep the maximum level(deepest) we have go through
		max_level = 1
		visited = set()
		while stack:
			char, level = stack.pop()
			for i in range(len(char)+1):
				# create all the possible transformation if we replace one character with *
				# for example: ab -> a*, *b
				possible = char[:i] + '*' + char[i:]
				if possible in refer and possible not in visited:
					visited.add(possible)
					for w in refer[possible]:
						max_level = max(level+1, max_level)
						stack.append((w, level+1))

		return max_level

	def longestStrChain(self, words: List[str]) -> int:
		map_dict = defaultdict(list)
		res = 1
		for w in words:
			for i in range(len(w)):
				# get all possible next char
				# for example: a -> a*, *a; ab -> ab*, a*b, *ab
				transform = w[:i] + '*' + w[i+1:]
				map_dict[transform].append(w)
				# map_dict: {'*a': ['a'], 'a*': ['a'] ...}

		words = sorted(words, key=len)
		for i, w in enumerate(words):
			ans = self.find(w, map_dict)
			res = max(res, ans)
		return res

leetcode/scripts/algorithms/L/Longest String Chain/Longest String Chain.py


class Solution:
    def levelOrder(self, root: Optional[TreeNode]) -> List[List[int]]:

        q = collections.deque()
        q.append(root)
        allLevel = []

        if not root:
            return allLevel

        while q:
            curLevel = []
            for _ in range(len(q)):
                curr = q.popleft()

                curLevel.append(curr.val)

                if curr.left:
                    q.append(curr.left)
                if curr.right:
                    q.append(curr.right)

            allLevel.append(curLevel)

        return allLevel


"""The main app in the backend application"""
from fastapi import FastAPI
from database.db import init_db
from routers.product_review import product_reviews_router

app = FastAPI(
    title="Beanie Rules",
    description="Some Dummy backend which has beanie as the ODM",
    version="0.10"
)

@app.on_event("startup")
async def start_db():
    """The call to initialize the database on startup"""
    await init_db()

@app.get(
    "/",
    tags=["Root"]
)
async def root() -> dict[str, str]:
    """The root endpoint for the application"""
    return {"message": "Hello World"}

app.include_router(product_reviews_router,
                   prefix="/product_reviews", tags=["Product Reviews"])


import psutil
import pyttsx3
import warnings
import speech_recognition as sr
import os
import webbrowser
import wikipedia
import random
import subprocess
import string
import time
from sys import platform
import psutil as p
import datetime
from src.va import speak, ask

# engine = pyttsx3.init()
# voices = engine.getProperty("voices")
# engine.setProperty("voice", voices[1].id)

def open_chrome():

    if platform == "linux" or platform == "linux2":
        subprocess.run("chrome", shell=True)
    elif platform == "darwin":
        subprocess.run("open -a Chrome --args -private-window", shell=True)
    elif platform == "win32":
        subprocess.run("start chrome", shell=True)
    else:
        speak("Unsupported OS")

    # webbrowser.register('chrome', None, webbrowser.BackgroundBrowser(chrome_path))
    # webbrowser.get('chrome').open_new_tab('https://google.com')

def chrome_caller(input_string):
    open_chrome()

def main_date():

    now = datetime.datetime.now()
    speak("Sir, the Current Date is : " + now.strftime("%m-%d-%Y") + "\n")
    print("Sir, the Current Date is : ")
    print(now.strftime("%d-%m-%Y") + "\n")
    print("")

def date_caller(input_string):
    main_date()

def search_wiki():
    speak("What do you want to search on wikipedia ?")
    sinwiki = ask()
    sinwiki = sinwiki.replace("wikipedia", "")
    results = wikipedia.summary(sinwiki, sentences=2)
    print(results)
    speak("According to Wikipedia,")
    speak(results)

def search_wiki_caller(input_string):
    search_wiki()

hash_201801012 = {
    "Open Chrome": chrome_caller,
    "open chrome": chrome_caller,
    "start chrome": chrome_caller,
    "begin chrome": chrome_caller,
    "begin the chrome": chrome_caller,
    "initiate chrome": chrome_caller,
    "open chrome": chrome_caller,
    "start chrome": chrome_caller,
    "what is date": date_caller,
    "show the date": date_caller,
    "show date": date_caller,
    "display the date": date_caller,
    "display date": date_caller,
    "say date": date_caller,
    "search on wiki": search_wiki_caller,
    "search on wikipedia": search_wiki_caller,
}


import os
import threading
import numpy as np

from pupillib.core.workers.processors.decorator_registrar import *
from pupillib.core.utilities.MPLogger import MultiProcessingLog
from pupillib.core.utilities.utilities import *

# Imports for pre and post processing functions go below this line and above
# the end line below. This is the recommended method of adding new and long
# pre and post processing functions. Import them from the folder and run
# them with some sort of main function. Also, they must only ever accept two
# parameters. Use the config dictionary to modify what you get without
# complicating the code.
#
# --------------------------- Imports start line ----------------------------#
from pupillib.core.workers.processors.processing_functions.testing_functions import *

# --------------------------- Imports end line ----------------------------#

logger = MultiProcessingLog.get_logger()

DSTREAMS_BLACKLIST = [
    'name',
    'markers',
    'dir',
    'custom_data',
    'merged',
    'dataname_list',
    'dataset_name'
]

class DatasetDefaults():
    @staticmethod
    def pre_defaults():
        return [{'name': 'custom_resample_stream', 'config': [{'srate': 256}]}]

    @staticmethod
    def post_defaults():
        return []

# Provided by Pupil Labs:
# https://github.com/pupil-labs/pupil-docs/blob/master/user-docs/data-format.md#synchronization
def correlate_data(data,timestamps):
    '''
    data:  list of data :
        each datum is a dict with at least:
            timestamp: float

    timestamps: timestamps list to correlate  data to

    this takes a data list and a timestamps list and makes a new list
    with the length of the number of timestamps.
    Each slot contains a list that will have 0, 1 or more associated data points.

    Finally we add an index field to the datum with the associated index
    '''
    timestamps = list(timestamps)
    data_by_frame = [[] for i in timestamps]

    frame_idx = 0
    data_index = 0

    data.sort(key=lambda d: d['timestamp'])

    while True:
        try:
            datum = data[data_index]
            # we can take the midpoint between two frames in time: More appropriate for SW timestamps
            ts = ( timestamps[frame_idx]+timestamps[frame_idx+1] ) / 2.
            # or the time of the next frame: More appropriate for Sart Of Exposure Timestamps (HW timestamps).
            # ts = timestamps[frame_idx+1]
        except IndexError:
            # we might loose a data point at the end but we don't care
            break

        if datum['timestamp'] <= ts:
            datum['index'] = frame_idx
            data_by_frame[frame_idx].append(datum)
            data_index +=1
        else:
            frame_idx+=1

    return data_by_frame

class DatasetProcessor():
    def __init__(self):
        pre = makeregistrar()
        post = makeregistrar()

        @pre
        def tester(trigger_data, config):
            print('helloooooo')

        @post
        def tester2(trigger_data, config):
            print('done.')

        @post
        def tester3(trigger_data, config):
            a_test_to_do('Print this!')

        @post
        def synch_streams(dataset_data, config):
            print('here')

        def pick_best_stream(dataset_data):
            all_data = {
                dset: dataset_data['dataset'][dset]
                for dset in dataset_data['dataset']
                if dset not in DSTREAMS_BLACKLIST
            }

            data_mins = {}
            data_maxs = {}
            for dataname in all_data:
                data_mins[dataname] = all_data[dataname]['timestamps'][0]
                data_maxs[dataname] = all_data[dataname]['timestamps'][-1]

            abs_min = 0
            abs_max = 0
            min_dataname = ''
            max_dataname = ''
            done = 0
            for dataname in data_mins:
                if done == 0:
                    abs_min = data_mins[dataname]
                    abs_max = data_maxs[dataname]
                    min_dataname = dataname
                    max_dataname = dataname
                    done += 1
                    continue
                if data_mins[dataname] > abs_min:
                    abs_min = data_mins[dataname]
                    min_dataname = dataname
                if data_maxs[dataname] < abs_max:
                    abs_max = data_maxs[dataname]
                    max_dataname = dataname

            marker_start = dataset_data['dataset']['markers']['timestamps'][0]
            marker_end = dataset_data['dataset']['markers']['timestamps'][-1]

            # Log a warning if the differences in streams
            # is greater than the marker start
            if abs_min > marker_start:
                logger.error(
                    "Marker times start before the data stream starts, caused by data stream %s" %
                    min_dataname
                )
                logger.warning(
                    "Clipping markers out..."
                )
                import time
                time.sleep(5)

                orig_trials = len(dataset_data['dataset']['markers']['timestamps'])

                inds = np.where(dataset_data['dataset']['markers']['timestamps'] > abs_min)[0]
                mtarr = np.asarray(dataset_data['dataset']['markers']['timestamps'])
                mearr = np.asarray(dataset_data['dataset']['markers']['eventnames'])

                dataset_data['dataset']['markers']['timestamps'] = list(mtarr[inds])
                dataset_data['dataset']['markers']['eventnames'] = list(mearr[inds])

                logger.warning("%s/%s markers left after clipping" % (len(inds), orig_trials))

            if abs_max < marker_end:
                logger.error(
                    "Marker times end after the data stream, caused by data stream %s" %
                    max_dataname
                )
                logger.warning(
                    "Clipping markers out..."
                )
                import time
                time.sleep(5)

                orig_trials = len(dataset_data['dataset']['markers']['timestamps'])

                inds = np.where(dataset_data['dataset']['markers']['timestamps'] < abs_max)[0]
                mtarr = np.asarray(dataset_data['dataset']['markers']['timestamps'])
                mearr = np.asarray(dataset_data['dataset']['markers']['eventnames'])

                dataset_data['dataset']['markers']['timestamps'] = list(mtarr[inds])
                dataset_data['dataset']['markers']['eventnames'] = list(mearr[inds])

                logger.warning("%s/%s markers left after clipping" % (len(inds), orig_trials))

            if len(dataset_data['dataset']['markers']['timestamps']) == 0:
                raise Exception(
                    "No markers found in the requested data streams after clipping."
                )

            return min_dataname, max_dataname

        def clean_nan_data(dataset_data):
            '''
            Removes nan rows from the data streams before performing
            the resampling operation.
            :param dataset_data: All data, with data streams in 'dataset'
            :return: cleaned dataset_data
            '''
            new_dataset_data = dataset_data

            for dset in dataset_data['dataset']:
                d = dataset_data['dataset'][dset]

                if type(d) is not dict:
                    continue
                if 'data' not in d:
                    continue

                numbers = np.where(np.isfinite(d['data']))[0]

                arrd = np.asarray(d['data'])
                arrt = np.asarray(d['timestamps'])

                new_dataset_data['dataset'][dset] = {
                    'data': list(arrd[numbers]),
                    'timestamps': list(arrt[numbers])
                }
                new_dataset_data['dataset'][dset]['srate'] = len(new_dataset_data['dataset'][dset]['data'])/(
                    new_dataset_data['dataset'][dset]['timestamps'][-1] -
                    new_dataset_data['dataset'][dset]['timestamps'][0]
                )

            return new_dataset_data

        @pre
        def custom_resample_stream(dataset_data, config):
            print('Resampling data streams to regular sampling rates...')
            args = config['config']
            logger = MultiProcessingLog.get_logger()

            # Get srate
            srate = args[0]['srate']
            if len(dataset_data['dataset']) <= 1:
                return dataset_data

            dataset_data = clean_nan_data(dataset_data)

            for dset in dataset_data['dataset']:
                d = dataset_data['dataset'][dset]
                if type(d) is not dict:
                    continue
                if 'data' not in d:
                    continue
                dataset_data['dataset'][dset]['data'] = list(np.nan_to_num(d['data']))

            min_dataname, max_dataname = pick_best_stream(dataset_data)
            min_datastream = dataset_data['dataset'][min_dataname]['timestamps']
            max_datastream = dataset_data['dataset'][max_dataname]['timestamps']
            global_min = min_datastream[0]
            global_max = max_datastream[-1]

            all_data = {
                dataname: dataset_data['dataset'][dataname]
                    for dataname in dataset_data['dataset']
                    if dataname not in DSTREAMS_BLACKLIST
            }
            for dataname in all_data:
                dstream = all_data[dataname]
                data_ts = dstream['timestamps']
                data_dset = dstream['data']

                # Replace start points
                cur_start = 0
                cur_min = data_ts[0]
                new_data_ts = data_ts
                new_data_dset = data_dset
                if cur_min != global_min:
                    new_data_ts = []
                    for count, _ in enumerate(data_ts[:-1]):
                        if data_ts[count] < global_min <= data_ts[count+1]:
                            cur_start = count
                            break

                    # Replace first data points
                    new_data_ts = data_ts[cur_start:]
                    new_data_dset = data_dset[cur_start:]

                    # Interpolate a new data point
                    new_data_dset[0] = linear_approx(
                        new_data_dset[0], new_data_ts[0],
                        new_data_dset[1], new_data_ts[1],
                        global_min
                    )
                    new_data_ts[0] = global_min

                # Replace end points
                cur_max = data_ts[-1]
                if cur_max != global_max:
                    cur_end = len(data_ts) - 1
                    cur_end_offset = 0
                    reved_ts = data_ts[::-1]
                    for count, _ in enumerate(reved_ts):
                        if reved_ts[count] >= global_max > reved_ts[count+1]:
                            cur_end_offset = count
                            break

                    # Second value of slice is exclusive, add 1
                    # to keep the value (it's going to be replaced)
                    cur_end = cur_end - cur_end_offset + 1
                    new_data_ts = new_data_ts[:cur_end]
                    new_data_dset = new_data_dset[:cur_end]

                    # Interpolate a new data point
                    new_data_dset[-1] = linear_approx(
                        new_data_dset[-2], new_data_ts[-2],
                        new_data_dset[-1], new_data_ts[-1],
                        global_max
                    )
                    new_data_ts[-1] = global_max

                new_dstream = dstream
                if srate != 'None' and srate is not None:
                    print('Resampling data to ' + str(srate) + 'Hz...')
                    total_time = global_max - global_min
                    new_xrange = np.linspace(global_min, global_max, num=srate * (total_time))

                    new_dstream['data'] = np.interp(new_xrange, new_data_ts, new_data_dset)
                    new_dstream['timestamps'] = new_xrange

                    dataset_data['dataset'][dataname] = new_dstream

            for dset in dataset_data['dataset']:
                if dset in DSTREAMS_BLACKLIST:
                    continue
                print(
                    "Len for " + dset + ": " + str(len(dataset_data['dataset'][dset]['data']))
                )
            return dataset_data

        self.pre_processing = pre
        self.post_processing = post
        self.rejected_streams = []

    def reject_streams(self, dataset_data):
        all_data = {
            dset: dataset_data['dataset'][dset]
                for dset in dataset_data['dataset']
                if dset not in DSTREAMS_BLACKLIST
        }

        if not all_data or len(all_data.keys()) <= 0:
            self.rejected_streams.append('all')
            return ['all']

        for dataname in all_data:
            try:
                data = all_data[dataname]['timestamps']
                min_data = data[0]
                max_data = data[-1]
                if min_data == max_data:
                    print("ERROR: Datastream " + dataname + "has only one value in it, skipping it.")
                    self.rejected_streams.append(dataname)
            except Exception as e:
                self.rejected_streams.append(dataname)
        return self.rejected_streams


import os
import hashlib
from io import open

import emails
import markdown
from bs4 import BeautifulSoup

from mdmail.helpers import sanitize_email_address, is_string

def send(email, subject=None,
         from_email=None, to_email=None,
         cc=None, bcc=None, reply_to=None,
         smtp=None):
    """Send markdown email

    Args:
        email (str/obj): A markdown string or EmailContent object
        subject (str): subject line
        from_email (str): sender email address
        to_email (str/list): recipient email addresses
        cc (str/list): CC email addresses (string or a list)
        bcc (str/list): BCC email addresses (string or a list)
        reply_to (str): Reply-to email address
        smtp (dict): SMTP configuration (dict)

    Schema of smtp dict:
        host (str): SMTP server host. Default: localhost
        port (int): SMTP server port. Default: 25
        tls (bool): Use TLS. Default: False
        ssl (bool): Use SSL. Default: False
        user (bool): SMTP login user. Default empty
        password (bool): SMTP login password. Default empty
    """
    if is_string(email):
        email = EmailContent(email)

    from_email = sanitize_email_address(from_email or email.headers.get('from'))
    to_email = sanitize_email_address(to_email or email.headers.get('to'))
    cc = sanitize_email_address(cc or email.headers.get('cc'))
    bcc = sanitize_email_address(bcc or email.headers.get('bcc'))
    reply_to = sanitize_email_address(reply_to or email.headers.get('reply-to'))

    message_args = {
        'html': email.html,
        'text': email.text,
        'subject': (subject or email.headers.get('subject', '')),
        'mail_from': from_email,
        'mail_to': to_email
    }
    if cc:
        message_args['cc'] = cc
    if bcc:
        message_args['bcc'] = bcc
    if reply_to:
        message_args['headers'] = {'reply-to': reply_to}

    message = emails.Message(**message_args)

    for filename, data in email.inline_images:
        message.attach(filename=filename, content_disposition='inline', data=data)

    message.send(smtp=smtp)

class EmailContent(object):
    def __init__(self, content, css=None, image_root='.'):
        """ Constructor

        Args:
            content (str): Markdown text
            css (str): Custom CSS style. If not set, use default CSS style.
            image_root (str): Root directory for inline images.
        """
        self._md = markdown.Markdown(extensions=[
            'markdown.extensions.tables',
            'markdown.extensions.meta'])
        self._html = None
        self._inline_images = None
        self._convert(content, css, image_root)

    def _replace_inline_images(self, html, image_root):
        soup = BeautifulSoup(html, "lxml")
        imgs = soup.findAll('img')
        inlines = {}
        for img in imgs:
            src = img['src']
            if not src.startswith('http'):
                src_uid = '{}_{}'.format(
                    hashlib.md5(src.encode('utf-8')).hexdigest()[:6],
                    os.path.basename(src))
                if src_uid not in inlines:
                    img_path = os.path.join(image_root, src)
                    inlines[src_uid] = open(img_path, 'rb')
                img['src'] = 'cid:'+src_uid
        self._inline_images = inlines.items()
        return soup

    def _inline_css(self, html, css):
        if not css:
            default_css = os.path.join(os.path.dirname(__file__), 'default.css')
            css = open(default_css, encoding='utf-8').read()
        email_html_template = u"""
        <!doctype html>
        <html>
            <head>
                <meta charset="UTF-8">
                <style>
                    {css}
                </style>
            </head>
            <body>
                {content}
            </body>
        </html>"""
        return email_html_template.format(css=css, content=html)

    def _convert(self, content, css, image_root):
        raw_html = self._md.convert(content)
        html = self._replace_inline_images(raw_html, image_root)
        html = self._inline_css(html, css)
        self._html = BeautifulSoup(html, "lxml").prettify()

    @property
    def html(self):
        return self._html

    @property
    def text(self):
        return u'\n'.join(self._md.lines)

    @property
    def headers(self):
        return {k.lower(): (v[0] if len(v) == 1 else v) for k,v in self._md.Meta.items()}

    @property
    def inline_images(self):
        return self._inline_images


from django.shortcuts import render, redirect, get_object_or_404
from django.http import HttpResponse
from django.contrib.auth.forms import UserCreationForm
from django.conf import settings
from django.contrib.auth.models import User
from .models import DogPost, Comment, CommentVoteLog
from .forms import QuestionForm, CommentForm, CommentVoteLogForm
from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_GET, require_POST
from django.utils import timezone

# Create your views here.

def homepage(request):
    if request.user.is_authenticated:
        dogposts = DogPost.objects.order_by('-created_at')
        return render(request, 'core/home.html', {'dogposts': dogposts})
    else:
        return redirect('accounts/login/')

@login_required
def profile(request):
    posts = DogPost.objects.filter(user = request.user)
    comments = Comment.objects.filter(commentUser = request.user)
    return render(request, 'core/profile.html', {'posts': posts, 'comments': comments, } )

@login_required
def postdetail(request, pk):
    dogpost = DogPost.objects.get(pk=pk)
    coments = Comment.objects.filter(dogpost_id=pk).order_by('-created_at')
    if request.method == "POST" and "subcomment":
        form = CommentForm(request.POST)
        commentBody = request.POST.get('body')
        if form.is_valid():
            post = form.save(commit=False)
            post.commentUser = request.user
            post.created_at = timezone.now()
            post.dogpost = dogpost
            post.save()
            return redirect(f'/goodboi/{pk}/')

    else:
        form = CommentForm()
    return render(request, 'core/dogpostdetail.html', {'dogpost': dogpost, 'comments': coments, 'form': form, })

@login_required
def vote(request, pk):
    if request.method == "POST" and "upvote":
        form = CommentVoteLogForm(request.POST)
        comment = get_object_or_404(Comment, pk=pk)
        if form.is_valid():
            obj, created = CommentVoteLog.objects.update_or_create(
                voter = request.user, comment = comment,
                defaults={'rankValue': 1}
            )
            # obj.save()
            return redirect(f'/goodboi/{comment.dogpost.pk}/')
    elif request.method == "POST" and "downvote":
        form = CommentVoteLogForm(request.POST)
        comment = get_object_or_404(Comment, pk=pk)
        if form.is_valid():
            obj, created = CommentVoteLog.objects.update_or_create(
                voter = request.user, comment = comment,
                defaults={'rankValue': -1}
            )
            # obj.save()
            return redirect(f'/goodboi/{comment.dogpost.pk}/')

@login_required
def create_post(request):
    if request.method == "POST":
        form = QuestionForm(request.POST)
        title = request.POST.get('title')
        body = request.POST.get('body')
        if form.is_valid():
            post = form.save(commit=False)
            post.user = request.user
            post.created_at = timezone.now()
            post.save()
            return redirect('profile')
    else:
        form = QuestionForm()
    return render(request, 'core/newbork.html', {'form': form})

@login_required
def createComment(request):

    pass


from turtle import Turtle
from paddle import Paddle
from bricks import Bricks
from scoreboard import Scoreboard

TRAVEL_DIST = 10
paddle = Paddle()
bricks = Bricks()
score = Scoreboard()

class Ball(Turtle):
    def __init__(self):
        super().__init__()
        self.shape("circle")
        self.color("alice blue")
        self.penup()
        self.x_move_dist = TRAVEL_DIST
        self.y_move_dist = TRAVEL_DIST
        self.reset()

    def move(self):
        new_x = self.xcor() + self.x_move_dist
        new_y = self.ycor() + self.y_move_dist
        self.goto(new_x, new_y)

    def bounce(self, x_bounce, y_bounce):
        if x_bounce:
            self.x_move_dist *= -1
        if y_bounce:
            self.y_move_dist *= -1

    def reset(self):
        self.goto(x=0, y=-240)
        self.y_move_dist = 10


from django.test import TestCase
from django.test import Client
from django.contrib.auth.models import User
from forward_app.core_models import Policy
from forward_app.core_models import Comment
from forward_app.core_models import Popularity
from forward_app.core_models import Thread

"""
Tests for all functionality with Policy object
"""
class PolicyTest(TestCase):

    def setUp(self):
        Policy.objects.create(category=1, name='Universal Health Care', statement='Healthcare for all', description='Medical services to all citizens')

    def testCategory(self):
        policy = Policy.objects.get(name='Universal Health Care')
        self.assertEqual(policy.category, 1)

    def testStatement(self):
        policy = Policy.objects.get(name='Universal Health Care')
        self.assertEqual(policy.statement, 'Healthcare for all')

    def testDescription(self):
        policy = Policy.objects.get(description='Medical services to all citizens')
        self.assertEqual(policy.description, 'Medical services to all citizens')

"""
Tests for all functionality with Popularity object
"""

class PopularityTest(TestCase):

    def setUp(self):
        Policy.objects.create(category=1, name='Universal Health Care', statement='Healthcare for all', description='Medical services to all citizens')

    def testPopularity(self):
        policy = Policy.objects.get(name='Universal Health Care')
        popularity = Popularity.objects.create(policy=policy, likes=200)
        self.assertEquals(popularity.likes, 200)
        self.assertEqual(popularity.policy, policy)

"""
Tests for all functionality with Comment object
"""

class CommentTest(TestCase):

    def setUp(self):
        Policy.objects.create(category=1, name='Universal Health Care', statement='Healthcare for all', description='Medical services to all citizens')

    def testComment(self):
        user = User.objects.create_user(username='brian')
        policy = Policy.objects.get(name='Universal Health Care')
        popularity = Popularity.objects.create(policy=policy, likes=20)
        thread = Thread.objects.create(popularity=popularity)

        comment = Comment.objects.create(user=user, thread=thread, content='good policy', likes=5)
        thread.lead_comment_id = comment.id
        comment.next_comment_id = comment.id

        self.assertEqual(thread.popularity, popularity)
        self.assertEqual(comment.user, user)
        self.assertEqual(comment.content, 'good policy')
        self.assertEqual(comment.likes, 5)

    def testThread(self):
        user = User.objects.create_user(username='brian')
        policy = Policy.objects.get(name='Universal Health Care')
        popularity = Popularity.objects.create(policy=policy, likes=20)
        thread = Thread.objects.create(popularity=popularity)
        comment = Comment.objects.create(user=user, thread=thread, content='good policy', likes=5)
        thread.lead_comment_id = comment.id
        comment.next_comment_id = comment.id

        comment2 = Comment.objects.create(user=user, thread=thread, content='I agree', likes=2)
        comment2.next_commend_id = comment2.id
        comment.next_comment_id = comment2.id

        self.assertEqual(Comment.objects.get(id=thread.lead_comment_id), comment)
        self.assertEqual(comment.next_comment_id, comment2.id)


# Patrick Komiske, Eric Metodiev, MIT, 2017
#
# Contains two basic Keras models, a convolutional one with three convolutional
# layers and one dense layer and a dense model with an extensible number of
# layers.

from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D, MaxPooling2D, SpatialDropout2D
from keras import backend as K
from keras.callbacks import EarlyStopping

# ensure Theano dimension ordering
K.set_image_dim_ordering('th')

def conv_net_construct(hps):

    nb_conv = hps['nb_conv']
    nb_pool = hps['nb_pool']
    img_size = hps['img_size']
    nb_filters = hps['nb_filters']
    nb_channels = hps['nb_channels']
    nb_neurons = hps['nb_neurons']
    dropout = hps['dropout']
    act = hps.setdefault('act', 'relu')

    model = Sequential()
    model.add(Convolution2D(nb_filters[0], nb_conv[0], nb_conv[0],
                            input_shape = (nb_channels, img_size, img_size),
                            init = 'he_uniform', border_mode = 'valid'))
    model.add(Activation(act))
    model.add(MaxPooling2D(pool_size = (nb_pool[0], nb_pool[0])))
    model.add(SpatialDropout2D(dropout[0]))

    model.add(Convolution2D(nb_filters[1], nb_conv[1], nb_conv[1],
                            init='he_uniform', border_mode = 'valid'))
    model.add(Activation(act))
    model.add(MaxPooling2D(pool_size=(nb_pool[1], nb_pool[1])))
    model.add(SpatialDropout2D(dropout[1]))

    model.add(Convolution2D(nb_filters[2], nb_conv[2], nb_conv[2],
                            init='he_uniform', border_mode = 'valid'))
    model.add(Activation(act))
    model.add(MaxPooling2D(pool_size=(nb_pool[2], nb_pool[2])))
    model.add(SpatialDropout2D(dropout[2]))

    model.add(Flatten())

    model.add(Dense(nb_neurons))
    model.add(Activation(act))
    model.add(Dropout(dropout[3]))

    model.add(Dense(2))
    model.add(Activation('softmax'))

    model.compile(loss = 'categorical_crossentropy', optimizer = 'adam',
                  metrics = ['accuracy'])
    model.summary()

    return model

def dense_net_construct(hps):
    layer_dims = hps['layer_dims']
    input_dim = hps['input_dim']

    model = Sequential()
    for i,dim in enumerate(layer_dims):
        if i == 0:
            model.add(Dense(dim, input_dim = input_dim, init = 'he_uniform'))
        else:
            model.add(Dense(dim, init = 'he_uniform'))
        model.add(Activation(hps.setdefault('act', 'relu')))

    model.add(Dense(2))
    model.add(Activation('softmax'))

    model.compile(loss = 'categorical_crossentropy', optimizer = 'adam',
                  metrics = ['accuracy'])
    model.summary()

    return model


from random import random
import cv2
import time
from datetime import datetime
import os
import Switcher
import random
import HandTrackingModule as htm
wCam , hCam = 648, 468
cap = cv2.VideoCapture(0)
cap.set(3, wCam)
cap.set(4, hCam)
pTime = 0
checkFinger = Switcher.PythonSwitch()
detector = htm.handDetector()
tipIds = [4, 8, 12, 16, 20]
random_number = random.randint(1, 5)
time_count = 0
timer_counter = 0
start_time = 0
duration = 2

def setTimer(timer_counter, start_time):
    timer_counter += 1
    start_time = datetime.now()
    timer = True
    return start_time, timer, timer_counter

while True:
    ret, img = cap.read()
    cTime = time.time()
    fps = 1/(cTime-pTime)
    pTime = cTime
    img, landMarks = detector.findHands(img, draw=True)
    lmList = detector.findPosition(img, draw=False)
    cv2.putText(img, "Show finger number:" + str(random_number), (0, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (255,0,0), 2)
    if timer_counter == 0:
            start_time, timer, timer_counter = setTimer(timer_counter, start_time)
    if timer:
        end_time = time.time()
        diff = (datetime.now() - start_time).seconds
        cv2.putText(img, "Count Down" + str(diff), (0, 300), cv2.FONT_HERSHEY_PLAIN, 2, (255,0,255), 2)
        if diff >= duration:
            timer = False
    if len(lmList) != 0:
        fingers = []

        # Thumb
        if lmList[tipIds[0]][1] > lmList[tipIds[0]-1][1]:
            fingers.append(1)
        else:
            fingers.append(0)
        # Fingers
        for id in range(1,5):
            if lmList[tipIds[id]][2] < lmList[tipIds[id]-2][2]:
                fingers.append(1)
            else:
                fingers.append(0)

        # if timer_counter == 0:
        #     start_time, timer, timer_counter = setTimer(timer_counter, start_time)
        if landMarks != None:
            if timer:
                res = checkFinger.CheckShwonFinger(random_number, fingers)
                if res:
                    cv2.putText(img, "User authenticated as dynamic", (0, 400), cv2.FONT_HERSHEY_PLAIN, 1, (0,255,0), 1)
                else:
                    cv2.putText(img, "User not authenticated as dynamic", (0, 400), cv2.FONT_HERSHEY_PLAIN, 1, (0,0,255), 1)
                # end_time = time.time()
                # diff = (datetime.now() - start_time).seconds
                # cv2.putText(img, "Count Down" + str(diff), (0, 300), cv2.FONT_HERSHEY_PLAIN, 2, (255,0,255), 2)

    cv2.putText(img, f'FPS: {int(fps)}', (400, 70), cv2.FONT_HERSHEY_PLAIN, 3, (255,0,0), 3)

    cv2.imshow("image", img)
    cv2.waitKey(1)


# -*- coding: utf-8 -*-
"""
# -*- coding: utf-8 -*-
@author: cami_
Created on Thu Feb 27 11:02:16 2020
NOISYmputer python FILTERING ON HTZ %, MISSING DATA %, MAF
@author: Chrystian Sosa
@Correspondence author: Dr Mathias Lorieux
"""
#IMPORTING MODULES
import pandas as pd
import os.path
from tqdm import tqdm

#nthreads = 4
#size = 10**5  # CHANGED

def filtering_snp_step_chr(chr_dir,n_chrs,maxFreqMD,maxFreqH,minFreqA,minFreqB,log_dir):
    for i in tqdm(range(len(n_chrs)),desc="filtering snps"):
        n_chr=str(n_chrs[i])
        print(n_chr)
        filtering_snp_step(chr_dir,n_chr,maxFreqMD,maxFreqH,minFreqA,minFreqB)
    return("DONE!")

def filtering_snp_step(chr_dir,n_chr,maxFreqMD,maxFreqH,minFreqA,minFreqB):

    export_file_path = (chr_dir+"/"+"Chr"+n_chr+"_step_2.txt")
    export_file_path_filt = (chr_dir+"/"+"Chr"+n_chr+"_step_3.txt")

    retain =[]
    with open(export_file_path) as fp:
       for line in fp:
            nbA = line.count("A")
            nbB = line.count("B")
            nbH = line.count("H")
            nbMD = line.count("-")
            Npop = nbA + nbB + nbH + nbMD

            if Npop > 0:
                freqA = nbA / Npop
                freqB = nbB / Npop
                freqH = nbH / Npop
                freqMD = nbMD / Npop

                if freqH < maxFreqH and freqMD < maxFreqMD and freqA > minFreqA and freqB > minFreqB:
                    retain.append(True)
                else:
                    retain.append(False)
            else:
                retain.append(False)

    #retain.remove(0)
    del retain[0]
    if(os.path.isfile(export_file_path_filt)):
        return(print(export_file_path+" already processed!"))
    else:

        split_file2 = pd.read_csv(export_file_path, sep=" ")
        split_file = pd.read_csv(export_file_path, sep=" ")
        split_file['filtered'] = retain
        split_file = split_file[split_file.filtered==True]
        split_file = split_file.drop(columns=['filtered'])
#        split_file = split_file.drop_duplicates()
        split_file.to_csv(export_file_path_filt,index = False, header=True,sep=" ")

        changed =split_file2.shape[0]-split_file.shape[0]
        data =[["step","prefiltering"],
                                  ["chr",n_chr],
                                  ["changes",changed]]
        log_file_df = pd.DataFrame(data,columns=["item","status"])
        log_file_df.to_csv(log_dir+"/"+"Chr"+n_chr+"_step3.log",index = False, header=True)

        return(print(export_file_path+" filtered!"))
########################################

#chr_dir = "E:/CHR"
#maxFreqMD = 0.666
#maxFreqH = 0.800
#minFreqA = 0.010
#minFreqB = 0.010
#n_chrs=12
#n_chr = str(1)
#x_filtered = filtering_snp_step(chr_dir,n_chr,maxFreqMD,maxFreqH,minFreqA,minFreqB)

#filtering_snp_step_chr(chr_dir,n_chrs,maxFreqMD,maxFreqH,minFreqA,minFreqB)


import os

from google.protobuf import message
from google.protobuf import text_format
from tensorflow.core.protobuf import saved_model_pb2
from tensorflow.python.lib.io import file_io
from tensorflow.python.saved_model import constants
from tensorflow.python.util import compat

def read_saved_model(saved_model_dir):
  """Reads the saved_model.pb or saved_model.pbtxt file containing `SavedModel`.

  Args:
    saved_model_dir: Directory containing the SavedModel file.

  Returns:
    A `SavedModel` protocol buffer.

  Raises:
    IOError: If the file does not exist, or cannot be successfully parsed.
  """
  # Build the path to the SavedModel in pbtxt format.
  path_to_pbtxt = os.path.join(
      compat.as_bytes(saved_model_dir),
      compat.as_bytes(constants.SAVED_MODEL_FILENAME_PBTXT))
  # Build the path to the SavedModel in pb format.
  path_to_pb = os.path.join(
      compat.as_bytes(saved_model_dir),
      compat.as_bytes(constants.SAVED_MODEL_FILENAME_PB))

  # Ensure that the SavedModel exists at either path.
  if not file_io.file_exists(path_to_pbtxt) and not file_io.file_exists(
      path_to_pb):
    raise IOError("SavedModel file does not exist at: %s" % saved_model_dir)

  # Parse the SavedModel protocol buffer.
  saved_model = saved_model_pb2.SavedModel()
  if file_io.file_exists(path_to_pb):
    with file_io.FileIO(path_to_pb, "rb") as f:
      file_content = f.read()
    try:
      saved_model.ParseFromString(file_content)
      return saved_model
    except message.DecodeError as e:
      raise IOError("Cannot parse proto file %s: %s." % (path_to_pb, str(e)))
  elif file_io.file_exists(path_to_pbtxt):
    with file_io.FileIO(path_to_pbtxt, "rb") as f:
      file_content = f.read()
    try:
      text_format.Merge(file_content.decode("utf-8"), saved_model)
      return saved_model
    except text_format.ParseError as e:
      raise IOError("Cannot parse pbtxt file %s: %s." % (path_to_pbtxt, str(e)))
  else:
    raise IOError("SavedModel file does not exist at: %s/{%s|%s}" %
                  (saved_model_dir, constants.SAVED_MODEL_FILENAME_PBTXT,
                   constants.SAVED_MODEL_FILENAME_PB))

def get_saved_model_tag_sets(saved_model_dir):
  """Retrieves all the tag-sets available in the SavedModel.

  Args:
    saved_model_dir: Directory containing the SavedModel.

  Returns:
    List of all tag-sets in the SavedModel, where a tag-set is represented as a
    list of strings.
  """
  saved_model = read_saved_model(saved_model_dir)
  all_tags = []
  for meta_graph_def in saved_model.meta_graphs:
    all_tags.append(list(meta_graph_def.meta_info_def.tags))
  return all_tags

def get_meta_graph_def(saved_model_dir, tag_set):
  """Gets MetaGraphDef from SavedModel.

  Returns the MetaGraphDef for the given tag-set and SavedModel directory.

  Args:
    saved_model_dir: Directory containing the SavedModel to inspect.
    tag_set: Group of tag(s) of the MetaGraphDef to load, in string format,
        separated by ','. The empty string tag is ignored so that passing ''
        means the empty tag set. For tag-set contains multiple tags, all tags
        must be passed in.

  Raises:
    RuntimeError: An error when the given tag-set does not exist in the
        SavedModel.

  Returns:
    A MetaGraphDef corresponding to the tag-set.
  """
  saved_model = read_saved_model(saved_model_dir)
  # Note: Discard empty tags so that "" can mean the empty tag set.
  set_of_tags = set([tag for tag in tag_set.split(",") if tag])

  valid_tags = []
  for meta_graph_def in saved_model.meta_graphs:
    meta_graph_tags = set(meta_graph_def.meta_info_def.tags)
    if meta_graph_tags == set_of_tags:
      return meta_graph_def
    else:
      valid_tags.append(",".join(meta_graph_tags))

  raise RuntimeError(
      f"MetaGraphDef associated with tag-set {tag_set} could not be found in "
      f"the SavedModel. Please use one of the following tag-sets: {valid_tags}")


# -*- encoding: utf-8 -*-
'''
@project :   LeetCode
@File    :   minimumLengthEncoding.py
@Contact :   9824373@qq.com
@Desc    :
                给定一个单词列表，我们将这个列表编码成一个索引字符串 S 与一个索引列表 A。

                例如，如果这个列表是 ["time", "me", "bell"]，我们就可以将其表示为 S = "time#bell#" 和 indexes = [0, 2, 5]。

                对于每一个索引，我们可以通过从字符串 S 中索引的位置开始读取字符串，直到 "#" 结束，来恢复我们之前的单词列表。

                那么成功对给定单词列表进行编码的最小字符串长度是多少呢？

                示例：

                输入: words = ["time", "me", "bell"]
                输出: 10
                说明: S = "time#bell#" ， indexes = [0, 2, 5] 。

                提示：

                1 <= words.length <= 2000
                1 <= words[i].length <= 7
                每个单词都是小写字母 。

                来源：力扣（LeetCode）
                链接：https://leetcode-cn.com/problems/short-encoding-of-words

@Modify Time                @Author     @Version    @Desciption
------------                -------     --------    -----------
2020-03-28     zhan        1.0         None
'''
from typing import List
import collections
from functools import reduce

class Solution:
    def minimumLengthEncoding(self, words: List[str]) -> int:
        # return self.minimumLengthEncoding_sort(words)
        return self.minimumLengthEncoding_trie(words)

    def minimumLengthEncoding_sort(self, words: List[str]) -> int:
        n = len(words)
        words.sort(key=lambda word:word[::-1])

        S = ''
        i = 0
        while i < len(words):
            while i < len(words)-1 and len(words[i]) <= len(words[i+1]) and words[i] == words[i+1][-len(words[i]):]:
                i +=1
            S += words[i]
            S += '#'
            i += 1

        return len(S)

    def minimumLengthEncoding_trie(self, words: List[str]) -> int:
        words = list(set(words))  # remove duplicates
        # Trie is a nested dictionary with nodes created
        # when fetched entries are missing
        Trie = lambda: collections.defaultdict(Trie)
        trie = Trie()

        # reduce(..., S, trie) is trie[S[0]][S[1]][S[2]][...][S[S.length - 1]]
        # nodes = []
        # for word in words:
        #     v = reduce(dict.__getitem__, word[::-1], trie)
        #     nodes.append(v)
        # 构建 trie ，同时对于前缀，返回非控制，对于取到叶节点，返回空值
        nodes = [reduce(dict.__getitem__, word[::-1], trie)
                 for word in words]

        # Add word to the answer if it's node has no neighbors
        return sum(len(word) + 1
                   for i, word in enumerate(words)
                   if len(nodes[i]) == 0)

if __name__ == '__main__':
    words = ["me", "time", "bell"]

    ans = Solution().minimumLengthEncoding(words)
    print(ans)


from .models import Book, Author, Seller, Genre, Format
from django.contrib.auth.models import User
from .fetch import isbn, author

def db_loader():
   data = author()
   for i in range(len(data)):
      try:
         d = data[i]
         u1 = User.objects.all()[0]
         s1 = Seller.objects.all()[0]
         try:
            a = Author.objects.get(name__contains=d['author'])
            print(d['author'],'-',d['title'])
         except:
            print(f"Invalid data!!")
            continue

         book = Book.objects.create(user=u1, author=a, seller=s1, title=d['title'], ISBN=d['ISBN'], image=d['image'], price=d['price'],
                                    discount=d['discount'], language=d['language'], publisher=d['publisher'],
                                    publication_date=d['publication_date'], pages=d['pages'], origin=d['origin'],
                                    age_group=d['age_group'], dimensions=d['dimensions'], description=d['description'],
                                    rating=d['rating'], num_reviews=d['num_reviews'], count_in_stock=d['count_in_stock'])

         for genre in d['genres']:
            try:
               g = Genre.objects.get(genre__contains=genre)
            except:
               g = Genre.objects.create(genre=genre)
            book.genres.add(g)

         for format, link in d['formats'].items():
            Format.objects.create(format=format, link=link, book=book)
      except:
         print(f"Can't load in DB!! Invalid data")


def swap(lst, index1, index2):
	lst = lst.copy()

	lst[index1], lst[index2] = lst[index2], lst[index1]

	return lst

def selection_sort(lst):
	lst = lst.copy()

	for i, x in enumerate(lst):
		min_index = i
		for j in range(i, len(lst)):
			if lst[j] < lst[min_index]:
				min_index = j

		lst = swap(lst, i, min_index)
		# lst[i], lst[min_index] = lst[min_index], lst[i]

	return lst

def insertion_sort(lst, result_list=None):
	lst = lst.copy()

	if result_list is None:
		result_list = [lst.pop(0)]

	if len(lst) == 0:
		return result_list

	new_card = lst.pop(0)

	insert = False
	for i in range(len(result_list) - 1, -1, -1):
		if new_card > result_list[i]:
			result_list = result_list[:i + 1] + [new_card] + result_list[i + 1:]
			insert = True
			break

	if not insert:
		result_list = [new_card] + result_list

	return insertion_sort(lst, result_list=result_list)

def quick_sort(lst):
	if len(lst) < 2:
		return lst

	pivot = lst[0]
	equals = [x for x in lst if x == pivot]
	smallers = [x for x in lst if x < pivot]
	largers = [x for x in lst if x > pivot]

	return quick_sort(smallers) + equals + quick_sort(largers)

def __merge_sorted(lst1, lst2):
	lst1, lst2 = lst1.copy(), lst2.copy()

	if len(lst1) == 0:
		return lst2

	if len(lst2) == 0:
		return lst1

	result_list = []

	elt1, elt2 = None, None

	while (len(lst1) != 0 or len(lst2) != 0) or (elt1 is not None or elt2 is not None):

		if elt1 is None:
			try:
				elt1 = lst1.pop(0)
			except IndexError:
				if elt2 is not None:
					return result_list + [elt2] + lst2
				else:
					return result_list + lst2

		if elt2 is None:
			try:
				elt2 = lst2.pop(0)
			except IndexError:
				if elt1 is not None:
					return result_list + [elt1] + lst1
				else:
					return result_list + lst1

		if elt1 == min(elt1, elt2):
			result_list.append(elt1)
			elt1 = None

		else:
			result_list.append(elt2)
			elt2 = None

	return result_list

def merge_sort(lst):
	if len(lst) < 2:
		return lst

	mid_index = len(lst) // 2

	left = merge_sort(lst[:mid_index])
	right = merge_sort(lst[mid_index:])

	return __merge_sorted(left, right)

def __test_merge_sorted():
	test_list_sorted = __merge_sorted(test_list, test_list2)

	assert test_list_sorted == sorted(test_list + test_list2)
	print("__merge_sorted passed.")

def __test_merge_sort():
	test_list_sorted = merge_sort(test_list)
	assert test_list_sorted == sorted(test_list)
	print("merge_sort passed.")

def __test_selection_sort():
	test_list_sorted = selection_sort(test_list)

	assert test_list_sorted == sorted(test_list)

	print("selection_sort passed.")

def __test_insertion_sort():
	test_list_sorted = insertion_sort(test_list)

	assert test_list_sorted == sorted(test_list)

	print("insertion_sort passed.")

def __test_quick_sort():
	test_list_sorted = quick_sort(test_list)
	assert test_list_sorted == sorted(test_list)
	print("quick_sort passed.")

if __name__ == '__main__':
	# test_list = [1, 1, 8, 9, 4, 5, 6, 8]

	import random
	import time

	test_list = random.sample(range(1, 100), 5)
	test_list2 = random.sample(range(1, 100), 5)
	# test_list = [1, 3, 5, 9, 10, 11, 143]
	# test_list2 = [2, 4, 6, 8, 12]

	# __test_merge_sorted()
	__test_selection_sort()
	__test_insertion_sort()
	__test_quick_sort()
	__test_merge_sort()


# Read from a CSV
with open('input.csv', 'r') as file:
    lines = file.read().split('\n')
    print(lines)

dummy_data = 'name,age,city\nsara,33,portland\nbob,99,tampa'

# Write to different csv
with open('output.csv', 'w') as file:
        file.write(dummy_data)