##################################################
# Import
##################################################
# Varie
import os, shutil
from math import ceil

# Per Random
from datetime import datetime
from random import randint
import random
random.seed(datetime.now())

# Per minizinc
import json
from minizinc import Instance, Model, Solver

##################################################
# Globali
##################################################
# Inputs
INPUT_DIR = "./inputs/"

INPUT_PREFIX = "input_"
INPUT_EXT = ".dzn"

OUTPUT_DIR = "./outputs/"
OUTPUT_PREFIX = "output_"
OUTPUT_EXT = ".json"

SOLVER='chuffed'

# Pretty print dell'Output
SYMBOLS = {
        "empty"        : "--",
        "malato"       : "-M",
        "malati"       : "MM",
        "positivo"     : "-P",
        "positivi"     : "PP",
        "osservazione" : "-O",
        "quarantena"   : "-Q",
        "quaranteni"   : "QQ"
        }

##################################################
# Lettura Scrittura Input
##################################################

# Dato un numero e suffisi e prefissi torna
# directory + fname_prefix + numero in formato {:02d} + fname_suffix
def gen_fpath(num, directory, prefix, suffix):
    fpath = directory + prefix
    fpath += '{:02d}'.format(num)
    fpath += suffix
    return fpath

# Dato un dizionario di un input lo salva nel file indicato
def write_dzn(values, fpath):
    # TODO try catch
    f = open(fpath,'w')

    comment_char = '%'

    ref_dict = {'K':-1,'H':-1,'M':-1,'P':-1,'O':-1,'Q':-1}
    if not values.keys() == ref_dict.keys():
        print("Values non contiene tutti i valori")
        exit(1)

    for k in values.keys():
        if values[k] == -1:
            print("Values non ha tutti i valori inizzializzati")
            exit(2)
        text = str(k) +  "=" + str(values[k]) + ";\n"
        f.write(text)

# Dato un file path ritorna il dizionario dell'input
def read_dzn(fpath):
    # TODO try catch
    if not os.path.isfile(fpath):
        return None

    f = open(fpath)

    comment_char = '%'
    values = {'K':-1,'H':-1,'M':-1,'P':-1,'O':-1,'Q':-1}

    lines = f.readlines()
    for l in lines:
        if comment_char in l:
            index = l.find(comment_char)
            l = l[0:index]

        l = l.replace(" ","")
        l = l.replace("\t","")
        l = l.replace("\n","")
        if not l=='' and '=' in l and ';' in l:
            l = l.replace(";","")
            k,v = l.split('=')
            if k in values.keys():
                # TODO try catch su int
                values[k] = int(v)

    for k in values.keys():
        if values[k] == -1:
            print("Not well formatted input")
            exit(2)

    return values

# Dato un numero ritorna il dizionario dell'input relativo
def get_input(num, dest_dir=INPUT_DIR):
    #if num >  100:
    #    print("ERROR! get_input() num troppo grande")
    #    exit(2)
    fpath = gen_fpath(num, dest_dir, INPUT_PREFIX, INPUT_EXT)
    # TODO try catch
    return read_dzn(fpath)

# Dato un numero ritorna il testo dell'input relativo
def get_input_text(num, dest_dir=INPUT_DIR):
    #if num >  100:
    #    t = "ERROR! get_input() num troppo grande"
    #    print(t)
    #    return t

    fpath = gen_fpath(num, dest_dir, INPUT_PREFIX, INPUT_EXT)

    if not os.path.isfile(fpath):
        t = "ERROR! File does not exists!"
        print(t)
        return t

    f = open(fpath)
    t = f.read()
    return t

##################################################
# Generazione Input
##################################################

# Ritorna il massimo numero di persone nelle stanze
# Ipotizzo di poter mettere 2 persone per stanza
def capienza_max(values):
    return 2*values['H']*values['K']*2

# Ritorna il numero di stanze
def numero_stanze(values):
    return 2*values['H']*values['K']

# Ritorna il numero di stanze minimo
# per ospitare le persone indicate
def stanze_necessarie(values):
    return ceil(values['M']/2) +\
           ceil(values['P']/2) +\
           values['O'] +\
           ceil(values['Q']/2)

# Ritorna dizionario con chiavi M,P,O,Q
# con i valori per occupare tutte le stanze con
# due ospiti oppure uno in Osservazione
def satura_stanze(values):
    values = values.copy()
    if stanze_necessarie(values) > numero_stanze(values):
        print("IMPOSSIBLE")
        exit(1)
    keys = ['M','P','O','Q']
    while stanze_necessarie(values) <= numero_stanze(values):
        rk = randint(0,len(keys)-1)
        values[keys[rk]] += 1
        last_key = keys[rk]

    values[last_key] -= 1
    # Se ho valori dispari significa che ho una sola persona
    # in una stanza, se posso ne inserisco un'altra
    for k in ['M','P','Q']:
        if values[k]%2 == 1:
            values[k] += 1

    if stanze_necessarie(values) > numero_stanze(values):
        print("DEBUG: qualcosa non va...")
        print(values)

    return values

def istanza_casuale(values):
    values = satura_stanze(values)
    if randint(0,2) != 2: # due volte su tre
        keys = ['M','P','O','Q']
        # rimuovo un po' di persone a caso
        n_rimuovere = randint(1, capienza_max(values)//4)
        rimosse=0
        while rimosse != n_rimuovere:
            key = keys[randint(0,len(keys)-1)]
            da_rimuovere = randint(1,n_rimuovere-rimosse)
            if values[key] >= da_rimuovere:
                values[key] -= da_rimuovere
                rimosse += da_rimuovere
    else:
        # dimezzo Malati, Positivi o Quarantena
        keys = ['M','P','Q']
        key = keys[randint(0,len(keys)-1)]
        values[key] = values[key]//2 + values[key]%2 #mantengo dispari se era disp

    # Se nessun valore e' > 0 ... qualcosa non va
    qulcs_non_va = True
    for key in keys:
        qulcs_non_va = qulcs_non_va and (values[key] == 0)
    if qulcs_non_va:
        print("ERROR! Qualcosa non va... Values tutti == 0")

    # Verifico che ci sia almeno un malato
    # Se non c'e' allora converto uno degli altri in malato
    if values['M'] == 0:
        key = keys[randint(0,len(keys)-1)]
        while not values[key] > 0:
            key = keys[randint(0,len(keys)-1)]
        if values[key] > 0:
            values[key] -=1
            values['M'] +=1
    return values

# Genera n istanze casuali con H e K dato
def gen_istanze(n, K, H):
    istanze = []
    values = {'K':K,'H':H,'M':0,'P':0,'O':0,'Q':0}

    for i in range(n):
        istanze.append(istanza_casuale(values))

    return istanze

# Dati K e H massimi da raggiungere genera n istanze per ogni coppia K,H
# TODO Eventualemnte modificare i for perche'
# Creare input cosi' genera istanze con difficolta' a "dente di sega"
def gen_inputs(K_max, H_max, n,
        K_min=1,H_min=2,
        dest_dir = INPUT_DIR, input_prefix=INPUT_PREFIX,
        input_extension=INPUT_EXT):

    os.makedirs(dest_dir, exist_ok=True)

    num=0 # numero istanza corrente
    for k in range(K_max,K_max+1):
        for h in range(H_min, H_max+1):
            values = {'K':k,'H':h,'M':0,'P':0,'O':0,'Q':0}
            for i in range(n):
                ist = istanza_casuale(values)

                fpath = gen_fpath(num, dest_dir,
                        input_prefix, input_extension)
                write_dzn(ist,fpath)

                num += 1
    print("Generati %d input nella cartella %s" %(num, dest_dir))

# Elimina la cartella degli input
def del_inputs(input_dir = INPUT_DIR):
    #TODO try catch?
    shutil.rmtree(input_dir, ignore_errors=True)

# Elimina la cartella degli outpus
def del_outputs(output_dir = OUTPUT_DIR):
    #TODO try catch?
    shutil.rmtree(output_dir, ignore_errors=True)

##################################################
# Minizinc Models and Intances Stuff
##################################################

# Dati instanza e numero dell'input recupera dzn relativo
# a num e lo carica nell'istanza
def initialize_instance(instance, num, dest_dir=INPUT_DIR):
    #TODO try catch
    values = get_input(num,dest_dir)

    instance["K"]=values["K"]; # corridoi
    instance["H"]=values["H"]; # stanze per lato

    instance["M"]=values["M"]; # malati
    instance["P"]=values["P"]; # positivi
    instance["O"]=values["O"]; # osservazione
    instance["Q"]=values["Q"]; # quarantena precauzionale

def show_instance(instance):
    print("K:",instance["K"])
    print("H:",instance["H"])

    print("M:",instance["M"])
    print("P:",instance["P"])
    print("O:",instance["O"])
    print("Q:",instance["Q"])

# Data una soluzione e un numero di una stanza s
# ritorna il simbolo corretto per quella stanza
# Es: -M oppure MM oppure -Q oppure -O
def get_symbol(s,sol):
    if (sol['M'].count(s) == 1):
        symb = SYMBOLS['malato']
    elif (sol['M'].count(s) == 2):
        symb = SYMBOLS['malati']

    elif (sol['P'].count(s) == 1):
        symb = SYMBOLS['positivo']
    elif (sol['P'].count(s) == 2):
        symb = SYMBOLS['positivi']

    elif (sol['O'].count(s) == 1):
        symb = SYMBOLS['osservazione']

    elif (sol['Q'].count(s) == 1):
        symb = SYMBOLS['quarantena']
    elif (sol['Q'].count(s) == 2):
        symb = SYMBOLS['quaranteni']

    else:
        symb = SYMBOLS['empty']

    return symb;

# Data una soluzione e l'istanza ritorna il dizionario sol
# della soluzione (se solution e' None torna None)
def get_sol(solution,instance, show=False):
    if solution == None:
        print("No solution")
        return None

    sol = {}
    sol['K'] = instance['K']
    sol['H'] = instance['H']

    if type(solution) != dict:
        sol['M'] = solution.malati
        sol['P'] = solution.positivi
        sol['O'] = solution.osservazione
        sol['Q'] = solution.quarantena
    else:
        print("ERROR! In get_solution solution e' gia' dict")

    if show:
        show_sol(sol)
    return sol

# Date le statistics torna il dizionario stats
def get_stats(statistics):
    stats = {}
    # valore, tempo di flat, tempo di resol, tempo totale)
    stats['method']    = statistics['method']

    try:
        stats['time'] = float(statistics['time'].total_seconds())
    except:
        stats['time'] = .0
    try:
        stats['solveTime'] = float(statistics['solveTime'].total_seconds())
    except:
        stats['solveTime'] = .0
    try:
        stats['flatTime']  = float(statistics['flatTime'].total_seconds())
    except:
        stats['flatTime']  = .0
    try:
        stats['solutions'] = statistics['solutions']
    except:
        stats['solutions'] = .0

    return stats

def show_sol(sol):
    if sol == None:
        print("No solution")
        return
    if type(sol) != dict:
        print("ERROR! In show_sol solution deve essere dict")
        return

    K = sol['K']
    H = sol['H']
    s = 0
    for k in range(K):
        for h in range(2*H):
            code = get_symbol(s,sol)
            e = " "
            if (h==H-1 or h==2*H-1 or h==2*H*K-1):
                e = "\n"
            if (h==H-1):
                code += "\t" + str(k)

            print(code, end=e)
            s+=1
        print()

def to_string_sol(sol):
    if sol == None:
        t = "No solution"
        print(t)
        return t
    if type(sol) != dict:
        t = "ERROR! In show_sol solution deve essere dict"
        print(t)
        return t

    t = ""

    K = sol['K']
    H = sol['H']
    s = 0
    for k in range(K):
        for h in range(2*H):
            code = get_symbol(s,sol)
            e = " "
            if (h==H-1 or h==2*H-1 or h==2*H*K-1):
                e = "\n"
            if (h==H-1):
                code += "\t" + str(k)

            t += code + e
            s+=1
        t += '\n'
    return t

def show_objective(obj):
    if obj is None:
        print('{:10}:{:>10}'.format('objective', 'None'))
    else:
        print('{:10}:{:10d}'.format('objective',obj))

def get_objective(obj):
    if obj is None:
        return str('{:10}:{:>10}'.format('objective', 'None'))
    else:
        return str('{:10}:{:10d}'.format('objective',obj))

def show_statistic(statistics):
    # TODO fix se statistics non e stats
    if not type(statistics) == dict:
        stats = get_stats(statistics)
    else:
        stats = statistics

    for k in stats.keys():
        val = stats[k]
        if type(val) == str:
            print("{:10}:{:>10}".format(k, stats[k]))
        elif type(val) == float:
            print("{:10}:{:10f}".format(k, stats[k]))
        elif type(val) == int:
            print("{:10}:{:10d}".format(k, stats[k]))
        elif type(val) == None:
            print("None")
        else:
            print("ERROR! in show_statistic %s tipo chiave imprevisto!" %(str(type(val))))

def to_string_statistic(statistics):
    # TODO fix se statistics non e stats
    if not type(statistics) == dict:
        stats = get_stats(statistics)
    else:
        stats = statistics
    t = ""
    for k in stats.keys():
        val = stats[k]
        if type(val) == str:
            t += str("{:10}:{:>10}".format(k, stats[k]))
            t += '\n'
        elif type(val) == float:
            t += str("{:10}:{:10f}".format(k, stats[k]))
            t += '\n'
        elif type(val) == int:
            t += str("{:10}:{:10d}".format(k, stats[k]))
            t += '\n'
        else:
            t += str("error! in to_string_statistic tipo chiave imprevisto!")
            t += '\n'
    return t

# Dati result ed instance stampa il recap della soluzione trovata
def show_result(result, instance):
    show_statistic(result.statistics)
    show_objective(result.objective)
    print()
    get_sol(result.solution, instance, show=True)

# Dati result, instance ed un file path salva il
# dizionario relativo nel path
def write_output(result, instance, fpath):
    stats = get_stats(result.statistics)
    obj = result.objective
    sol = get_sol(result.solution, instance)

    data = {}
    data['obj'] = obj
    data['stats'] = stats
    data['sol'] = sol

    # TODO try catch
    json.dump(data, open(fpath, 'w+'), indent=True)

# Dati il numero e la dir carica ritorna il dizionario
# della soluzione relativa all input col numero dato
# Se il file non esiste retun None
def read_output(num, directory=OUTPUT_DIR, suppress_error=False):
    if not directory[-1] == '/':
        directory = directory + '/'
    fpath = gen_fpath(num, directory, OUTPUT_PREFIX, OUTPUT_EXT)
    if not os.path.isfile(fpath):
        if not suppress_error:
            print("File does not exist!\n path: %s " %(fpath))
        return None

    return json.load(open(fpath))

def get_output(num, directory=OUTPUT_DIR):
    data = read_output(num, directory)
    output = ""

    output += str(to_string_statistic(data['stats']))
    output += str(get_objective(data['obj']))
    output += '\n\n'

    output += str(to_string_sol(data['sol']))
    output += '\n'
    return output

def show_output(num, directory=OUTPUT_DIR):
    data = read_output(num, directory)
    print(data)
    print()
    show_statistic(data['stats'])
    show_objective(data['obj'])
    print()
    show_sol(data['sol'])
    print()

def dir_exists(directory, label="dir", verbose=True):
    b = os.path.exists(directory)
    if ((not b) and verbose):
        print("ERRORE! La %s %s non esiste!" %(label, input_dir))
    return b

# Carica il modello del path dato e lo esegue sull'input del num dato
def run_on_input(model_path, input_num, input_dir=INPUT_DIR):
    if not dir_exists(input_dir, label='input_dir', verbose=True):
        return
    # TODO try catch
    model = Model(model_path)
    solver = Solver.lookup(SOLVER)

    instance = Instance(solver, model)
    initialize_instance(instance, input_num)
    result = instance.solve(free_search=True)

    show_result(result, instance)

# Carica il modello del path dato e lo esegue su tutti gli input
# per ciascuno salva un json nella cartella data
def run_on_all_inputs(model_path, output_dir, input_dir=INPUT_DIR):
    if not dir_exists(input_dir, label='input_dir', verbose=True):
        return

    if not output_dir[-1] == '/':
        output_dir += '/'
    os.makedirs(output_dir, exist_ok=True)

    model = Model(model_path)
    solver = Solver.lookup(SOLVER)

    input_num = 0
    while not get_input(input_num) is None:
        # Se l'output e' gia' stato calcolato non ricalcolo
        if read_output(input_num,suppress_error=True) == None:
            print("Lavoro su input num %d" %(input_num))
            instance = Instance(solver, model)
            initialize_instance(instance, input_num)
            result = instance.solve()

            output_fpath = gen_fpath(input_num, output_dir,
                    OUTPUT_PREFIX, OUTPUT_EXT)

            write_output(result, instance, output_fpath)
        else:
            print("Trovato output per input num %d. Skip" %(input_num))

        input_num += 1

def run_minizinc_model(num=-1, show_output=True):
    model_path= './covid19.mzn'
    out_dir='./outputs/'
    in_dir='./inputs/'

    if num <= -1:
        print("Running:\nmodel_path=%s\noutput_dir=%s\ninput_dir=%s"\
                %(model_path,out_dir,in_dir))

        run_on_all_inputs(model_path, out_dir, in_dir)
    else:
        print("Running:\nmodel_path=%s\nOn input num=%d" \
                %(model_path,num))
        run_on_input(model_path, num, in_dir)

##################################################
# Main
##################################################
def main():
    print("Main non ancora implementato")
    print("Probabilmente fara' dei test")
    #run_minizinc_model()

if __name__ == "__main__":
    main()


from get_data import get_data
import tensorflow as tf
import tensorflow_hub as hub
import fire
from sklearn.metrics.pairwise import cosine_similarity
import time

BATCH_SIZE = 32
MAX_SEQ_LENGTH = 128

def get_elmo_embedding(list_sentence):
    """ Returns embedding for documents """
    list_sentence = list(map(lambda x: ' '.join(x.split()[:MAX_SEQ_LENGTH]), list_sentence))
    embedding_array = []
    with tf.Graph().as_default():
        elmo = hub.Module("https://tfhub.dev/google/elmo/2")
        with tf.Session() as sess:
            sess.run(tf.global_variables_initializer())
            sess.run(tf.tables_initializer())

            total_num_documents = len(list_sentence)
            for i in range(total_num_documents//BATCH_SIZE + 1):
                start = time.time()
                try:
                    batch_sentence = list_sentence[i*BATCH_SIZE:i*BATCH_SIZE + BATCH_SIZE]
                except IndexError: # for last batch
                    batch_sentence = list_sentence[i*BATCH_SIZE:]
                embeddings = elmo(batch_sentence, signature="default", as_dict=True)["default"]

                batch_embeddings = sess.run(embeddings)

                embedding_array.append(batch_embeddings)
                end = time.time()
                print(batch_embeddings.shape)
                print(f"batch {i}, time taken: {end-start}")

    embedding_matrix = np.concatenate(embedding_array, 0)

    return embedding_matrix

def main(domain):
    docs = get_data(domain)
    texts = list(docs.values())
    embedding_matrix = get_elmo_embedding(texts)

    print(embedding_matrix.shape)
    sim = cosine_similarity(embedding_matrix)

    print(sim)

if __name__ == '__main__':
    fire.Fire(main)


# Copyright (C) 2017 Google Inc.
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>

"""
  Audit permission hooks

  The following permission hooks make sure archived audits are not editable
"""
import itertools

from werkzeug.exceptions import Forbidden

from ggrc import db
from ggrc.models import all_models
from ggrc.services import signals

_AUDIT_MODEL_NAME = 'Audit'

@signals.Restful.collection_posted.connect_via(all_models.Audit)
def handle_audit_post(sender, objects=None, sources=None):
  """Handles creating issue tracker related info."""
  del sender  # Unused
  for obj, src in itertools.izip(objects, sources):
    issue_tracker_info = src.get('issue_tracker')
    if not issue_tracker_info:
      continue
    all_models.IssuetrackerIssue.create_or_update_from_dict(
        _AUDIT_MODEL_NAME, obj.id, issue_tracker_info)

@signals.Restful.model_put.connect_via(all_models.Audit)
def handle_audit_put(sender, obj=None, src=None, service=None):
  """Handles updating issue tracker related info."""
  del sender, service  # Unused
  issue_tracker_info = src.get('issue_tracker')
  if issue_tracker_info:
    all_models.IssuetrackerIssue.create_or_update_from_dict(
        _AUDIT_MODEL_NAME, obj.id, issue_tracker_info)

@signals.Restful.model_deleted_after_commit.connect_via(all_models.Audit)
def handle_audit_deleted_after_commit(sender, obj=None, service=None,
                                      event=None):
  """Handles deleting issue tracker related info."""
  del sender, service, event  # Unused
  issue_obj = all_models.IssuetrackerIssue.get_issue(
      _AUDIT_MODEL_NAME, obj.id)
  if issue_obj:
    db.session.delete(issue_obj)

def init_hook():
  """Initializes audit permission hooks."""
  # pylint: disable=unused-variable
  @signals.Restful.model_put.connect_via(all_models.Audit)
  @signals.Restful.model_deleted.connect_via(all_models.Audit)
  def handle_audit_permission_put(sender, obj, src=None, service=None):
    """Make sure admins cannot delete/update archived audits"""
    # pylint: disable=unused-argument
    if obj.archived and not db.inspect(
       obj).get_history('archived', False).has_changes():
      raise Forbidden()

  # pylint: disable=unused-variable
  @signals.Restful.model_deleted.connect_via(all_models.Assessment)
  @signals.Restful.model_deleted.connect_via(all_models.AssessmentTemplate)
  @signals.Restful.model_posted.connect_via(all_models.Assessment)
  @signals.Restful.model_posted.connect_via(all_models.AssessmentTemplate)
  @signals.Restful.model_put.connect_via(all_models.Assessment)
  @signals.Restful.model_put.connect_via(all_models.AssessmentTemplate)
  @signals.Restful.model_put.connect_via(all_models.Snapshot)
  def handle_archived_object(sender, obj=None, src=None, service=None):
    """Make sure admins cannot delete/update archived audits"""
    # pylint: disable=unused-argument
    if obj.archived:
      raise Forbidden()

  @signals.Restful.model_deleted.connect_via(all_models.Comment)
  @signals.Restful.model_deleted.connect_via(all_models.Document)
  @signals.Restful.model_deleted.connect_via(all_models.UserRole)
  @signals.Restful.model_posted.connect_via(all_models.Comment)
  @signals.Restful.model_posted.connect_via(all_models.Document)
  @signals.Restful.model_posted.connect_via(all_models.Snapshot)
  @signals.Restful.model_posted.connect_via(all_models.UserRole)
  def handle_archived_context(sender, obj=None, src=None, service=None):
    """Make sure admins cannot delete/update archived audits"""
    # pylint: disable=unused-argument
    if (hasattr(obj, 'context') and
        hasattr(obj.context, 'related_object') and getattr(
            obj.context.related_object, 'archived', False)):
      raise Forbidden()

  @signals.Restful.model_posted.connect_via(all_models.Relationship)
  @signals.Restful.model_deleted.connect_via(all_models.Relationship)
  def handle_archived_relationships(sender, obj=None, src=None, service=None):
    """Make sure users can not map objects to archived audits"""
    # pylint: disable=unused-argument
    if (getattr(obj, 'source_type', None) == 'Issue' or
       getattr(obj, 'destination_type', None) == 'Issue'):
      # Issues can be mapped even if audit is archived so skip the permission
      # check here
      return
    if (hasattr(obj, 'context') and
        hasattr(obj.context, 'related_object') and getattr(
            obj.context.related_object, 'archived', False)):
      raise Forbidden()


#!flask/bin/python
from flask import Flask, jsonify
from flask import abort
from flask import make_response
from flask import request
from flask import url_for
from flask.ext.httpauth import HTTPBasicAuth

app = Flask(__name__)

auth = HTTPBasicAuth()

@auth.get_password
def get_password(username):
    if username == 'prasanth':
        return 'password'
    return None

@auth.error_handler
def unauthorized():
    return make_response(jsonify({'error': 'Please Enter Your Username and Password'}), 401)

reminders = [
    {
        'id': 1,
        'title': u'Market',
        'desc': u'Chips and Dips',
        'finish': False
    },
    {
        'id': 2,
        'title': u'Interview at 3 p.m',
        'desc': u'Get shirt ironed',
        'finish': False
    },
     {
        'id': 3,
        'title': u'Meeting with the Boss',
        'desc': u'Bring the report',
        'finish': False
    }

]

@app.route('/todo/api/v1/todoabc/<int:todoabc_id>', methods=['DELETE'])
def delete_todoabc(todoabc_id):
    todoabc = [todoabc for todoabc in reminders if todoabc['id'] == todoabc_id]
    if len(todoabc) == 0:
        abort(404)
    reminders.remove(todoabc[0])
    return jsonify({'result': True})

@app.route('/todo/api/v1/todoabc/<int:todoabc_id>', methods=['PUT'])
def update_todoabc(todoabc_id):
    todoabc = [todoabc for todoabc in reminders if todoabc['id'] == todoabc_id]
    if len(todoabc) == 0:
        abort(404)
    if not request.json:
        abort(400)
    if 'title' in request.json and type(request.json['title']) != unicode:
        abort(400)
    if 'desc' in request.json and type(request.json['desc']) is not unicode:
        abort(400)
    if 'finish' in request.json and type(request.json['finish']) is not bool:
        abort(400)
    todoabc[0]['title'] = request.json.get('title', todoabc[0]['title'])
    todoabc[0]['desc'] = request.json.get('desc', todoabc[0]['desc'])
    todoabc[0]['finish'] = request.json.get('finish', todoabc[0]['finish'])
    return jsonify({'todoabc': todoabc[0]})

@app.route('/todo/api/v1/todoabc', methods=['POST'])
def create_todoabc():
    if not request.json or not 'title' in request.json:
        abort(400)
    todoabc = {
        'id': reminders[-1]['id'] + 1,
        'title': request.json['title'],
        'desc': request.json.get('desc', ""),
        'finish': False
    }
    reminders.append(todoabc)
    return jsonify({'listofreminders': reminders}), 201

@app.route('/todo/api/v1/todoabc', methods=['GET'])
@auth.login_required
def get_todolists():
    return jsonify({'todolists': [make_public_todoabc(todoabc) for todoabc in reminders]})

def make_public_todoabc(todoabc):
    new_todoabc = {}
    for field in todoabc:
        if field == 'id':
            new_todoabc['uri'] = url_for('get_todoabc', todoabc_id=todoabc['id'], _external=True)
        else:
            new_todoabc[field] = todoabc[field]
    return new_todoabc
@app.errorhandler(404)
def not_found(error):
    return make_response(jsonify({'error': 'Not found'}), 404)

@app.route('/todo/api/v1/todoabc/<int:todoabc_id>', methods=['GET'])
def get_todoabc(todoabc_id):
    todoabc = [todoabc for todoabc in reminders if todoabc['id'] == todoabc_id]
    if len(todoabc) == 0:
        abort(404)
    return jsonify({'todoabc': todoabc[0]})

if __name__ == '__main__':
    app.run(debug=True)


#!/bin/env python
# coding:utf-8

# fasttext

# XML parse
import os
import argparse
import zipfile
import traceback
import xml.etree.ElementTree as ET
import json
import copy
import re
import MeCab

class Runner:
    def __init__(self,args):
        self.args=args
        self.sentences=KYDString(args).run()

        self.docs=[]
        for s in self.sentences:
            self.docs.append("".join(s))

    def flatten(self, l):
        try:
            return ([item for sublist in l for item in sublist])
        except:
            import traceback
            traceback.print_exc()
            print(l)

    def run(self):
        # http://pyansblog.blogspot.jp/2011/12/mpi4py.html

        my_words = []
        for line in self.docs:
            my_words.append(self.split_sentence(self.remove_brackets(line)))

        with open(self.args.outfile, "w") as f:
            for words in my_words:
                f.write(" ".join(words)+"\n")

    def remove_brackets(self, line):
        s = line.replace("\n", "")
        s = re.sub("（.*?）", "", s)
        s = re.sub("【.*?】", "", s)
        s = re.sub("▽", "", s)
        s = re.sub("〇…", "", s)
        s = re.sub("／＊／…", "", s)
        return (s)

    def split_sentence(self, line):
        # 単語分割
        tagger = MeCab.Tagger("-Owakati")
        s = line.replace("\n", "")
        result = tagger.parse(s)
        words = result.replace("\n", "").split(" ")
        words.remove("")
        return (words)

    # TODO
    def split_sentence_with_joshi(self, line):
        return (None)

    # 単語から数値へ置き換える
    def convert(self, dict, words, mode="train"):
        id_words = []
        for word in words:
            if mode == "train":
                if word not in dict:
                    dict[word] = len(dict)
            else:
                if word not in dict:
                    word = "<unk>"  # when in test, unknown word convert to unk
            id_words.append(dict[word])

        return id_words

    def load_dict(self, dictfile):

        if os.path.exists(dictfile) is False:
            return (copy.deepcopy(self.dict))

        # 辞書の読み込み
        with open(dictfile, "r") as f:
            dict = json.load(f)
        return (dict)

class KYDString:
    def __init__(self,args):
        self.args=args

    def run(self):
        ret=[]
        if self.args.indata.endswith(".xml"):
            res=self.__extract_file(self.args.indata)
            if res is not None:
                ret= ret + [res]
        elif self.args.indata.endswith(".zip"):
            res = self.__extract_zipfile(self.args.indata)
            if res is not None:
                ret= ret + res
        else:
            for filename in os.listdir(self.args.indata):
                if filename.startswith("."):
                    continue
                print(filename)
                if filename.endswith(".zip"):
                    res= self.__extract_zipfile(self.args.indata+"/"+filename)
                    if res is not None:
                        ret= ret + res
                elif filename.endswith(".xml"):
                    res= self.__extract_file(self.args.indata+"/"+filename)
                    if res is not None:
                        ret= ret+ [res]
        return ret

    def __analyze_xml(self,root):
        sentences=[]
        tail=""
        for dhp in root.findall(".//InData/KdDHKgML/dhdiv/dhp"):
            str=[]
            for ite in dhp.iter():
                if ite.tag == "KdRuby":
                    tail=ite.tail
                elif ite.tag == "KdRt":
                    continue
                elif ite.tag == "KdRb":
                    str.append(ite.text)
                    if tail != "":
                        str.append(tail)
                    tail = ""
                elif ite.tag == "dhem":
                    str.append(ite.text)
                    if ite.tail is not None and ite.tail!="":
                        str.append(ite.tail)
                else:
                    if ite.text is not None and ite.text!="":
                        str.append(ite.text)
            if str is not None:
                #print(type(str))
                #print(len(str))
                s="".join(str)
                items=s.split("。")
                for l in items:
                    if len(l)==0:
                        continue
                    sentences.append(l+"。")
                #sentences.append("".join(str))
        return sentences   # [[abcde.],[fghi.]] this is from 1 file

    def __extract_file(self,filename):
        tree = ET.parse(filename)
        root = tree.getroot()
        return self.__analyze_xml(root)

    def __extract_zipfile(self,filename):
        try:
            sentences=[]
            with zipfile.ZipFile(filename, "r") as zf:
                for fname in zf.namelist():
                    if fname.index("KYD_") != 0:
                        continue
                    try:
                        utf16str = zf.read(fname)

                        root = ET.fromstring(utf16str)
                        sentences.append(self.__analyze_xml(root))
                    except:
                        traceback.print_exc()
            return sentences
        except:
            traceback.print_exc()

def main():
    # 引数の処理
    parser = argparse.ArgumentParser(description='')
    #parser.add_argument('--indata', default=os.environ["DATA"]+"/kiji/zip/KYD_20170410113157/KYD_ECM20170407X00001823_20170410113334.xml", type=str)
    parser.add_argument('--indata', default=os.environ["DATA"]+"/proofreading/zip_train", type=str)
    #parser.add_argument('--indata', default=os.environ["DATA"]+"/kiji/zip/KYD_20170410113157", type=str)
    #parser.add_argument('--indata', default=os.environ["DATA"]+"/kiji/zip", type=str)
    parser.add_argument('--outfile', default=os.environ["DATA"]+"/proofreading/kiji.txt", help='output file name of train data')

    args = parser.parse_args()
    Runner(args).run()

if __name__ == '__main__':
    main()


# Copyright 2015 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import datetime

from oslotest import base as test_base

from oslo_utils import fixture
from oslo_utils.fixture import keystoneidsentinel as keystids
from oslo_utils.fixture import uuidsentinel as uuids
from oslo_utils import timeutils
from oslo_utils import uuidutils

class TimeFixtureTest(test_base.BaseTestCase):

    def test_set_time_override_using_default(self):
        # When the fixture is used with its default constructor, the
        # override_time is set to the current timestamp.
        # Also, when the fixture is cleaned up, the override_time is reset.

        self.assertIsNone(timeutils.utcnow.override_time)
        with fixture.TimeFixture():
            self.assertIsNotNone(timeutils.utcnow.override_time)
        self.assertIsNone(timeutils.utcnow.override_time)

    def test_set_time_override(self):
        # When the fixture is used to set a time, utcnow returns that time.

        new_time = datetime.datetime(2015, 1, 2, 3, 4, 6, 7)
        self.useFixture(fixture.TimeFixture(new_time))
        self.assertEqual(new_time, timeutils.utcnow())
        # Call again to make sure it keeps returning the same time.
        self.assertEqual(new_time, timeutils.utcnow())

    def test_advance_time_delta(self):
        # advance_time_delta() advances the overridden time by some timedelta.

        new_time = datetime.datetime(2015, 1, 2, 3, 4, 6, 7)
        time_fixture = self.useFixture(fixture.TimeFixture(new_time))
        time_fixture.advance_time_delta(datetime.timedelta(seconds=1))
        expected_time = datetime.datetime(2015, 1, 2, 3, 4, 7, 7)
        self.assertEqual(expected_time, timeutils.utcnow())

    def test_advance_time_seconds(self):
        # advance_time_seconds() advances the overridden time by some number of
        # seconds.

        new_time = datetime.datetime(2015, 1, 2, 3, 4, 6, 7)
        time_fixture = self.useFixture(fixture.TimeFixture(new_time))
        time_fixture.advance_time_seconds(2)
        expected_time = datetime.datetime(2015, 1, 2, 3, 4, 8, 7)
        self.assertEqual(expected_time, timeutils.utcnow())

class UUIDSentinelsTest(test_base.BaseTestCase):

    def test_different_sentinel(self):
        uuid1 = uuids.foobar
        uuid2 = uuids.barfoo
        self.assertNotEqual(uuid1, uuid2)
        keystid1 = keystids.foobar
        keystid2 = keystids.barfoo
        self.assertNotEqual(keystid1, keystid2)

    def test_returns_uuid(self):
        self.assertTrue(uuidutils.is_uuid_like(uuids.foo))
        self.assertTrue(uuidutils.is_uuid_like(keystids.foo))

    def test_returns_string(self):
        self.assertIsInstance(uuids.foo, str)
        self.assertIsInstance(keystids.foo, str)

    def test_with_underline_prefix(self):
        ex = self.assertRaises(AttributeError, getattr, uuids, '_foo')
        self.assertIn("Sentinels must not start with _", str(ex))
        ex = self.assertRaises(AttributeError, getattr, keystids, '_foo')
        self.assertIn("Sentinels must not start with _", str(ex))


__author__ = "TheComet"

from Updateable import Updateable
import Config
import pygame

DTHROW_ADVANTAGE = 18

class PlatformBridgesHitstun(Updateable):
    def __init__(self, calc):
        self.calc = calc

        self.font = pygame.font.Font("sans.ttf", 18)
        self.text = self.font.render("Hitstun", True, (255, 255, 255))

    def draw(self, surface):
        # Draw horizontal line
        start = (Config.OFFSETX2, Config.OFFSETY2 + 100)
        end = (Config.OFFSETX2 + self.calc.uair3_end * Config.SCALEX2, Config.OFFSETY2 + 100)
        pygame.draw.line(surface, Config.TRAJECTORY_COLOR, start, end, 1)

        # "hitstun" text
        rect = self.text.get_rect()
        rect.left = Config.OFFSETX2
        rect.bottom = Config.OFFSETY2 + 100
        surface.blit(self.text, rect)

        self.draw_hit(surface, 0, DTHROW_ADVANTAGE, self.calc.hit1)
        self.draw_hit(surface, self.calc.hit1, self.calc.hit1_end, self.calc.hit2)
        self.draw_hit(surface, self.calc.hit2, self.calc.hit2_end, self.calc.hit3)

    def draw_hit(self, surface, hit_start, hit_end, next_hit_start):
        if next_hit_start < hit_end:
            color = (100, 255, 100)
        elif next_hit_start > hit_end:
            color = (255, 100, 100)
        else:
            color = (255, 255, 100)

        hit = hit_end if hit_end < next_hit_start else next_hit_start
        start = (hit_start * Config.SCALEX2 + Config.OFFSETX2, Config.OFFSETY2 + 100)
        end = (hit * Config.SCALEX2 + Config.OFFSETX2, Config.OFFSETY2 + 100)
        pygame.draw.line(surface, color, start, end, 1)

        start = (end[0], self.calc.get_height_at_frame(hit) * Config.SCALEY2 + Config.OFFSETY2 - 10)
        end = (end[0], end[1] + 10)
        pygame.draw.line(surface, Config.TRAJECTORY_COLOR, start, end, 1)
        text = self.font.render(f"{next_hit_start - hit_end}", True, color)
        rect = text.get_rect()
        rect.top = Config.OFFSETY2 + 100 + 2
        rect.left = hit * Config.SCALEX2 + Config.OFFSETX2 + 5
        surface.blit(text, rect)


# 1 - Reading and writing files

# Open a file for writing and create it if it doesn't exist
# + = Datei erstellen falls schon existiert
f = open("textfile.txt", "w+")

# write some lines of data to the file
for i in range(10):
    f.write("This is line " + str(i) + "\r\n")

# close the file when done
f.close()

# Open the file back up and read the contents
f = open("textfile.txt", "r")

if f.mode == "r":
    #wird ganze Datei ausgelesen
    contents = f.read()
    print(contents)
f.close()

f = open("textfile.txt", "r")
if f.mode == 'r':
    fl = f.readlines()
    for x in fl:
        print(x)
f.close()
# Open the file for appending text to the end
f = open("textfile.txt", "a")
for i in range(10):
    f.write("This is line " + str(i) + "\r\n")
f.close()

# 2 - Working with OS path utilities

#Klassen um mit Verzeichnisse und OS zu arbeiten
import os
from os import path
import datetime
from datetime import date, time, timedelta
import time

# Print the name of the OS
print(os.name)

# Check for item existence and type
print("Item exists: " + str(path.exists("textfile.txt")))
print ("Item is a file " + str(path.isfile("textfile.txt")))
print("Item is a directory " + str(path.isdir("textfile.txt")))
# Work with file paths
print ("Item path: " + str(path.realpath("textfile.txt")))
#trennt Pfad und Dateiname
print("Item path and name: " + str(path.split(path.realpath("textfile.txt"))))

# Get the modification time
#path.getmtime()  = gibt das Änderungsdatum zurück
t = time.ctime(path.getmtime("textfile.txt"))
print(t)
print(datetime.datetime.fromtimestamp(path.getmtime("textfile.txt")))
# Calculate how long ago the item was modified
td = datetime.datetime.now() - datetime.datetime.fromtimestamp(path.getmtime("textfile.txt"))
print("It has been " + str(td) + " since the file was modified")
print("Or, " + str(td.total_seconds()) + "seconds")

# 3 - Using file system shell methods
# = Dateien manipuliren mit Hilfe der Shell-Utilities
from os import path
import shutil
# make a duplicate of an existing file
if path.exists("textfile.txt"):
    # get the path to the file in the current directory
    src = path.realpath("textfile.txt")
    # let's make a backup copy by appending "bak" to the name
    dst = src + ".bak"
    #kopiert nur Inhalt der Datei, Dateieigenschaften nicht
    shutil.copy(src, dst)

    # copy over the permissions, modification times, and other info
    #kopiert Inhalt + Dateieigenschaften
    shutil.copystat(src, dst)

    # rename the original file
    os.rename("textfile.txt", "newfile.txt")
    os.rename("newfile.txt", "textfile.txt")

    # now put things into a ZIP archive
    from shutil import make_archive
    root_dir, tail = path.split(src)
    shutil.make_archive("arhive", "zip", root_dir)

# more fine-grained control over ZIP files
    from zipfile import ZipFile
    #Konstruktor aufruf für ZipFile() + Name des Objket is dann newzip
    with ZipFile("testzip.zip", "w") as newzip:
        newzip.write("textfile.txt")
        newzip.write("textfile.txt.bak")

ONLINE-KURSE/Become a Python Developer/3 - Learning Python/code-training/Code der Kapitel/4 - Working with Files/working_with_files-1.py


# -*- coding: utf-8 -*-
from odoo import api, fields, models

import logging
_logger = logging.getLogger(__name__)

class FireDrill(models.Model):
    _name = 'fire.drill'
    _description = 'Fire Drill'

    date = fields.Date(
        string='Fecha'
    )
    emergency_type = fields.Selection(
        selection=[
            ('general','General'),
            ('partial','Parcial')
        ],
        string='Tipo de emergencia'
    )
    drill = fields.Boolean(
        string='Simulacro'
    )
    description = fields.Text(
        string='Descripcion'
    )
    teams_must_act = fields.Text(
        string='Equipos deben actuar'
    )
    internal_communications = fields.Text(
        string='Comunicaciones internas'
    )
    description_causes = fields.Text(
        string='Descripcion de las causas'
    )
    description_consequences = fields.Text(
        string='Descripcion de las consecuencias'
    )
    general_description_intervention_performed = fields.Text(
        string='Descripcion general de la intervencion realizada'
    )
    fire_drill_decision_ids = fields.One2many('fire.drill.decision', 'fire_drill_id', string='Decisiones')


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('raps', '0019_auto_20150210_2113'),
    ]

    operations = [
        migrations.AlterField(
            model_name='myuser',
            name='playlist_tracks',
            field=models.ManyToManyField(related_name='playlist_tracks', null=True, to='raps.Track'),
            preserve_default=True,
        ),
        migrations.AlterField(
            model_name='myuser',
            name='removed_tracks',
            field=models.ManyToManyField(related_name='removed_tracks', null=True, to='raps.Track'),
            preserve_default=True,
        ),
    ]


import discord
from discord.ext import commands

class ErrorHandler:
    def __init__(self, bot):
        self.bot = bot
        self.ignored_errors = [
            discord.errors.NotFound,
            commands.errors.CommandNotFound,
            commands.errors.CheckFailure
        ]

    def is_ignored(self, error):
        # ierror: ignored error
        return any([isinstance(error, ie) for ie in self.ignored_errors])

    async def on_command_error(self, ctx, error):
        message = ctx.message.content

        if self.is_ignored(error):
            return

        if isinstance(error, commands.errors.CommandInvokeError):
            await self.bot.log(message, error=error.original)
        elif isinstance(error, discord.errors.Forbidden):
            await ctx.author.send(f"You must use `>{ctx.command.name}` in a "
                                  f"server that both of us are in!")
        else:
            await self.bot.log(message, error=error)

def setup(bot):
    bot.add_cog(ErrorHandler(bot))


import requests
import pandas as pd
import json
from datetime import datetime
from dateutil.relativedelta import relativedelta
from xml.dom import minidom
from gen import *

def get_top_n(df, top_n):
	if (len(df)>top_n):
		print('limit to top ' + str(top_n))
		first = df.sort_values(['platform','share'], ascending = False).groupby('domain').head(top_n)
		if top_n <= 200:
			last = df.sort_values(['platform','share'], ascending = True).groupby('domain').head(top_n)
			first['label'] = 'top '+str(top_n)
			last['label'] = 'bottom '+str(top_n)
			return(pd.concat([first,last]))
		return(first)
	else:
		return df

# max date range = 18 months
def get_date_range(months):
	end_date = datetime.today()+relativedelta(months=-1)
	end = end_date.strftime('%Y-%m')
	start_date = end_date+relativedelta(months=-months)
	start = start_date.strftime('%Y-%m')
	return start, end

def get_category(test_mode, url='https://api.similarweb.com/v1/TopSites/categories'):
	categories = []
	xml = minidom.parse('categories.xml')
	items = xml.getElementsByTagName('Value')
	for s in items:
		if s.childNodes:
			category = s.childNodes[0].nodeValue.strip()
			if category:
				categories.append(category)
	if test_mode:
		return(categories[:5])
	return(categories)

# def get_top_media(engine,schema,table,n,last_updated,test_mode):
# 	if test_mode:
# 		n = 5
# 	query = """select domain from {}.{}
# 	where category like 'News_and_Media%%' and last_updated = '{}'
# 	group by domain, rank order by rank limit {};""".format(schema,table,last_updated,n)
# 	df_top = pd.read_sql_query(query, engine)
# 	return (df_top)

# def get_top_domain(engine,schema,table,platform,last_updated,test_mode):
# 	query = """select d.domain,category, platform, rank, monthly_visits from {0}.{1} d
# 				join {0}.top_sites t
# 				on d.domain = t.domain
# 				where t.last_updated = '{2}'
# 				and platform = '{3}'
# 				group by d.domain, category, platform, rank, monthly_visits
# 				order by rank {4};""".format(schema,table,last_updated,platform,'limit 6' if test_mode else '')
# 	df_top = pd.read_sql_query(query, engine)
# 	return (df_top)

def traffic_bydomain(schema, engine, test_mode=False):
	result = pd.DataFrame()
	doms = pd.read_sql_query('''
		select date_range, domain, start_date, last_updated
		from {}.popular_pages
		group by date_range, domain, start_date, last_updated
		{};'''.format(schema, 'order by domain limit 20' if test_mode else ''), engine)
	for idx, row in doms.iterrows():
		#try:
		dom = row['domain']
		date_range = row['date_range']
		start_date = row['start_date']
		end_date = row['last_updated']
		print(dom, start_date, end_date)
		dom_visits = pd.read_sql_query('''select domain, platform, visits from {}.traffic_daily_split
											where domain = '{}' and date >= '{}' and date <= '{}';
											'''.format(schema, dom, start_date, end_date), engine)
		dom_visits['date_range'] = date_range
		dom_visits['start_date'] = start_date
		dom_visits['last_updated'] = end_date
		dom_visits=dom_visits.groupby(['domain', 'date_range', 'platform', 'start_date', 'last_updated'], as_index=False).sum()
		dom_traffic = dom_visits.merge(doms, on=['domain','date_range','start_date', 'last_updated'], how='inner')
		result = pd.concat([result, dom_traffic],sort=False)
	return(result)

def traffic_bytracker(schema, engine, table, traffic_dom):
	#get sw start and end day first
	# result = pd.DataFrame()
	# #sum historical visits by domain
	# hist_sql = '''select 'historical' as date_range, platform, domain, sum(visits) as visits
	# 		from {1}.traffic_daily_split
	# 		where date >= '{2}' and date <= '{0}'
	# 		group by platform, domain;'''.format(hist_end.strftime('%Y-%m-%d'), schema, hist_start.strftime('%Y-%m-%d'))
	# hist = pd.read_sql_query(hist_sql, engine)
	# #sum up last 28 days visits by domain
	# lm_sql = '''select 'last 28 days' as date_range, platform, domain, sum(visits) as visits
	# 		from {1}.traffic_daily_split
	# 		where date >= '{2}' and date <= '{0}'
	# 		group by platform, domain;'''.format(end_day.strftime('%Y-%m-%d'), schema, start_day.strftime('%Y-%m-%d'))
	# lm = pd.read_sql_query(lm_sql, engine)
	#domain visits across platforms
	traffic_dom['domain_visits'] = traffic_dom.groupby(['domain','date_range','start_date','last_updated'])['visits'].transform('sum')
	#match popular pages
	traffic_page = pd.read_sql_query(
		'''select date_range, platform, toppages.start_date, toppages.last_updated, toppages.domain, share, page
			from {0}.{1} toppages
			join (select *, regexp_replace( {0}.tracker.link, 'https?://(www\.)?') from {0}.tracker) tracker
			on toppages.page=tracker.regexp_replace;'''.format(schema, table), engine)
	traffic_page.drop_duplicates(keep='first',inplace=True)
	print('platform visits: ', len(traffic_page))
	traffic_df = traffic_page.merge(traffic_dom, on=['date_range', 'platform', 'domain', 'start_date', 'last_updated'], how='inner')
	traffic_df['platform_visits'] = traffic_df['visits'] * traffic_df['share']
	#total visits on a page across platforms
	traffic_df['page_visits'] = traffic_df.groupby(['page','date_range','start_date','last_updated','domain_visits'])['platform_visits'].transform('sum')
	traffic_df.drop(['platform_visits','share','visits','platform'],axis=1,inplace=True)
	traffic_df.drop_duplicates(keep='first',inplace=True)
	return(traffic_df)

def join_engagement(schema, engine, engagement_table):
	#use traffic last updated day for engagement metrics date. Historical is currently hard coded as there was no data for 06-30
	sql = '''
		select * from {0}.traffic_tracker t
		join (select *, regexp_replace({0}.engagement_{1}.link, 'https?://(www\.)?') from {0}.engagement_{1}) e
		on t.page=e.regexp_replace
		join {0}.tracker tr
		on e.link = tr.link
		where left(e.inserted_date, 10) = t.last_updated;'''.format(schema,engagement_table)
	#print(sql)
	df = pd.read_sql_query(sql, engine)
	df.drop(['id','page','month','day','year','regexp_replace'],axis=1,inplace=True)
	df = df.loc[:,~df.columns.duplicated()]
	return(df)

class SimilarWeb:

	#end_day = datetime.today()+relativedelta(days=-2) #test
	#start_day = end_day+relativedelta(days=-27)
	platforms = {
		'desktop':{
			'visits':['traffic-and-engagement/visits','visits'],
			'visitor':['unique-visitors/desktop_mau','unique_visitors']
			},
		'mobile':{
			'visits':['mobile-web/visits','visits'],
			'visitor':['unique-visitors/mobileweb_mau','unique_visitors']
			}
	}

	def __init__(self, key):
		#print(SimilarWeb.end_day, SimilarWeb.start_day)
		self.key = key

	#last 28 days + historical
	#must run on the same day of desktop_mobile_split
	def popular_pages(self, doms, top_n, start=None, end=None, dom_start=None):
		#use SW last updated dates
		output = pd.DataFrame()
		for idx, series in doms.iterrows():
			dom = series['domain']
			try:
				dom_start = datetime.strptime(series['min_date'],'%Y-%m-%d') #must convert string to datetime
			except KeyError:
				print('no start and end date defined; use generic')
			finally:
				print(dom)
				for x in SimilarWeb.platforms.keys():
					#last 28 days
					url = 'https://api.similarweb.com/v1/website/{}/{}/popular-pages?api_key={}&main_domain_only=false&format=json'.format(dom, 'website-content-mobile' if x == 'mobile' else 'website-content', self.key)
					req = requests.get(url = url)
					response = json.loads(req.text)
					try:
						df = pd.DataFrame(response['popular_pages'])
						df['date_range'] = 'last 28 days'
						end_day = response['meta']['last_updated']
						df['last_updated'] = end_day
						df['start_date'] = (datetime.strptime(end_day, '%Y-%m-%d')+relativedelta(days=-27)).strftime('%Y-%m-%d')
						#historical
						if start and end:
							if dom_start:
								start_month = start.strftime('%Y-%m') if start >= dom_start else dom_start.strftime('%Y-%m')
							else:
								start_month = start.strftime('%Y-%m')
							url_hist = 'https://api.similarweb.com/v1/website/{}/{}/popular-pages?api_key={}&main_domain_only=false&format=json&start_date={}&end_date={}'.format(dom, 'website-content-mobile' if x == 'mobile' else 'website-content', self.key, start_month, end.strftime('%Y-%m'))
							try:
								req_hist = requests.get(url = url_hist)
								response_hist = json.loads(req_hist.text)
								df_hist = pd.DataFrame(response_hist['popular_pages'])
								df_hist['date_range'] = 'historical'
								df_hist['last_updated'] = response_hist['meta']['last_updated']
								df_hist['start_date'] = start_month + '-01'
								df = pd.concat([df,df_hist], sort=False)
							except Exception as e:
								print ('no historical data', dom, url_hist, e.args)
						df['domain'] = dom
						df['platform'] = x
					#limit to top x; default 2000
						if top_n:
							df = get_top_n(df, top_n)
						output = pd.concat([output, df], sort=False)
					except Exception as e:
						print('no current data: ', dom, url, e.args)
						continue
		return(output)

	#last month
	def top_sites_category(self, categories, end):
	#last months only
		output = pd.DataFrame()
		for category in categories:
			print(category)
			for x in SimilarWeb.platforms.keys():
				url = 'https://api.similarweb.com/v1/website/${0}/topsites{1}/topsitesinternal?api_key={2}&start_date={3}&end_date={3}'.format(category, 'mobile' if x == 'mobile' else '', self.key, end.strftime('%Y-%m'))
				#print(url)
				req = requests.get(url = url)
				response = json.loads(req.text)
				#print(response)
				df = pd.DataFrame(response['top_sites'])
				df['platform'] = x
				df['category'] = category
				df['last_updated'] = response['meta']['last_updated']
				output = pd.concat([output, df], sort=False)
		return(output)

	#daily
	def desktop_mobile_split(self, doms, granularity, start=None, end=None):
		#no last updated day needed; use max(date) in table
		output = pd.DataFrame()
		for idx, series in doms.iterrows():
			dom = series['domain']
			print(dom)
			#last 28 days
			desktop_url = 'https://api.similarweb.com/v1/website/{}/traffic-and-engagement/visits?api_key={}&main_domain_only=false&format=json&granularity={}'.format(dom, self.key, granularity)
			mobile_url = 'https://api.similarweb.com/v2/website/{}/mobile-web/visits?api_key={}&format=json&main_domain_only=false&granularity={}'.format(dom, self.key, granularity)
			try:
				desktop_req = requests.get(url = desktop_url)
				desktop_response = json.loads(desktop_req.text)
				desktop_curr = pd.DataFrame(desktop_response['visits'])
				desktop_curr['platform'] = 'desktop'
				mobile_req = requests.get(url = mobile_url)
				mobile_response = json.loads(mobile_req.text)
				mobile_curr = pd.DataFrame(mobile_response['visits'])
				mobile_curr['platform'] = 'mobile'

			except Exception as e:
				print(e.args, dom)
				continue
			#historical
			if start and end:
				desktop_url_hist = 'https://api.similarweb.com/v1/website/{}/traffic-and-engagement/visits?api_key={}&start_date={}&end_date={}&main_domain_only=false&format=json&granularity={}'.format(dom, self.key, start.strftime('%Y-%m'), end.strftime('%Y-%m'), granularity)
				mobile_url_hist = 'https://api.similarweb.com/v2/website/{}/mobile-web/visits?api_key={}&start_date={}&end_date={}&format=json&main_domain_only=false&granularity={}'.format(dom, self.key, start.strftime('%Y-%m'), end.strftime('%Y-%m'), granularity)
				try:
					desktop_req_hist = requests.get(url = desktop_url_hist)
					desktop_response_hist = json.loads(desktop_req_hist.text)
					desktop_hist = pd.DataFrame(desktop_response_hist['visits'])
					desktop_hist['platform'] = 'desktop'
					mobile_req_hist = requests.get(url = mobile_url_hist)
					mobile_response_hist = json.loads(mobile_req_hist.text)
					mobile_hist = pd.DataFrame(mobile_response_hist['visits'])
					mobile_hist['platform'] = 'mobile'
					hist = pd.concat([desktop_hist,mobile_hist], sort=False)

					desktop = pd.concat([desktop_curr, desktop_hist], sort=False)
					desktop.drop_duplicates(keep='first', inplace=True)
					mobile = pd.concat([mobile_curr, mobile_hist], sort=False)
					mobile.drop_duplicates(keep='first', inplace=True)

					domain = pd.concat([desktop, mobile], sort=False)
					domain['domain'] = dom
					output = pd.concat([output,domain], sort=False)

				except Exception as e:
					print(e.args, dom)
					continue
			else:
				domain = pd.concat([desktop_curr,mobile_curr], sort=False)
			domain['domain'] = dom
			output = pd.concat([output,domain], sort=False)
		return(output)

	def monthly_visitor(self, doms, start, end):
		#last 18 months
		output = pd.DataFrame()
		for idx, series in doms.iterrows():
			dom = series['domain']
			print(dom)

			desktop_url = 'https://api.similarweb.com/v1/website/{}/{}?api_key={}&start_date={}&end_date={}&main_domain_only=false&format=json'.format(dom,SimilarWeb.platforms['desktop']['visitor'][0], self.key, start.strftime('%Y-%m'), end.strftime('%Y-%m'))
			mobile_url = 'https://api.similarweb.com/v1/website/{}/{}?api_key={}&start_date={}&end_date={}&format=json&main_domain_only=false'.format(dom, SimilarWeb.platforms['mobile']['visitor'][0], self.key, start.strftime('%Y-%m'), end.strftime('%Y-%m'))
			try:
				desktop_req = requests.get(url = desktop_url)
				desktop_response = json.loads(desktop_req.text)
				desktop = pd.DataFrame(desktop_response[SimilarWeb.platforms['desktop']['visitor'][1]])
				desktop['platform'] = 'desktop'
				mobile_req = requests.get(url = mobile_url)
				mobile_response = json.loads(mobile_req.text)
				mobile = pd.DataFrame(mobile_response[SimilarWeb.platforms['mobile']['visitor'][1]])
				mobile['platform'] = 'mobile'
				domain = pd.concat([desktop,mobile], sort=False)
				domain = domain.groupby('date', as_index=False).sum()
				domain['domain'] = dom
				domain['last_updated'] = desktop_response['meta']['last_updated']

				output = pd.concat([output,domain],sort=False)
			except Exception as e:
				print(e.args, dom)
				continue
		return(output)

	def get_subdomains(self, top_doms, end_day=datetime.today()+relativedelta(days=-2)):
		#last updated day needs revision
		output = pd.DataFrame()
		for index, row in top_doms.iterrows():
			dom = row['domain']

			# last 28 days
			desktop_url = 'https://api.similarweb.com/v1/website/{}/website-content-subdomains/subdomains?api_key={}'.format(row["domain"], self.key)
			mobile_url = 'https://api.similarweb.com/v1/website/{}/website-content-subdomains-mobile/subdomains?api_key={}'.format(row["domain"], self.key)

			try:
				desktop_req = requests.get(url = desktop_url)
				desktop_response = json.loads(desktop_req.text)
				desktop_curr = pd.DataFrame(desktop_response['subdomains'])
				desktop_curr['platform'] = 'desktop'
				desktop_curr['date_range'] = 'last 28 days'

				mobile_req = requests.get(url = mobile_url)
				mobile_response = json.loads(mobile_req.text)
				mobile_curr = pd.DataFrame(mobile_response['subdomains'])
				mobile_curr['platform'] = 'mobile'
				mobile_curr['date_range'] = 'last 28 days'

			except Exception as e:
				print(e.args, dom)
				continue

			# historical data
			if self.start and self.end:
				desktop_url_hist = 'https://api.similarweb.com/v1/website/{}/website-content-subdomains/subdomains?api_key={}&start_date={}&end_date={}'.format(row["domain"], self.key, self.start, self.end)
				mobile_url_hist = 'https://api.similarweb.com/v1/website/{}/website-content-subdomains-mobile/subdomains?api_key={}&start_date={}&end_date={}'.format(row["domain"], self.key, self.start, self.end)

				try:

					desktop_req_hist = requests.get(url = desktop_url_hist)
					desktop_response_hist = json.loads(desktop_req_hist.text)
					desktop_hist = pd.DataFrame(desktop_response_hist['subdomains'])
					desktop_hist['platform'] = 'desktop'
					desktop_hist['date_range'] = 'historical'

					mobile_req_hist = requests.get(url = mobile_url_hist)
					mobile_response_hist = json.loads(mobile_req_hist.text)
					mobile_hist = pd.DataFrame(mobile_response_hist['subdomains'])
					mobile_hist['platform'] = 'mobile'
					mobile_hist['date_range'] = 'historical'

					desktop = pd.concat([desktop_curr, desktop_hist], sort=False)
					#desktop.drop_duplicates(keep='first', inplace=True)

					mobile = pd.concat([mobile_curr, mobile_hist], sort=False)
					#mobile.drop_duplicates(keep='first', inplace=True)

					domain = pd.concat([desktop, mobile], sort=False)
					domain['domain'] = dom
					output = pd.concat([output, domain], sort=False)

				except Exception as e:
					print(e.args, dom)
					continue
			else:
				domain = pd.concat([desktop_curr, mobile_curr], sort=False)
				domain['domain'] = dom
				output = pd.concat([output,domain], sort=False)
			#output['last_updated'] = end_day.strftime('%Y-%m-%d')
		return(output)

	def capabilities(self):
		url = 'https://api.similarweb.com/capabilities?api_key={}'.format(self.key)
		try:
			date = datetime.now().strftime('%Y-%m-%d')
			r = requests.get(url = url)
			response =  json.loads(r.text)
			remain = response['remaining_hits']
			day = []
			for key in ['web_desktop_data','web_mobile_data']:
				start_date = response[key]['snapshot_interval']['start_date']
				end_date = response[key]['snapshot_interval']['end_date']
				day.append([date,remain,key[:-5],start_date,end_date])
			return(pd.DataFrame(day,columns=['date','remaining_hits','data','start_date','end_date']))
		except Exception as e:
			print(e.args)
			sys.exit()


import myokit
import myokit.pacing as pacing
import numpy as np
import matplotlib
import matplotlib.pyplot as pl
import myokit.lib.markov as markov
import pints
import argparse
import os
import sys

# Load project modules
sys.path.append(os.path.abspath(os.path.join('../', 'python')))
import cells
import data
import biomarkers

pl.rcParams['axes.axisbelow'] = True

# Check input arguments
parser = argparse.ArgumentParser(
    description='Plot model and experimental data')
parser.add_argument('--cell', type=int, default=2, metavar='N',
                    help='repeat number : 1, 2, 3, 4, 5, 6')
parser.add_argument('--model', type=str, default='wang', metavar='N',
                    help='which model to use')
parser.add_argument('--protocol', type=int, default=1, metavar='N',
                    help='which protocol is used to fit the data: 1 for staircase #1, 2 for sine wave')
parser.add_argument("--show", action='store_true',
                    help="whether to show figures instead of saving them",
                    default=False)
parser.add_argument('--params', type=int, default=1, metavar='N',
                    help='which params to use')
parser.add_argument('--figsize1', type=float, nargs='+', default=[3.5, 2], \
                    help='Figure size in x and y, e.g. --figsize1 2.5 3.5')
parser.add_argument('--figsize2', type=float, nargs='+', default=[2.5, 3.5], \
                    help='Figure size in x and y, e.g. --figsize2 2.5 3.5')
parser.add_argument("--grid", action='store_true',
                    help="whether to add grid to figures or not",
                    default=False)
args = parser.parse_args()

cell = args.cell

pr_steps = [
    (6000, 1000), #-140
    (26000, 1000), #-110
    (46000, 1000), #-80
    (66000, 1000), #-50
    (86000, 1000), #-20
    (106000, 1000), #10
    (126000, 1000), #40
]

pr_voltages = np.array([-140, -110, -80, -50, -20, 10, 40])

pr_steps_deact = [
    (6025, 7000), #-140
    (26025, 7000), #-110
    (46025, 7000), #-80
    (66025, 7000), #-50
]

pr_voltages_deact = np.array([-140, -110, -80, -50])

#
# Simple IKr test script
#

# Get model
p = myokit.load_protocol('../model-and-protocols/inactivation-ramp.mmt')

current = 'ikr.IKr'

ek = cells.ek(cell)

print('Reversal potential ' + str(ek) + ' mV')

if args.protocol == 1:
    protocol_str = 'staircase1'
else:
    protocol_str = 'sine-wave'

# Run simulation
dt = 0.1

fig1 = pl.figure(1, figsize=args.figsize1, dpi=200)
ax11 = fig1.add_subplot(1,2,1)
pl.title('Inactivation', fontsize=10)
ax11.set_xlabel('Voltage (mV)')
ax11.set_ylabel('Norm. current')
if args.grid:
    ax11.grid(True)
ax12 = fig1.add_subplot(1,2,2)
pl.title('Deactivation', fontsize=10)
ax12.set_xlabel('Voltage (mV)')
ax12.set_ylabel(r'$\tau$ (ms)')
if args.grid:
    ax12.grid(True)
ax12.set_yscale('log')

fig2 = pl.figure(2, figsize=args.figsize2, dpi=200)
ax21 = fig2.add_subplot(311)
[label.set_visible(False) for label in ax21.get_xticklabels()]
ax21.set_xlim([500, 1600])
if args.grid:
    ax21.grid(True)
ax21.set_ylabel( 'Voltage (mV)' )
ax22 = fig2.add_subplot(312)
[label.set_visible(False) for label in ax22.get_xticklabels()]
ax22.set_xlim([500, 1600])
if args.grid:
    ax22.grid(True)
ax22.set_ylabel( 'Predicted\ncurrent (nA)' )
ax23 = fig2.add_subplot(313, sharey=ax22)
ax23.set_xlim([500, 1600])
if args.grid:
    ax23.grid(True)
ax23.set_xlabel( 'Time (ms)' )
ax23.set_ylabel( 'Experimental\ncurrent (nA)' )

colors = ['orange', 'red', 'limegreen']
linestyles = ['solid', 'dashed', 'dashdot']

models = ['wang', 'wang-r6', 'wang-r8']
for n, model in enumerate(models):

    m = myokit.load_model('../model-and-protocols/' + model + '-ikr-markov.mmt')
    if model == 'mazhari':
        model_str = 'Mazhari'
        states = ['ikr.c2', 'ikr.c3', 'ikr.o', 'ikr.i']
    elif model == 'mazhari-reduced':
        model_str = 'Maz-red'
        states = ['ikr.c1', 'ikr.c3', 'ikr.o', 'ikr.i']
    elif model == 'wang':
        model_str = 'Wang'
        states = ['ikr.c2', 'ikr.c3', 'ikr.o', 'ikr.i']
    elif model == 'wang-r1':
        model_str = 'Wang-r1'
        states = ['ikr.c2', 'ikr.c3', 'ikr.o', 'ikr.i']
    elif model == 'wang-r2':
        model_str = 'Wang-r2'
        states = ['ikr.c3', 'ikr.o', 'ikr.i']
    elif model == 'wang-r3':
        model_str = 'Wang-r3'
        states = ['ikr.c3', 'ikr.o', 'ikr.i']
    elif model == 'wang-r4':
        model_str = 'Wang-r4'
        states = ['ikr.c3', 'ikr.o', 'ikr.i']
    elif model == 'wang-r5':
        model_str = 'Wang-r5'
        states = ['ikr.o', 'ikr.i']
    elif model == 'wang-r6':
        model_str = 'Wang-r6'
        states = ['ikr.o', 'ikr.i']
    elif model == 'wang-r7':
        model_str = 'Wang-r7'
        states = ['ikr.o', 'ikr.i']
    elif model == 'wang-r8':
        model_str = 'Wang-r8'
        states = ['ikr.o']
    else:
        pass
    n_params = int(m.get('misc.n_params').value())
    m = markov.convert_markov_models_to_full_ode_form(m)

    # Number of steps in pacing protocol
    n_steps = 7

    # Set steady state potential
    LJP = m.get('misc.LJP').value()
    ss_V = -80 - LJP

    x_found = np.loadtxt('../cmaesfits/' + model_str + '-model-fit-' + protocol_str + '-iid-noise-parameters-' + str(args.params) + '.txt', unpack=True)

    parameters = []
    for i in range(n_params):
        parameters.append('ikr.p'+str(i+1))

    d = ['engine.time', 'membrane.V', 'ikr.IKr']

    # Run simulation
    m.get('nernst.EK').set_rhs(ek)

    # print('Updating model to steady-state for ' + str(ss_V) + ' mV ')
    m.get('membrane.V').set_label('membrane_potential')

    mm = markov.LinearModel.from_component(m.get('ikr'))

    x = mm.steady_state(ss_V, x_found)
    for i in range(len(states)):
        m.get(states[i]).set_state_value(x[i])

    m.get('membrane.V').set_rhs('engine.pace - misc.LJP')

    s = myokit.Simulation(m, p)
    s.set_tolerance(1e-8, 1e-8)
    s.set_max_step_size(0.1)

    # Update model parameters
    for i in range(n_params):
        s.set_constant('ikr.p'+str(i+1), x_found[i])

    d = s.run(p.characteristic_time(), log_interval=dt, log=d)

    e = myokit.DataLog.load_csv('../data/SFU-data/Inactivation/inactivation-WT-cell-' + str(cell) + '.csv').npview()

    # Apply capacitance filtering for experiment and simulated data
    signals = [e.time(), e['current']]
    voltage = 'voltage' in e
    if voltage:
        signals.append(e['voltage'])
    signals = data.capacitance(p, dt, *signals)

    e = myokit.DataLog()
    e.set_time_key('time')
    e['time'] = signals[0]
    e['current'] = signals[1] / 1000
    if voltage:
        e['voltage'] = signals[2]

    signals2 = [d.time(), d['ikr.IKr'], d['membrane.V']]
    d = myokit.DataLog()
    d.set_time_key('time')
    d['time'] = signals2[0]
    d['current'] = signals2[1]
    d['voltage'] = signals2[2]

    # Create colormap for plotting
    cmap = matplotlib.cm.get_cmap('viridis')
    norm = matplotlib.colors.Normalize(0, n_steps)

    # Filtered simulated data
    d = d.npview()
    df = d.fold(2000)

    # Filtered experimental data
    e = e.npview()
    e = e.fold(2000)

    # Compute inactivation curve
    v1, g1, voltages1, peaks1 = biomarkers.steady_state_inactivation(pr_steps=pr_steps, pr_voltages=pr_voltages, log=d, erev=ek)
    v2, g2 = biomarkers.time_constant_of_deactivation(pr_steps_deact, pr_voltages_deact, log=d, erev=ek)

    g1n = g1 / np.max(g1)

    a = np.loadtxt('Biomarkers/inact_ss_exp_WT_cell' + str(cell) + '.txt', unpack=True)
    b = np.loadtxt('Biomarkers/inact_deact_exp_WT_cell' + str(cell) + '.txt', unpack=True)

    exp_ss_v = a[0]
    exp_ss_i = a[1]
    exp_ss_in = a[1] / np.max(a[1])

    exp_deact_v = b[0]
    exp_deact_i = b[1]

    ax11.plot(v1, g1n, linestyle=linestyles[n], color=colors[n], lw=1, label=model_str)
    if n == 0:
        ax11.scatter(exp_ss_v, exp_ss_in, s=36, facecolors='none', edgecolors='#1f77b4', marker='o', label='Expt.')
    ax12.plot(v2, g2, linestyle=linestyles[n], color=colors[n], lw=1)
    if n == 0:
        ax12.scatter(exp_deact_v, exp_deact_i, s=36, facecolors='none', edgecolors='#1f77b4', marker='o')

    for k in range(n_steps):
        if n == 0:
            ax21.plot(df.time(), df['voltage',k], color='#1f77b4', lw=1)
            ax23.plot(e.time(), e['current',k], color='#1f77b4', lw=1)
        ax22.plot(df.time(), df['current',k], color=colors[n], lw=1, linestyle=linestyles[n], label=model_str if k == 0 else '')

# ax11.legend(fontsize=8)
# ax22.legend(fontsize=8)
fig1.tight_layout()
fig2.tight_layout()

if args.show == True:
    pl.show()
else:
    filename1 = 'All_figures/Biomarkers-inactivation-models-compare-fit-' + protocol_str + '-iid-noise'
    filename2 = 'All_figures/Inactivation-models-compare-fit-' + protocol_str + '-iid-noise'
    fig1.savefig(filename1 + '.svg')
    fig2.savefig(filename2 + '.svg')


import scrapy
import json
import time
import os

from datetime import datetime
from datetime import timedelta

from scrapy.item import Field
from scrapy.item import Item
from scrapy.spiders import Spider
from scrapy.selector import Selector
from scrapy.loader import ItemLoader

from scrapy.spiders import CrawlSpider
from scrapy.http import Request
from scrapy.http import FormRequest
from scrapy.selector import Selector
from .filesystem import FileCsv

class YyyyyyyyyyyyyySpider(CrawlSpider):
    name = 'xxxxxxxxxxx'
    file_obj = None
    stations = []

    def start_requests(self):
        self.file_obj = FileCsv()
        urls = ['https://Yyyyyyyyyy.com']
        for url in urls:
            yield Request(
                url=url,
                callback=self.parse_origins,)

    def parse_origins(self, response):
        source = json.loads(response.body)
        stations = source['data']['stations']
        for origin in stations:
            url = 'https://Yyyyyyyyyy.com'
            yield Request(
                url=url,
                callback=self.parse_destinations,
                meta={'origin': origin})

    def parse_destinations(self, response):
        source = json.loads(response.body)
        origin = response.meta['origin']
        destinations = source['data']['stations']
        today = datetime.today()
        for destination in destinations:
            for n in range(1, int(os.environ.get('DAYS_SCRAPING', 10))):
                date = today + timedelta(days=n)
                date = date.date().strftime('%d-%m-%Y')
                url = 'https://Yyyyyyyyyy.com'
                yield Request(
                    url=url,
                    callback=self.parse_items,
                    headers={
                        'Accept': 'application/json, text/plain, */*',
                        'Accept-Language': 'es',
                        'Origin': 'https://Yyyyyyyyyy.com',
                        'Referer': 'https://Yyyyyyyyyy.com',
                        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36',
                        'x-api-key': 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX',
                        },
                    meta={
                        'origin': origin,
                        'destination': destination
                        })

    def parse_items(self, response):
        data = json.loads(response.body)
        categories = {
            'luxury': 'Lujo',
            'economy plus': 'Económico',
            'economy': 'Económico'
        }

        if data:
            if "error" in data:
                print('Error in parse_items')
            else:
                results = data['data']['outbound_routes']

            for result in results:
                today = datetime.now()
                origin = result['origin_name']
                destination = result['destination_name']
                base_price = "{:.2f}".format(result["original_price"])
                promo_price = "0.00"
                if base_price != "{:.2f}".format(result["price"]):
                    promo_price = "{:.2f}".format(result["price"])
                if result['local']:
                    route_type = 'Local'
                else:
                    route_type = 'De paso'

                item = {
                    'date_process': today.date().strftime('%d/%m/%Y'),
                    'time_process': today.time().strftime('%H:%M:%S'),
                    'id_sitio': 'www.xxxxxxxxxxx.com.mx',
                    'departure_date': result['departure_date'],
                    'departure_time': result["departure_time"],
                    'brand': result['cabin_type_ws'],
                    'origin': origin,
                    'destination': destination,
                    'base_price': base_price,
                    'promo_price': promo_price,
                    'service_type': categories[result['cabin_type']],
                    'route_type': route_type
                }
                self.file_obj.add_row(item)


import datetime
import os
import signal
import sys
import time

from multiprocessing import Process, Manager

POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
                                   os.pardir, os.pardir))
if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')):
    sys.path.insert(0, POSSIBLE_TOPDIR)

from stacktach import db, stacklog
from stacktach import version
from django.db import close_connection

import worker.worker as worker
from worker import config

processes = {}
log_listener = None
stacklog.set_default_logger_name('worker')

DEFAULT_PROC_TIMEOUT = 600
RUNNING = True

def _get_parent_logger():
    return stacklog.get_logger('worker', is_parent=True)

def create_proc_table(manager):
    for deployment in config.deployments():
        if deployment.get('enabled', True):
            name = deployment['name']
            db_deployment, new = db.get_or_create_deployment(name)
            for exchange in deployment.get('topics').keys():
                stats = manager.dict()
                proc_info = dict(process=None,
                                 pid=0,
                                 deployment=deployment,
                                 deploy_id=db_deployment.id,
                                 exchange=exchange,
                                 stats=stats)
                processes[(name, exchange)] = proc_info

def is_alive(proc_info):
    process = proc_info['process']
    if not proc_info['pid'] or process is None:
        return False
    return process.is_alive()

def needs_restart(proc_info):
    timeout = config.workers().get('process_timeout', DEFAULT_PROC_TIMEOUT)
    process = proc_info['process']
    stats = proc_info['stats']
    age = datetime.datetime.utcnow() - stats['timestamp']
    if age > datetime.timedelta(seconds=timeout):
        process.terminate()
        return True
    return False

def start_proc(proc_info):
    logger = _get_parent_logger()
    if is_alive(proc_info):
        if needs_restart(proc_info):
            logger.warning("Child process %s (%s %s) terminated due to "
                "heartbeat timeout. Restarting..." % (proc_info['pid'],
                proc_info['deployment']['name'], proc_info['exchange']))
        else:
            return False
    stats = proc_info['stats']
    stats['timestamp'] = datetime.datetime.utcnow()
    stats['total_processed'] = 0
    stats['processed'] = 0
    args = (proc_info['deployment'], proc_info['deploy_id'],
            proc_info['exchange'], stats)
    process = Process(target=worker.run, args=args)
    process.daemon = True
    process.start()
    proc_info['pid'] = process.pid
    proc_info['process'] = process
    logger.info("Started child process %s (%s %s)" % (proc_info['pid'],
        proc_info['deployment']['name'], proc_info['exchange']))
    return True

def check_or_start_all():
    for proc_name in sorted(processes.keys()):
        if RUNNING:
            start_proc(processes[proc_name])

def stop_all():
    procs = sorted(processes.keys())
    for pname in procs:
        process = processes[pname]['process']
        if process is not None:
            process.terminate()
    for pname in procs:
        process = processes[pname]['process']
        if process is not None:
            process.join()
        processes[pname]['process'] = None
        processes[pname]['pid'] = 0

def kill_time(signal, frame):
    global RUNNING
    RUNNING = False
    stop_all()

if __name__ == '__main__':
    logger = _get_parent_logger()
    log_listener = stacklog.LogListener(logger)
    log_listener.start()
    manager = Manager()

    create_proc_table(manager)

    # NOTE (apmelton)
    # Close the connection before spinning up the child process,
    # otherwise the child process will attempt to use the connection
    # the parent process opened up to get/create the deployment.
    close_connection()

    signal.signal(signal.SIGINT, kill_time)
    signal.signal(signal.SIGTERM, kill_time)

    ver = "(Version: %s )" % version.get_version()
    logger.info("Starting Workers %s..." % ver)
    while RUNNING:
        check_or_start_all()
        time.sleep(30)
    logger.info("Workers Shutting down...")

    #make sure.
    stop_all()

    log_listener.end()
    sys.exit(0)


import cv2

cap = cv2.VideoCapture('C:/Users/QIU/Desktop/Code/CircleDetect/data/video/moulde1.mp4')
frame_count = 217
while True:
    ret, frame = cap.read()
    if ret is not True:
        print("That's all!")
        exit()
    else:
        frame_name = 'C:/Users/QIU/Desktop/Code/CircleDetect/data/dataset' + '/' + str(frame_count) + '.jpg'
        cv2.imwrite(frame_name, frame)
        frame_count += 1
        if cv2.waitKey(3) & 0xFF == ord('q'):
            break
# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()


import datetime as dt

class SizeContainer(object):

    def __init__(self, n_days):
        self.n_days = n_days
        self.sizes = {}

    def update_sizes(self, sizes, date):
        """
        Note: this implementation prevents two entries for same date.
        """
        self.sizes[date] = sizes
        # Clean out old
        dates = list(set(self.sizes.keys()))
        dates.sort()
        if len(dates) > self.n_days:
            d = dates.pop(0)
            del self.sizes[d]

    def get_sizes(self):
        # Init output with all seccods
        output = {x: 0 for x in set(sum([x.keys() for x
                                         in self.sizes.values()], []))}
        for i in self.sizes.keys():
            for j in self.sizes[i].keys():
                output[j] += self.sizes[i][j]

        return output

    def to_json(self):
        # Convert datetime to strings
        out_sizes = {}
        for k in self.sizes.keys():
            new_k = k.strftime('%Y%m%d')
            out_sizes[new_k] = self.sizes[k]
        packet = {
            'n_days': self.n_days,
            'sizes': out_sizes,
        }
        return packet

    def from_json(self, packet):
        self.n_days = packet['n_days']
        # Convert strings to datetime
        for k, v in packet['sizes'].iteritems():
            new_k = dt.date(int(k[:4]), int(k[4:6]), int(k[6:]))
            self.sizes[new_k] = v

    # Implementation-specific functionality
    def kill_seccode(self, seccode):
        for s in self.sizes.values():
            if seccode in s:
                del s[seccode]
        return

    # Checking/debugging functionality
    def _check_seccode(self, seccode):
        seccode_size = 0
        flag = False
        for s in self.sizes.values():
            if seccode in s:
                flag = True
                seccode_size += s[seccode]
        if flag:
            return seccode_size
        return False


import json
import zmq
import os
import json
import pickle
import numpy as np
from pdb import set_trace as stop
from ast import literal_eval
from datetime import datetime as dt
from django.shortcuts import render
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_protect, csrf_exempt
from django.shortcuts import render, render_to_response
from celery import chain

from mansi.utils.urls import MSIxURL
from nsapi.tasks import msi_exec_task
from nsapi.models import IntradayData, CeleryResultData
from mansi.backend.tst.system.ns_handle_requests import REQUEST_IDS

# min number of transactions for a ticker
# within a day for analysis
MIN_NR_TRANSACTIONS = 300

# URL_MSI_PREFIX = "https://samarkanov.info/mansi/d/"
PORT_BACKEND = 5555
PICKLE_STORE = "/var/log/nsapi/stores"
API_MAP_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "api.conf")
HOME_DIR_MSI = os.path.join("/home", "samarka", "mansi")
TEST_CONFIG_FILE = os.path.join(HOME_DIR_MSI, "utils", "conint.json")

# UTILITIES
class NSxAPIxEmptyReplyFromBackend(Exception):
    pass

def log(text):
    with open("/var/log/nsapi/django.log", "a+") as ifile:
        ifile.write("\n%s: %s" %(dt.now(), text))

def store(data):
    with open(os.path.join(PICKLE_STORE, "tmp.data"), 'wb') as fp:
        pickle.dump(data, fp)

# POST-PROCESS REPRESENTATION OF SERIES
class Transaq(object):

    def __init__(self, *args, **kwargs):
        self.vol = kwargs["vol"]
        date_ = kwargs["date"]
        year, month, day = date_[:4], date_[4:6], date_[6:8]
        dt_ = kwargs["datetime"]
        hour, minute, sec = dt_[:2], dt_[2:], "00"
        self.datetime = str("%s-%s-%s %s:%s:%s" \
                                %(year, month, day, hour, minute, sec))
        # self.datetime = str(datetime_)
        self.high = kwargs["high"]
        self.low = kwargs["low"]
        self.close = kwargs["close"]
        self.open = kwargs["open"]

    def get_properties(self):
        items = []
        for item, value in self.__dict__.iteritems():
            if item[0] is not "_":
                items.append(item)
        return items

    @staticmethod
    def get_all(transacs, param):
        return [item.__dict__[param] for item in transacs]

    def __repr__(self):
        return "{vol: %s, datetime: %s, high: %s, low: %s, close: %s, open: %s}" \
                    %(self.vol, self.datetime, self.high, self.low, self.close, self.open)

# BACK-END HANDLER
class BackendHandler(object):

    def __init__(self, type_request):
        self.res = {"data":{}}
        self.type_request = type_request # string any of these two: (raw, ui)
        self._urlobj = {"errormsg": None, "url": None}

    def ask_backend(self, url=None):
        if url:
            self._urlobj["url"] = url
        if self._urlobj["url"]:
            context = zmq.Context()
            socket = context.socket(zmq.REQ)
            socket.connect("tcp://localhost:%i" %PORT_BACKEND)
            socket.send_string(self._urlobj["url"])
            log("about to send the url=%s to the backend" %self._urlobj["url"])
            # stop()
            try:
                rawdata = socket.recv().decode("utf-8")
                message = json.loads(rawdata)
            except ValueError:
                # print("URL: %s, data returned: %s" %(self._urlobj["url"], rawdata))
                # in case no data returned, then return nothing
                return dict()
            # if user asks raw json data, then return it immediately as
            # no need to post process it. Postprocesing is only needed
            # for POST request (so in case of UI request)
            if self.type_request == "raw":
                return message
            else:
                date_ = message["date"]
                transacs = [Transaq(date=date_, **item) for item in message["data"]]
                try:
                    items = transacs[0].get_properties()
                except IndexError:
                    raise NSxAPIxEmptyReplyFromBackend
                log("items: %s" %(items))
                for item in items:
                    # self OHLC and volume, datetime:
                    self.res["data"][item] = Transaq.get_all(transacs, item)
        else:
            self.res["data"]["errormsg"] = self._urlobj["errormsg"]
        return self.res

    def compose_url(self, request):
        if self.type_request == "raw":
            self._urlobj["url"] = request.urlencode()

        elif self.type_request == "ui":
            # request must be parsed and url string
            # to be composed out of the request
            ticker_ = request.get("ticker")
            datefrom_ = request.get("datefrom")
            dateto_ = request.get("dateto")
            step_ = request.get("step")
            if not ticker_ or not datefrom_ or not dateto_:
                self._urlobj["errormsg"] = "followind fields must be provided: "
                if not ticker_: self._urlobj["errormsg"] += "ticker, "
                if not datefrom_: self._urlobj["errormsg"] += "start date, "
                if not dateto_: self._urlobj["errormsg"] += "end date, "
                self._urlobj["errormsg"] = self._urlobj["errormsg"][:-2]
            else:
                # handle market type
                url_ =  "q=tradedata&"

                # handle date
                try:
                    dtfrom = dt.strptime(datefrom_, "%Y%m%d")
                    dtto = dt.strptime(dateto_, "%Y%m%d")
                except ValueError:
                    self._urlobj["errormsg"] = "wrongly formated date string (expected value: <Year><Month><Day>, example: 20170501)"
                    return self

                if dtfrom == dtto:
                    url_ += "date=%s&" %datefrom_
                else:
                    # check the input
                    if dtto < dtfrom:
                        self._urlobj["errormsg"] = "end day must be later than start day"
                        return self
                    url_ += "datefrom=%s&" %datefrom_
                    url_ += "dateto=%s&" %dateto_

                    # handle step (optional)
                    url_ += "step=%s" %step_

                # handle ticker
                url_ += "ticker=%s" %ticker_

                # return url
                self._urlobj["url"] = url_
        else:
            raise NotImplementedError
        return self

# VIEW FUNCTIONS
@csrf_exempt
def mansi(request):
    result = {"result": "here"}
    if request.method == "POST":
        reply = ""
        res = mansi_run(request.POST.get("code"))
        for item in res:
            if item: reply += "%s\n" %item
        result["result"] = reply
        return HttpResponse(json.dumps(result), content_type="application/json")

@csrf_exempt
def api_ui(request):
    # if request.method == "POST":
    #     bckend = BackendHandler(type_request="ui")
    #     reply = bckend.compose_url(request.POST).ask_backend()
    #     # store(reply)
    #     # reply = decorate_with_moving_average(reply)
    #     # post processing of data: calc of profit
    #     if reply and len(reply["data"]["vol"]) > MIN_NR_TRANSACTIONS:
    #         backtest = BackTester(strategy=Strategy1, data=reply)
    #         reply["data"]["profit_data"] = backtest.calc_profit()
    #         reply = backtest.decorate_with_moving_average(reply)
    #     return HttpResponse(json.dumps(reply), content_type="application/json")
    return render_to_response("nsindex.html", {})

def api_raw(request):
    bckend = BackendHandler(type_request="raw")
    reply = bckend.compose_url(request.GET).ask_backend()
    return HttpResponse(json.dumps(reply), content_type="application/json")

def api_raw_django(request):
    res = {}
    # log("api_raw_django: request.GET.date: %s" %request.GET.get("date"))
    # log("api_raw_django: request.GET.ticker: %s" %request.GET.get("ticker"))
    date = dt.strptime(request.GET.get("date"), "%Y%m%d").strftime("%Y-%m-%d")
    ticker = request.GET.get("ticker")
    series = IntradayData.objects.filter(ticker__abbrev=ticker, date=date)
    # log("api_raw_django: series: %s" %series)
    res["date"] = request.GET.get("date")
    res["data"] = []
    for item in series:
        ser = {}
        ser["vol"] = int(item.data.vol)
        ser["close"] = item.data.close
        ser["high"] = item.data.high
        ser["open"] = item.data.open
        ser["low"] = item.data.low
        ser["datetime"] = item.time.strftime("%H:%M:%S")
        res["data"].append(ser)
    return HttpResponse(json.dumps(res), content_type="application/json")

def make_command_str(idir, tstname):
    if ".py" in tstname:
        command = "python3 %s" %os.path.join(HOME_DIR_MSI, idir, tstname)
    else:
        command = os.path.join(HOME_DIR_MSI, idir, tstname)
    return command

@csrf_exempt
def testarena(request):
    res = {}

    if request.method == "POST":
        # user clicked button "retest"
        # clean cached celery data
        CeleryResultData.objects.all().delete()

    # Get test config
    with open(TEST_CONFIG_FILE, "r") as ifile:
        config = json.loads(ifile.read())

    for name, val in config.items():
        for tst in val:

            command = make_command_str(name, tst)
            res[command] = {}

            cached_obj = CeleryResultData.objects.filter(func=command)

            if cached_obj:
                log("DMSA cached_obj exists!")
                ob = cached_obj[0]
                # TODO: if results are too old then redo the execution
            else:
                # exec task and cache results (using DB backend)
                ob = CeleryResultData.objects.create(func=command, state="PENDING")
                msi_exec_task.apply_async((command,))

            res[command]["res"] = ob.res
            res[command]["date"] = str(ob.date)
            res[command]["time"] = str(ob.time)
            res[command]["tstdata"] = ob.data
            res[command]["state"] = ob.state

    return render_to_response("nstst.html", {"result": res})

@csrf_exempt
def listapis(request):
    res = {}
    res["APIs for charting"] = []
    res["General APIs"] = []
    res["Other APIs"] = []

    for idd, url in REQUEST_IDS.items():
        if "chartdata" in url["url"]:
            res["APIs for charting"].append(url["url"])
        elif "tradedata" in url["url"]:
            res["General APIs"].append(url["url"])
        else:
             res["Other APIs"].append(url["url"])
    return render_to_response("nsapis.html", {"result": res})
    # return HttpResponse(json.dumps(res), content_type="application/json")


from jax import numpy as jnp
from jax.nn import sigmoid

from .. import enum

class Sigmoid():

	def __init__(self):

		self.types = {enum.Sigmoid, enum.Activation, enum.NonParametric}

		self.scale = {
				'min': 0.0,
				'max': 1.0
			}

	def __call__(self, input_shape):

		self.input_shape = input_shape
		self.output_shape = (None, *input_shape[1:])

	def forward(self, X):

		X = sigmoid(X)

		X = X * (self.scale['max'] - self.scale['min']) + self.scale['min']

		return X

	def backward(self, Y):

		self.scale['min'], self.scale['max'] = jnp.min(Y), jnp.max(Y)

		Y = (Y - self.scale['min']) / (self.scale['max'] - self.scale['min'])

		Y = jnp.log(Y / (1 - Y))

		# -inf = log(0) ~= log(eps); inf = log(inf) ~= log(max)
		# log(float64.eps) = -36.04365, log(float32.max) = 88.72284
		# log(float32.eps) = -15.94239, log(float16.max) = 11.09
		# log(float16.eps) = -6.93, log(float16.max) = 11.09
		Y = jnp.nan_to_num(Y, neginf = jnp.log(jnp.finfo(jnp.float64).eps), posinf = jnp.log(jnp.finfo(jnp.float32).max))

		return Y


import requests
import json

headers = {
    "Referer": "https://m.douban.com/tv/american",
    "User-Agent": "Mozilla/5.0 (Linux; Android 8.0.0; Pixel 2 XL Build/OPD1.170816.004) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36"
}
request_url = "https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_american_hot/items?start={}&count=18&loc_id=108288"
# i=0
# url_list = []
# while i < 1084 :
#     url_list.append(request_url.format(i))
#     i += 18
# for url in url_list:
#     response = requests.get(url, headers=headers)
#     json_str = json.loads(response.content.decode("utf-8"))
#     print(json_str)
    # with open("american.json", "a+", encoding="utf-8") as f:
    #     ret = json.d
    #     f.write(ret)
# response = requests.get(request_url.format(0), headers=headers)
# json_str = json.loads(response.content.decode("utf-8"))
# # print(json_str)
# content = json_str["subject_collection_items"]
# print(content)

class DoubanSpider():
    def __init__(self):
        self.headers = {
            "Referer": "https://m.douban.com/tv/american",
            "User-Agent": "Mozilla/5.0 (Linux; Android 8.0.0; Pixel 2 XL Build/OPD1.170816.004) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36"
        }
        # self.url = "https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_american_hot/items?start={}&count=18&loc_id=108288"
        self.url_list = [
            {
                "url": "https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_american_hot/items?start={}&count=18&loc_id=108288",
                "country":"america"
            },
            {
                "url":"https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_english_hot/items?start=0&count=18&loc_id=108288",
                "country":"british"
            },
            {
                "url":"https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_domestic_hot/items?start=0&count=18&loc_id=108288",
                "country":"domestic"
            }
        ]

    def parse_url(self, url):
        response = requests.get(url, headers=self.headers)
        return response.content.decode("utf-8")

    def get_content_dict(self, json_str):
        content_dict = json.loads(json_str)
        return content_dict["subject_collection_items"], content_dict["total"]

    def save_content(self, content_list):
        with open("douban2.txt", "a", encoding="utf-8") as f:
            # f.write(country)
            for content in content_list:
                tv_info = json.dumps(content, ensure_ascii=False)
                # 将提取到的数据写入文件
                f.write(tv_info)
                f.write("\n")

    def run(self):
        # 组织url地址
        for url_dict in self.url_list:
            country_list = [url_dict["country"],]
            self.save_content(country_list)
            url_temp = url_dict["url"]
            i = 0
            count = 100
            while i < count:
                url = url_temp.format(i)
                i += 18
            # 获取响应内容json_str
                json_str = self.parse_url(url)
            # 提取数据
                content_list, count = self.get_content_dict(json_str)
                self.save_content(content_list)

if __name__ == '__main__':
    douban_spider = DoubanSpider()
    douban_spider.run()


from .download_manager import Payload
import pickle
import argparse
from datetime import datetime, timedelta

if __name__ == "__main__":

    parser = argparse.ArgumentParser(
        description="Preprocesses the dataset")
    parser.add_argument('--payload_path', default='payload.json',
                        help="Path to the .json file with all the information "
                             "about the payload")
    parser.add_argument('--output_file', default='dataset.pckl',
                        help="path where the created pickle file will be stored")
    parser.add_argument('--only_failed', action='store_true',
                        help="Load a saved payload and execute it's failed "
                             "components again")
    args = vars(parser.parse_args())

    before = datetime(2019, 5, 22)
    after = datetime(2014, 5, 22)

    if not args['only_failed']:
        """
        The default behavior:
         [1] read payload components from a file
         [2] execute the payload
         [3] pickle the finished payload
        """
        payload = Payload.load_from_json(args['payload_path'])

        # Download all data from the last 5 years
        before = datetime.today().date()
        after = before - timedelta(365 * 4 + 366)
        payload.execute(subreddits=['stocks', 'investing'],
                        before=before,
                        after=after)
        with open(args['output_file'], "wb") as f:
            pickle.dump(payload, f)
    else:
        """
        Only failed:
         [1] load a pickled payload
         [2] execute the payload with "only_failed" set to True
         [3] pickle the finished payload
        """
        with open(args['output_file'], "rb") as f:
            payload = pickle.load(f)

        payload.execute(subreddits=['stocks', 'investing'],
                        before=before,
                        after=after,
                        only_failed=True)

        with open(args['output_file'], "wb") as f:
            pickle.dump(payload, f)