import json
import logging
import random
import math

logging.basicConfig(level=logging.WARNING)

#logging.getLogger(__name__).setLevel(logging.INFO)
logger = logging.getLogger(__name__)
logger.setLevel(logging.ERROR)
#logger.setLevel(logging.WARNING)

def worth_goalie(p):
    worth = 0.0
    for pi in p:
        if 'worth' not in pi:
            continue
        if pi['pos'] in ['GK', 'Goalkeeper']:
            if pi['worth'] != None:
                worth = pi['worth']
            else:
                worth = 0.1
    return worth

def worth_defense(p):
    worth = 0.0
    for pi in p:
        if 'worth' not in pi:
            continue
        if pi['pos'] in ['CB', 'LB', 'RB']:
            if pi['worth'] != None:
                worth += pi['worth']
    return worth

def worth_midfield(p):
    worth = 0.0
    for pi in p:
        if 'worth' not in pi:
            continue
        if pi['pos'] in ['RM', 'LM', 'CM', 'AM', 'LW', 'RW', 'DM']:
            if pi['worth'] != None:
                worth += pi['worth']
    return worth

def worth_attack(p):
    worth = 0.0
    for pi in p:
        if 'worth' not in pi:
            continue
        if pi['pos'] in ['RF', 'LF', 'CF']:
            if pi['worth'] != None:
                worth += pi['worth']
    return worth

def worth_team(p):
    worth = 0
    for pi in p:
        if 'worth' not in pi:
            continue
        if pi['worth'] != None:
            worth += pi['worth']
    return worth

def predict(w, c1, c2, c3, c4, c5, c6):
    zone = 'midfield'
    possession = 'home'
    timer = 90

    score_home = 0
    score_away = 0

    norm1 = w['home']['g'] + w['home']['d'] + w['home']['m'] + w['home']['a']
    norm2 = w['away']['g'] + w['away']['d'] + w['away']['m'] + w['away']['a']
    norm = max(norm1, norm2)

    # home zone home goal
    hgk = w['home']['g']
    hgk /= norm
    # home zone home goal to zone home defense
    hgd = w['home']['d'] + c4 * w['home']['m']
    hgd /= norm
    # home zone home mid to zone away defense
    hmd = w['home']['a'] + w['home']['m'] + c5 * w['home']['d']
    hmd /= norm
    # home zone away defense to zone away goal
    hdg = w['home']['a'] + w['home']['m']
    hdg /= norm
    # home zone away goal
    hgs = w['home']['a'] + c6 * w['home']['m']
    hgs /= norm

    # away zone away goal
    agk = w['away']['g']
    agk /= norm
    # away zone away goal to zone away defense
    agd = w['away']['d'] + c4 * w['away']['m']
    agd /= norm
    # away zone away mid to zone home defense
    amd = w['away']['a'] + w['away']['m'] + c5 * w['away']['d']
    amd /= norm
    # away zone home defense to zone home goal
    adg = w['away']['a'] + w['away']['m']
    adg /= norm
    # away zone home goal
    ags = w['away']['a'] + c6 * w['away']['m']
    ags /= norm

    while timer > 0:
        timer -= 2
#        r1 = random.uniform(-0.5, 0.5)
#        r2 = random.uniform(-0.5, 0.5)
        r1 = random.gauss(0.6, 0.75)
        r2 = random.gauss(0.6, 0.75)

        if zone == 'midfield':
            if possession == 'home':
                m = r1 * hmd - r2 * amd
                logger.debug('h c1 {}'.format(m))
                if m > c1:
                    zone = 'defense away'
                else:
                    zone = 'midfield'
                    possession = 'away'
            elif possession == 'away':
                m = r1 * amd - r2 * hmd
                logger.debug('a c1 {}'.format(m))
                if m > c1:
                    zone = 'defense home'
                else:
                    zone = 'midfield'
                    possession = 'home'
        elif zone == 'defense away':
            if possession == 'home':
                m = r1 * hdg - r2 * adg
                logger.debug('h c2 {}'.format(m))
                if m > c2:
                    zone = 'goal away'
                else:
                    zone = 'midfield'
                    possession = 'away'
        elif zone == 'goal away':
            if possession == 'home':
                m = r1 * hgs - r2 * agk
                logger.debug('h c3 {}'.format(m))
                if m > c3:
                    score_home += 1
                zone = 'midfield'
                possession = 'away'
        elif zone == 'defense home':
            if possession == 'away':
                m = r1 * adg - r2 * hdg
                logger.debug('a c2 {}'.format(m))
                if m > c2:
                    zone = 'goal home'
                else:
                    zone = 'midfield'
                    possession = 'home'
        elif zone == 'goal home':
            if possession == 'away':
                m = r1 * ags - r2 * hgk
                logger.debug('a c3 {}'.format(m))
                if m > c3:
                    score_away += 1
                zone = 'midfield'
                possession = 'home'

    w['home']['pred score'] = score_home
    w['away']['pred score'] = score_away

    logger.info('Pred Score Home: {} Away: {}'.format(score_home, score_away))
    logger.info('Real Score Home: {} Away: {}'.format(w['home']['real score'], w['away']['real score']))

    return w

def train_model(data, c1, c2, c3, c4, c5, c6):
    quality = 0
    win = 0
    total = 0

    for ctr,d in enumerate(data):
        if ctr == 60:
            break

        w = {}
        for t in ['home', 'away']:
            w[t] = {}
            w[t]['g'] = worth_goalie(d[t]['start'])
            w[t]['d'] = worth_defense(d[t]['start'])
            w[t]['m'] = worth_midfield(d[t]['start'])
            w[t]['a']  = worth_attack(d[t]['start'])
            w[t]['name'] = d[t]['team']
            w[t]['real score'] = d[t]['score']
            w[t]['pred score'] = None

            s = 0
            if w[t]['g'] != None:
                s += w[t]['g']
            if w[t]['d'] != None:
                s += w[t]['d']
            if w[t]['m'] != None:
                s += w[t]['m']
            if w[t]['a'] != None:
                s += w[t]['a']
            logger.debug(d[t]['start'])
            if s < worth_team(d[t]['start']) - 0.001:
                logger.warning('Not all positions used for {}'.format(d[t]['team']))
                logger.info(s)
                logger.info(worth_team(d[t]['start']))
            else:
                logger.warning('{} {} {}'.format(d[t]['team'], 'Goalie', w[t]['g']))
                logger.warning('{} {} {}'.format(d[t]['team'], 'Defense', w[t]['d']))
                logger.warning('{} {} {}'.format(d[t]['team'], 'Midfield', w[t]['m']))
                logger.warning('{} {} {}'.format(d[t]['team'], 'Attack', w[t]['a']))

        sh = 0
        sa = 0
        rep = 100
        for i in range(rep):
            w = predict(w, c1, c2, c3, c4, c5, c6)
            sh += w['home']['pred score']
            sa += w['away']['pred score']

        w['home']['pred score'] = round(sh / rep, 2)
        w['away']['pred score'] = round(sa / rep, 2)
#        print('{} vs {}'.format(d['home']['team'], d['away']['team']))
#        print('Pred.', w['home']['pred score'], w['away']['pred score'])
#        print('Real.', w['home']['real score'], w['away']['real score'])

        quality += abs(w['home']['pred score'] - w['home']['real score']) + \
                abs(w['away']['pred score'] - w['away']['real score'])
        total += 1
        if (w['home']['real score'] > w['away']['real score']) \
                and (w['home']['pred score'] > w['away']['pred score']):
            win += 1
        if (w['home']['real score'] < w['away']['real score']) \
                and (w['home']['pred score'] < w['away']['pred score']):
            win += 1
        if (w['home']['real score'] == w['away']['real score']) \
                and (w['home']['pred score'] == w['away']['pred score']):
            win += 1

    return quality, win/total

def main():

    with open('qualifier_2.json', 'r') as infile:
        data = json.load(infile)

    c1 = 0.008
    c2 = 0.01
    c3 = 0.2
    c4 = 0.4
    c5 = 0.3
    c6 = 0.3

    last_fit = 1e99
    base_fit = 1e99

    while 1:
        c1 = random.uniform(0.0,0.1)
        c2 = random.uniform(0.0,0.1)
        c3 = random.uniform(0.0,0.4)
#        c4 = random.uniform(0.0,1.0)
#        c5 = random.uniform(0.0,1.0)
#        c6 = random.uniform(0.0,1.0)

        (last_fit, last_win) = train_model(data, c1, c2, c3, c4, c5, c6)

        if last_fit < base_fit:
            c1s = c1
            c2s = c2
            c3s = c3
            c4s = c4
            c5s = c5
            c6s = c6
            base_fit = last_fit
            print(last_fit, last_win, c1s, c2s, c3s, c4s, c5s, c6s)

if __name__ == '__main__':
    main()


# ---
# jupyter:
#   jupytext:
#     formats: py:percent
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.3'
#       jupytext_version: 1.14.5
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# %% [markdown]
# ## Locating and Copying Old Fluorescent Bead (FB) Measurements to a Single Folder for Testing

# %%
from pathlib import Path
import shutil
import numpy as np
import os

###############################################
# Move to project root to easily import modules
###############################################

try:  # avoid changes if already set
    print("Working from: ", PROJECT_ROOT)
except NameError:
    try:  # running from Spyder
        PROJECT_ROOT = Path(__file__).resolve()
    except NameError:  # running as Jupyter Notebook
        PROJECT_ROOT = Path(os.getcwd()).resolve().parent.parent
    os.chdir(PROJECT_ROOT)
    print("Working from: ", PROJECT_ROOT)

from utilities.display import Plotter
from utilities.file_utilities import (
    load_object,
    save_object,
)

# define main data directory
DATA_ROOT = Path("D:\OneDrive - post.bgu.ac.il\gSTED_sFCS_Data/")

# %% [markdown]
# List all paths for files in directories named "FB" in the main folder (`DATA_ROOT`)

# %%
OLD_DATA_ROOT = Path("D:\old\Google Drive\Dotan's old drive\Dotan TDC Measurments")

fb_paths = [path for path in OLD_DATA_ROOT.rglob("*.mat") if "fb" in str(path).lower()]

# TEST
print(f"Found {len(fb_paths)} files:\n")
print("\n".join([str(file_path) for file_path in fb_paths]))

# %% [markdown]
# Copy those files to a 'regular' date directory

# %%
# create the folder if necessary
destination_dir_path = DATA_ROOT / "24_05_2023" / "image"
destination_dir_path.mkdir(parents=True, exist_ok=True)

# # copy the files to new directory
for file_path in fb_paths:
    shutil.copy(file_path, destination_dir_path)

data_analysis/example_scripts/Getting all old fluorescent bead measurements from main folder.py


from flask import Flask
from flask import render_template
from flask import request
from flask import make_response
from flask import session

app = Flask(__name__)

app.secret_key = 'F12Zr47j\3yX R~X@H!jmM]Lwf/,?KT'

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/iniciarTest', methods=['POST'])
def iniciar():
    if request.method == 'POST':
        session['name'] = request.form['yourname']
        respuesta = make_response(render_template('pag1.html'))
        return respuesta

@app.route('/escogerCarrera', methods=['POST'])
def test1():
    carreras=[]
    pregunta1 = request.form['pregunta1']
    pregunta2 = request.form['pregunta2']
    pregunta3 = request.form['pregunta3']
    pregunta4 = request.form['pregunta4']
    pregunta5 = request.form['pregunta5']
    pregunta6 = request.form['pregunta6']
    pregunta7 = request.form['pregunta7']
    pregunta8 = request.form['pregunta8']
    pregunta9 = request.form['pregunta9']
    pregunta10 = request.form['pregunta10']
    pregunta11 = request.form['pregunta11']
    pregunta12 = request.form['pregunta12']
    pregunta13 = request.form['pregunta13']
    pregunta14 = request.form['pregunta14']
    pregunta15 = request.form['pregunta15']
    pregunta16 = request.form['pregunta16']
    pregunta17 = request.form['pregunta17']
    pregunta18 = request.form['pregunta18']
    pregunta19 = request.form['pregunta19']
    pregunta20 = request.form['pregunta20']
    pregunta21 = request.form['pregunta21']
    pregunta22 = request.form['pregunta22']
    pregunta23 = request.form['pregunta23']
    pregunta24 = request.form['pregunta24']
    pregunta25 = request.form['pregunta25']
    if pregunta1 == "si" and pregunta4 == "si" and pregunta18 == "si":
        carrera='Comunicacion y disenio'
        carreras.append(carrera)
    if pregunta2 == "si" and pregunta5 == "si" and pregunta14 == "si":
        carrera = 'Ciencias Administrativas'
        carreras.append(carrera)
    if pregunta7 == "si" and pregunta10 == "si" and pregunta15 == "si":
        carrera = 'Ingenieria y sistemas'
        carreras.append(carrera)
    if pregunta9 == "si" and pregunta11 == "si" and pregunta21 == "si":
        carrera = 'Ciencias de la salud'
        carreras.append(carrera)
    if pregunta6 == "si" and pregunta12 == "si" and pregunta16 == "si":
        carrera = 'Arquitectura'
        carreras.append(carrera)
    if pregunta13 == "si" and pregunta17 == "si" and pregunta20 == "si":
        carrera = 'Derecho y ciencias politicas'
        carreras.append(carrera)
    if pregunta3 == "si" and pregunta8 == "si" and pregunta19 == "si" and pregunta22 == 'si':
        carrera = 'Hotelería, Gastronomía y Turismo'
        carreras.append(carrera)
    if pregunta23 == "si" and pregunta24 == "si" and pregunta25 == "si":
        carrera = 'Logistica Maritma y portuaria'
        carreras.append(carrera)
    cantidad=len(carreras)

    respuesta = make_response(render_template('pag2.html', c=carreras, t=cantidad))
    return respuesta

@app.route('/SeleccionarFacultad', methods=['POST'])
def test2():
   facultad_final=request.form['pregunta']
   if facultad_final == 'Comunicacion y disenio':
       return render_template('pag3.html',facul=facultad_final)
   elif facultad_final == 'Ciencias Administrativas':
       return render_template()
   elif facultad_final == 'Ingenieria y sistemas':
       return render_template('sistemas.html', facul=facultad_final)
   elif facultad_final == 'Ciencias de la salud':
       return render_template()
   elif facultad_final == 'Arquitectura':
       return render_template()
   elif facultad_final == 'Derecho y ciencias politicas':
       return render_template()
   elif facultad_final == 'Hotelería, Gastronomía y Turismo':
       return render_template()
   elif facultad_final == 'Logistica Maritma y portuaria':
       return render_template()

@app.route('/ingenieria', methods=['POST'])
def ingenieria():
 if request.method == 'POST':
     carrera = []
     r1 = request.form['opcion1']
     r2 = request.form['opcion2']
     r3 = request.form['opcion3']
     r4 = request.form['opcion4']
     r5 = request.form['opcion5']
     r6 = request.form['opcion6']
     r7 = request.form['opcion7']
     if r1 == 'si':
         lic = 'Lic. en Ing. de Redes y Datos con énfasis en Sistemas Inalámbricos'
         carrera.append(lic)
         return render_template('resultados.html', l=lic)
     elif r2 == 'si':
         lic = 'Lic. en Ingeniería en Sistemas Computacionales '
         carrera.append(lic)
         return render_template('resultados.html', l=lic)
     elif r3 == 'si':
         lic = 'Lic. en Ingeniería en Electrónica y Comunicaciones '
         carrera.append(lic)
         return render_template('resultados.html', l=lic)
     elif r4 == 'si':
         lic = 'Lic. en Ingeniería en Industrial con énfasis en Gestión de Calidad'
         carrera.append(lic)
         return render_template('resultados.html', l=lic)
     elif r5 == 'si':
         lic = 'Lic. en Ingeniería en Industrial con énfasis en Gestión de Operaciones '
         carrera.append(lic)
         return render_template('resultados.html', l=lic)
     elif r6 == 'si':
         lic = 'Lic. en Ingeniería Industrial y de Sistemas '
         carrera.append(lic)
         return render_template('resultados.html', l=lic)
     elif r7 == 'si':
         lic = 'Lic. en Sistemas Comp. con énfasis en Desarrollo de Sistemas Avanzados de Redes y Software '
         carrera.append(lic)
         return render_template('resultados.html', l=lic)

if __name__ == "__main__":
    app.run(debug=True)


import logging as lg
from threading import Thread

from thrift.protocol import TBinaryProtocol
from thrift.server import TServer
from thrift.transport import TSocket
from thrift.transport import TTransport

from ...thrift.worker.filesystem import LocalFilesystem
from .handler import LocalFileSystemHandler

_logger = lg.getLogger(__name__)

class LocalFileSystemService(object):
    def __init__(self, address="0.0.0.0", port=8989):
        self.bind_address = address
        self.bind_port = port

    def _run(self):
        # Create Service handler
        handler = LocalFileSystemHandler()

        server = TServer.TThreadedServer(
            LocalFilesystem.Processor(handler),
            TSocket.TServerSocket(self.bind_address, self.bind_port),
            TTransport.TBufferedTransportFactory(),
            TBinaryProtocol.TBinaryProtocolFactory(),
            daemon=True,
        )

        server.serve()

    def start(self):

        _logger.info("Stating Tanit local file system service.")
        self.daemon = Thread(target=self._run, args=())
        self.daemon.setDaemon(True)
        self.daemon.start()
        _logger.info(
            "Tanit local file system service started, listening  at %s:%s",
            self.bind_address,
            self.bind_port,
        )


import os #para usar las funciones del sistema operativo
import os.path as path
import shutil #para eliminar

try:
    import cPickle as pickle
except ImportError:
    import pickle

import manejador_csv #para crear archivos csv para almancenar los datos

import datetime #para manejar fechas

import pandas #para sobreescribir csv

import time #medir el tiempo

import funciones

from avl_tree import AVLTree

from tabulate import tabulate

lista_indices = []
union_columnas = '-'
ruta_indices = "index/"

def procesar_comando(lista_comando):
    if lista_comando[0] == "CREATE":
        ruta = os.getcwd() + "/" #obteniendo la ruta actual
        comando_aux = lista_comando[1]
        comando_aux = comando_aux.split(maxsplit=1)
        if comando_aux[0] == "DATABASE":
            #Crea una carpeta para la base de datos que se crea
            nombre_db = comando_aux[1]
            if (funciones.exist_db(nombre_db) == True):
                print("Esa base de datos ya existe")
                return
            comando_system = "mkdir "+ ruta + nombre_db
            os.system(comando_system)
            #Agrega al archivo general de bases de datos el registro de la creacion
            #de una nueva base de datos
            today = datetime.date.today()
            input = [1,comando_aux[1],today,"null"]
            manejador_csv.escribir_csv(ruta,"database_info",input)
            #Crea el archivo que almacena informacion de las tablas
            header_tables_info = ["id", "name_tabla", "creacion", "eliminacion"]
            direccion = ruta + nombre_db +"/"
            manejador_csv.crear_archivo(direccion, "tables_info", header_tables_info)
            print("Base de datos creada")

        elif (comando_aux[0] == "TABLE"):
            tipos_aceptados = ['int','date','varchar']
            comando_aux2 = comando_aux[1]
            comando_aux2 = comando_aux2.split(maxsplit=3)
            if (comando_aux2[0]== "IN"):
                #Crea una carpeta para la tabla
                nombre_db = comando_aux2[1]
                if (funciones.exist_db(nombre_db) == False):
                    print("Esa base de datos no existe")
                    return
                nombre_tabla = comando_aux2[2]
                if(funciones.exist_tb(nombre_db,nombre_tabla) == True):
                    print("Error, esa base de datos ya existe")
                    return
                comando_system = "mkdir "+ ruta + nombre_db +"/"+nombre_tabla
                os.system(comando_system)
                #Actualizar archivos general de tablas
                today = datetime.date.today()
                input = [1,nombre_tabla,today,"null"]
                manejador_csv.escribir_csv(ruta + nombre_db +"/","tables_info",input)
                #Crea un directorio donde se almacena la informacion de los atributos
                header_table_info = ["id","name_data","type","size"]
                direccion = ruta +  nombre_db+"/"+nombre_tabla+ "/"
                manejador_csv.crear_archivo(direccion,nombre_tabla+"_info",header_table_info)
                #Crear la tabla propiamente dicha
                #Extraer las partes que tengan la metadata
                i = comando_aux2[3].find("(")
                j = comando_aux2[3].find(")")
                metadata = comando_aux2[3][i+1:j]
                metadata = metadata.replace(" ", "") #Quitando espacios
                metadata = metadata.split(",")
                header_metadata = []
                datos = []
                for elem in metadata:
                    aux = elem.split(":")
                    nombre_campo = aux[0]
                    it1 = aux[1].find("[")
                    if (it1 > 0):
                        tipo_campo = aux[1][0:it1]
                        it2 = aux[1].find("]")
                        tam_campo = aux[1][it1+1:it2]
                    else:
                        tipo_campo = aux[1]
                        tam_campo = "null"
                    #Comprobar si el tipo ingresado es valido
                    if ((tipo_campo not in tipos_aceptados ) == True):
                        print("Tipo de dato no aceptado", tipo_campo)
                        return
                    header_metadata.append(nombre_campo)
                    dato = [1,nombre_campo,tipo_campo,tam_campo]
                    datos.append(dato)

                manejador_csv.crear_archivo(direccion,nombre_tabla,header_metadata)
                manejador_csv.escribir_csv(direccion,nombre_tabla+"_info",datos)
                print("Tabla creada")
            else:
                print("Para crear tablas: CREATE TABLE IN nombre_db nombre_tabla")

        elif (comando_aux[0] == "INDEX"):
            comando_aux2 = comando_aux[1]
            #nombre ON tb1 db1 (col1,col2...)
            comando_aux2 = comando_aux2.split(maxsplit=4)
            nombre_indice = comando_aux2[0]
            if (comando_aux2[1] != "ON"):
                print("Error de sintaxis, debe ser CREATE INDEX name ON db1 tb1 (...)")
                return
            nombre_db = comando_aux2[2]
            if (funciones.exist_db(nombre_db) == False):
                print("Esa base de datos no existe")
                return
            nombre_tabla = comando_aux2[3]
            if (funciones.exist_tb(nombre_db, nombre_tabla) == False):
                print("Error, no existe esa tabla")
                return
            if (len(comando_aux2)<5):
                print("Faltan argumentos")
                return
            it1 = comando_aux2[4].find("(")
            it2 = comando_aux2[4].find(")")
            argumentos = comando_aux2[4][it1+1:it2]
            argumentos = argumentos.replace(" ","")
            argumentos = argumentos.split(",")
            for campo in argumentos:
                if (funciones.exist_campo(nombre_db, nombre_tabla, campo) == False):
                    print("Error, no existe ", campo)
                    return
            ruta = nombre_db+"/"+nombre_tabla+"/"+nombre_tabla+".csv"
            data = pandas.read_csv(ruta,usecols = argumentos)
            data_list = data.values.tolist()
            key = []
            for k in data_list:
                key.append(funciones.list_to_string(k,union_columnas))
            indexs = data.index.tolist()
            #print(argumentos)
            temporal = AVLTree(key,indexs,nombre_indice,nombre_db,nombre_tabla,argumentos)
            lista_indices.append(temporal)
            #Agregando a la tabla general de informacion de indices
            input = [nombre_indice]
            manejador_csv.escribir_csv(ruta_indices,"index_info",input)
            #Serializacion del indice
            with open(ruta_indices+nombre_indice+".idx", 'wb') as f:
                pickle.dump(temporal, f,2)
                print("Indice guardado")
            print("Indice creado")
            #Falta guardar en un archivo
        else:
            print("Error, comando no valido")
            return 0

    elif lista_comando[0] == "DROP":
        comando_aux = lista_comando[1]
        comando_aux = comando_aux.split(maxsplit=1)
        if comando_aux[0] == "DATABASE":
            nombre_db = comando_aux[1]
            if (funciones.exist_db(nombre_db) == False):
                print("Error, no existe base de datos")
                return
            shutil.rmtree(nombre_db)
            # Actulizar informacion general tabla
            df = pandas.read_csv("database_info.csv")
            df.loc[df["name_db"] == nombre_db, "eliminacion"] = datetime.date.today()
            df.to_csv("database_info.csv", index=False)
            #Print
            print("Base de datos eliminada")
        elif comando_aux[0] == "TABLE":
            comando_aux2 = comando_aux[1].split(maxsplit=3)
            if comando_aux2[0] == "IN":
                nombre_db = comando_aux2[1]
                if (funciones.exist_db(nombre_db) == False):
                    print("Error, no existe base de datos")
                    return
                nombre_tabla = comando_aux2[2]
                if(funciones.exist_tb(nombre_db,nombre_tabla) == False):
                    print("Error, no existe la table")
                    return
                ruta_delete = os.getcwd() + "/"+nombre_db+"/"+nombre_tabla
                shutil.rmtree(ruta_delete)
                #Actulizar informacion general tabla
                update_tabla_info_ruta = os.getcwd() + "/"+nombre_db+"/"+"tables_info.csv"
                df = pandas.read_csv(update_tabla_info_ruta)
                df.loc[df["name_tabla"] == nombre_tabla, "eliminacion"] = datetime.date.today()
                df.to_csv(update_tabla_info_ruta, index=False)
                #Print
                print("Tabla eliminada")
            else:
                print("La sintaxis correcta es DROP TABLE IN nombre_db nombre_tabla")

        elif comando_aux[0] == "INDEX":
            nombre_indice = comando_aux[1]
            if (funciones.exist_idx(nombre_indice) == False):
                print("Error, no existe el indice")
                return
            df = pandas.read_csv("index/index_info.csv")
            indices_to_delete = df.index[df["nombre_indice"] == nombre_indice].tolist()
            df.drop(df.index[indices_to_delete], inplace=True)
            df.to_csv("index/index_info.csv", index=False)
            indice_borrar = funciones.iterador_avl_by_indice(lista_indices,nombre_indice)
            lista_indices.pop(indice_borrar)
            os.remove(ruta_indices+nombre_indice+".idx")
            print("Indice borrado")
            return

        else:
            print("Error, comando no valido")
            return 0

    elif lista_comando[0] == "SELECT":
        it1 = lista_comando[1].find("(")
        it2 = lista_comando[1].find(")")
        datos_completos = True  # para saber si busca en todos los campos o solo en algunos
        header_in = []
        if it2 - it1 > 1:
            header_in = lista_comando[1][it1 + 1:it2]  # capturar las columnas a ingresar
            header_in = header_in.replace(" ","")
            header_in = header_in.split(",")
            datos_completos = False
        comando_aux1 = lista_comando[1][it2 + 1:]
        comando_aux1 = comando_aux1.split(maxsplit=4)
        if (comando_aux1[0] == "FROM"):
            nombre_db = comando_aux1[1]
            if (funciones.exist_db(nombre_db) == False):
                print("Error, no existe base de datos")
                return
            nombre_tabla = comando_aux1[2]
            if (funciones.exist_tb(nombre_db,nombre_tabla) == False):
                print("Error, no existe tabla")
                return
            if (datos_completos ==  False):
                for campo in header_in:
                    if (funciones.exist_campo(nombre_db,nombre_tabla,campo) == False):
                        print("Error, no existe ",campo)
                        return
            direccion_select = os.getcwd() + "/" + nombre_db + "/" + nombre_tabla + "/"
            header_tabla = manejador_csv.leer_csv(direccion_select, nombre_tabla, "header")  # Header de la tabla

            datos = pandas.read_csv(direccion_select + nombre_tabla + ".csv",index_col=False)
            if (len(comando_aux1) > 3): #hay WHERE
                if(comando_aux1[3]=="WHERE"):
                    #Solo valido 1 criterio
                    #datos_where = []
                    i_with = comando_aux1[4].find("WITH")
                    comando_indice = None
                    if (i_with > -1):
                        print("Buscando con indices")
                        comando_indice = comando_aux1[4][i_with:]
                        comando_aux1[4] = comando_aux1[4][0:i_with]
                    else:
                        print("Buscando sin indices")
                    comando_aux1[4] = comando_aux1[4].replace(" ", "")
                    if (comando_aux1[4].find("=") > 0):
                        datos_where = comando_aux1[4].split("=")
                        tipo_where = "="
                    elif (comando_aux1[4].find("<") > 0):
                        datos_where = comando_aux1[4].split("<")
                        tipo_where = "<"
                    elif (comando_aux1[4].find(">") > 0):
                        datos_where = comando_aux1[4].split(">")
                        tipo_where = ">"
                    else:
                        print("Error, comparacion no valida")
                        return
                    filtro_columna = datos_where[0]
                    filtro_criterio = datos_where[1]
                    if (funciones.exist_campo(nombre_db, nombre_tabla, filtro_columna) == False):
                        print("Error, no existe el campo")
                        return
                    if (filtro_columna=="id"):
                        filtro_criterio=int(filtro_criterio)
                    if (tipo_where == "="):
                        if(comando_indice!=None):
                            start_time1 = time.time()
                            comando_indice = comando_indice.split()
                            ix_indices = funciones.iterador_avl_by_indice(lista_indices,comando_indice[1])
                            indices_buscados = lista_indices[ix_indices].find_indexs(filtro_criterio)
                            if (indices_buscados == -1):
                                print("No se encontraron esos valores")
                                return
                            datos = datos.iloc[indices_buscados]
                            print("Con indices--- %s seconds ---" % (time.time() - start_time1))
                            #datos = datos.iloc[ix_indices]
                        else:
                            start_time1 = time.time()
                            filtro_criterio = funciones.toType(nombre_db,nombre_tabla,filtro_columna,filtro_criterio)
                            datos = datos[(datos[filtro_columna] == filtro_criterio)]
                            print("Sin indices--- %s seconds ---" % (time.time() - start_time1))
                    elif (tipo_where == "<"):
                        datos = datos[(datos[filtro_columna] < filtro_criterio)]
                    else:
                        datos = datos[(datos[filtro_columna] > filtro_criterio)]

                else:
                    print("Sintaxis invalida, es necesario un WHERE")
                    return
            if (datos_completos==False):
                #for columna in header_in:
                print(header_in)
                    #header_tabla.remove(columna)
                datos = datos.filter(items=header_in)

        else:
            print("Sintaxis invalida")
            return
        #datos.drop(datos.index, axis=1, inplace=True)
        #datos = datos.drop([''],axis=1)a
        #datos.set_index('id', inplace=True)

        #print (datos)

        print(tabulate(datos, headers=header_tabla, tablefmt='psql'))
        print("Datos mostrados")
        return

    elif lista_comando[0] == "INSERT":
        comando_aux = lista_comando[1]
        comando_aux = comando_aux.split(maxsplit=3)
        if (comando_aux[0] == "INTO"):
            nombre_db = comando_aux[1]
            #Comprobar si existe la base de datos
            if (funciones.exist_db(nombre_db) == False):
                print("Error, no existe base de datos")
                return
            nombre_tabla = comando_aux[2]
            #Comprobar si existe la tabla
            if (funciones.exist_tb(nombre_db,nombre_tabla) == False):
                print("Error, no existe base de datos")
                return
            direccion_insert = os.getcwd()+"/"+nombre_db+"/"+nombre_tabla+"/"
            #Capturar columnas a ingresar y datos
            it1 = comando_aux[3].find("(")
            it2 = comando_aux[3].find(")")
            datos_completos = True #para saber si insertera en todos los campos o solo en algunos
            header_in = []
            if it2-it1>1:
                header_in = comando_aux[3][it1+1:it2] #capturar las columnas a ingresar
                header_in = header_in.replace(" ","") #Quitando el espacio
                header_in = header_in.split(",")
                for campos in header_in:
                    if (funciones.exist_campo(nombre_db,nombre_tabla,campos) == False):
                        print ("No existe el campo,",campos)
                        return
                datos_completos = False

            comando_aux2 = comando_aux[3][it2+1:]
            comando_aux2 = comando_aux2.split(maxsplit=1)
            if comando_aux2[0]=="VALUES":
                it11 = comando_aux2[1].find("(")
                it22 = comando_aux2[1].find(")")
                data = comando_aux2[1][it11+1:it22] #capturo los datos a ingresar
                data = data.replace(" ","")
                data = data.split(",")
                for col,val in zip(header_in,data):
                    if (funciones.correct_type(nombre_db,nombre_tabla,col,val) == False):
                        print("Error, tipo de dato no valido",col,val)
                        return

            else:
                print("Error de sintaxis, falta VALUES")
                return

            #Comprobar con los campos de la tabla
            if datos_completos == False:
                header_tabla = manejador_csv.leer_csv(direccion_insert,nombre_tabla,"header") #Header de la tabla
                faltantes = [] #Guarda los indices de los campos que no se inserten
                newdata = []  #Reacomodar los datos en caso hayan sido ingresados en desorden
                for head in header_tabla:
                    if (head not in header_in) == True:
                        faltantes.append(header_tabla.index(head))
                for it in faltantes:
                    data.insert(it,"null")
            print(data)
            manejador_csv.escribir_csv(direccion_insert, nombre_tabla, data)

            #insertando en los indices
            indices_insertar = funciones.iterador_avl_by_tb(lista_indices,nombre_tabla)
            if (indices_insertar != None):
                datos = pandas.read_csv(direccion_insert + nombre_tabla + ".csv", index_col=False)
                ultimo_indice = datos.index[-1] #optener el ultimo indice para agregarlo
                datos = datos.iloc[ultimo_indice]
                for index in indices_insertar:
                    key = datos[lista_indices[index].colums].tolist()
                    key = funciones.list_to_string(key,union_columnas)
                    lista_indices[index].insert(key,ultimo_indice)
                print("Indices actualizados")

        elif comando_aux[0] == "BLOCK":
            nombre_db = comando_aux[1]
            nombre_tabla = comando_aux[2]
            file = comando_aux[3]
            data = manejador_csv.leer_csv(os.getcwd(),file)
            direccion_insert = os.getcwd()+"/"+nombre_db+"/"+nombre_tabla+"/"
            datos = pandas.read_csv(direccion_insert + nombre_tabla + ".csv", index_col=False)
            if (datos.empty == True):
                primer_indice = -1
            else:
                primer_indice = datos.index[-1]
            manejador_csv.escribir_csv(direccion_insert, nombre_tabla, data)
            print (nombre_db,nombre_tabla,file)
            #insertando indices
            indices_insertar = funciones.iterador_avl_by_tb(lista_indices, nombre_tabla)
            if (indices_insertar!=None):
                datos = pandas.read_csv(direccion_insert + nombre_tabla + ".csv", index_col=False)
                ultimo_indice = datos.index[-1]
                lista_index = list(range(primer_indice+1,ultimo_indice+1))
                datos = datos.iloc[lista_index]
                for ind in indices_insertar:
                    keys = []
                    aux = datos[lista_indices[ind].colums].values.tolist()
                    for dat in aux:
                        key = funciones.list_to_string(dat,union_columnas)
                        keys.append(key)
                    if (len(keys)!= len(lista_index)):
                        print("Lo hiciste mal",len(key),len(lista_index))
                        return
                    lista_indices[ind].insert_in_block(keys,lista_index)
                print("Indices actualizados")
        else:
            print("Error de sintexis, falta INTO")
            return
        print("Datos insertados")
        return

    elif lista_comando[0] == "UPDATE":
        comando_aux = lista_comando[1].split(maxsplit=3)
        print(comando_aux)
        nombre_db = comando_aux[0]
        nombre_tabla = comando_aux[1]
        direccion_update = os.getcwd() + "/" + nombre_db + "/" + nombre_tabla + "/"
        header_tabla = manejador_csv.leer_csv(direccion_update, nombre_tabla, "header")  # Header de la tabla
        tam_header = len(header_tabla)
        if (comando_aux[2]=="SET"):
            it1 = comando_aux[3].find("(")
            it2 = comando_aux[3].find(")")
            columnas_in = []
            valores_in = []
            if it2 - it1 > 1:
                header_in = comando_aux[3][it1 + 1:it2]  # capturar las columnas a ingresar
                header_in = header_in.replace(" ","")
                header_in = header_in.split(",")
                for criterio in header_in:
                    aux = criterio.split("=")
                    if (funciones.correct_type(nombre_db,nombre_tabla,aux[0],aux[1]) == False):
                        print("Error, tipo incorrecto")
                        return
                    columnas_in.append(aux[0])
                    valores_in.append(aux[1])
            else:
                print("Error al asignar datos")
                return
            comando_aux1 = comando_aux[3][it2 + 1:]
            comando_aux1 = comando_aux1.split(maxsplit=1)
            tipo_where = ""
            if (comando_aux1[0]=="WHERE"):
                # Solo valido 1 criterio
                comando_aux1[1] = comando_aux1[1].replace(" ", "")
                if (comando_aux1[1].find("=")>0):
                    datos_where = comando_aux1[1].split("=")
                    tipo_where = "="
                elif (comando_aux1[1].find("<") > 0):
                    datos_where = comando_aux1[1].split("<")
                    tipo_where = ">"
                elif (comando_aux1[1].find(">")>0):
                    datos_where = comando_aux1[1].split(">")
                    tipo_where = ">"
                else:
                    print("Error, comparacion no valida")
                    return

                filtro_columna = datos_where[0]
                filtro_criterio = datos_where[1]
                if (funciones.exist_campo(nombre_db, nombre_tabla, filtro_columna) == False):
                    print("Error, no existe el campo",filtro_columna)
                    return
                df = pandas.read_csv(direccion_update+nombre_tabla+".csv")

                for column,valor in zip(columnas_in,valores_in):
                    if (funciones.correct_type(nombre_db,nombre_tabla,column,valor) == False):
                        print("La columna tiene diferente tipo que el valor",column,valor)
                        return
                    if (filtro_columna=="id"):
                        filtro_criterio=int(filtro_criterio)
                    if (tipo_where == "="):
                        print ("here",df.loc[(df[filtro_columna] == filtro_criterio)])
                        df.loc[(df[filtro_columna] == filtro_criterio), column] = valor
                    elif (tipo_where == "<"):
                        df.loc[(df[filtro_columna] < filtro_criterio), column] = valor
                    else:
                        df.loc[(df[filtro_columna] > filtro_criterio), column] = valor

                #print(df)
                df.to_csv(direccion_update+nombre_tabla+".csv", index=False)

            else:
                print("Falta WHERE, error de sintaxis")
                return

        else:
            print("Falta SET")
            return
        print("Actualizado")
        return

    elif lista_comando[0] == "DELETE":
        comando_aux = lista_comando[1].split(maxsplit=4)
        if (comando_aux[0]== "FROM"):
            nombre_db = comando_aux[1]
            if (funciones.exist_db(nombre_db) == False):
                print("Error, no existe base de datos")
                return
            nombre_tabla = comando_aux[2]
            if (funciones.exist_tb(nombre_db,nombre_tabla) == False):
                print("Error, no existe la tabla o ha sido eliminada")
                return
            direccion_delete = os.getcwd() + "/" + nombre_db + "/" + nombre_tabla + "/"
            if (comando_aux[3]== "WHERE"):
                comando_aux[4] = comando_aux[4].replace(" ","")
                datos_where = []
                tipo_where = ""
                if (comando_aux[4].find("=") > 0):
                    datos_where = comando_aux[4].split("=")
                    tipo_where = "="
                elif (comando_aux[4].find("<") > 0):
                    datos_where = comando_aux[4].split("<")
                    tipo_where = ">"
                elif (comando_aux[4].find(">") > 0):
                    datos_where = comando_aux[4].split(">")
                    tipo_where = ">"
                else:
                    print("Error, comparacion no valida")
                    return
                filtro_columna = datos_where[0]
                if (funciones.exist_campo(nombre_db,nombre_tabla,filtro_columna) == False):
                    print("Error, no existe el campo")
                    return
                filtro_criterio = datos_where[1]
                df = pandas.read_csv(direccion_delete + nombre_tabla + ".csv")
                if(funciones.correct_type(nombre_db,nombre_tabla,filtro_columna,filtro_criterio) == False):
                    print("No son el mismo tipo",filtro_criterio,filtro_columna)
                    return
                else:
                    filtro_criterio = funciones.toType(nombre_db,nombre_tabla,filtro_columna,filtro_criterio)
#                if (filtro_columna == "id"):
 #                   filtro_criterio = int(filtro_criterio)
                if (tipo_where == "="):
                    indices_to_delete = df.index[df[filtro_columna] == filtro_criterio].tolist()
                elif (tipo_where == "<"):
                    indices_to_delete = df.index[df[filtro_columna] < filtro_criterio].tolist()
                else:
                    indices_to_delete = df.index[df[filtro_columna] > filtro_criterio].tolist()

                df.drop(df.index[indices_to_delete],inplace=True)
                #df.drop(df.query(filtro_columna+"=="+'"'+filtro_criterio+'"').sample(frac=0.90).index)
                df.to_csv(direccion_delete + nombre_tabla + ".csv", index=False)

                indices_eliminar = funciones.iterador_avl_by_tb(lista_indices,nombre_tabla)
                if (indices_eliminar != None):
                    for ind in indices_eliminar:
                        data = pandas.read_csv(direccion_delete + nombre_tabla + ".csv", usecols=lista_indices[ind].colums)
                        data_list = data.values.tolist()
                        key = []
                        for k in data_list:
                            key.append(funciones.list_to_string(k, union_columnas))
                        indexs = data.index.tolist()
                        lista_indices[ind].recontruir(key,indexs)
                    print("Indices actualizados")
            else:
                print("Error, falta WHERE")
                return
        else:
            print("Error, falta WHERE")
            return
        print("Borrado")
        return

    else:
        print("Error, comando no valido")
        return 0

#Al iniciar se crea una archivo donde se almacene informacion de todas las bases
#de datos que se creen
if (path.exists("database_info.csv")==False):
    header_database_info = ["id","name_db","creacion","eliminacion"]
    manejador_csv.crear_archivo(os.getcwd(),"database_info",header_database_info)
if (path.exists("index/index_info.csv")==False):
    header_database_info = ["nombre_indice"]
    manejador_csv.crear_archivo(os.getcwd()+"/index","index_info",header_database_info)
else:
    #Cargando indices ya creados
    df = pandas.read_csv(ruta_indices+"index_info.csv")
    if (df.empty == False):
        df = df["nombre_indice"].values.tolist()
        for ind in df:
            with open(ruta_indices+ind+".idx",'rb') as f:
                temp = pickle.load(f)
                lista_indices.append(temp)
        print("Indices cargados")
        print(len(lista_indices))
        for i in lista_indices:
            print(i.name)
#Preguntar comandos
comando = ""
while (comando != "exit"):
    comando = input("comando>>")
    start_time = time.time()
    lista_comando = comando.split(maxsplit=1)#comando.split(maxsplit=2)
   # print(lista_comando)
    procesar_comando(lista_comando)
    print("--- %s seconds ---" % (time.time() - start_time))


import numpy as np
import matplotlib.pyplot as plt
import glio
s = glio.GadgetSnapshot('snapshot_037')
s.load()

f = open('data_gas_xyz','w')
for i in range(0, len(s.pos[0][:])):
	f.write("%s %s %s\n" % (s.pos[0][i][0],s.pos[0][i][1],s.pos[0][i][2]))

f.close()

f2 = open('data_gas_xyz')
lines = f2.readlines()
f2.close()

gas_x = []
gas_y = []
gas_z = []

for line in lines:
	p = line.split()
	gas_x.append(float(p[0]))
	gas_y.append(float(p[1]))
	gas_z.append(float(p[2]))

gas_px = np.array(gas_x)
gas_py = np.array(gas_y)
gas_pz = np.array(gas_z)

#####################################
gas_px[:] = [x - -0.753 for x in gas_px]
gas_py[:] = [y - 0.796 for y in gas_py]
gas_pz[:] = [z - 1.000 for z in gas_pz]

plt.subplot(131)
plt.hist(gas_px, 1000)
plt.title('Gas x', fontsize=25)

plt.subplot(132)
plt.hist(gas_py, 1000)
plt.title('Gas y', fontsize=25)

plt.subplot(133)
plt.hist(gas_pz, 1000)
plt.title('Gas z', fontsize=25)

plt.show()

plt.plot(gas_px, gas_pz, '.', markersize=3,alpha=0.3)
plt.title('Gas x vs z', fontsize=30)
plt.show()


# -*- coding: utf-8 -*-

from os import cpu_count
import discord
from discord.ext import commands,tasks

import twitter

class twinotif(commands.Cog):

    def __init__(self, bot):
        self.bot = bot
        self.twi = twitter.Twitter(auth=twitter.OAuth(bot.T_Acs_Token, bot.T_Acs_SToken, bot.T_API_key, bot.T_API_SKey))
        self.target = "sina_status"
        self.last_id = self.gtwi_fu(self.target)[0]
        self.ch = self.bot.get_channel(1035026381853167676)
        self.mention = "<@&889157660837048350>"
        self.loop_task.start()

    def gtwi_fu(self,uname):
        ret = self.twi.statuses.user_timeline(screen_name=uname,count=1)
        return (int(ret[0]["id"]), ret[0])

    @tasks.loop(seconds=10)
    async def loop_task(self):
        ret = self.gtwi_fu(self.target)
        if not self.last_id == ret[0]:
            self.last_id = ret[0]
            tweet = ret[1]
            if tweet["text"].startswith("[Status]"):
                embed = discord.Embed(description=tweet["text"], color=int(
                    tweet["user"]["profile_background_color"], 16))
                embed.set_author(name=f'{tweet["user"]["name"]}(@{tweet["user"]["screen_name"]})',
                                    url=f'https://twitter.com/{tweet["user"]["screen_name"]}', icon_url=tweet["user"]["profile_image_url_https"])
                try:
                    embed.set_image(
                        url=tweet["entities"]["media"][0]["media_url_https"])
                except:
                    pass
                embed.add_field(name="Twitterで見る", value=f'https://twitter.com/{tweet["user"]["screen_name"]}/status/{tweet["id"]}')
                await self.ch.send(f"{self.mention}",embed=embed)

async def setup(bot):
    await bot.add_cog(twinotif(bot))


#!/usr/bin/env python3

import os
import sys
import platform
import datetime

import cv2 as cv
import numpy as np
import scipy.io as sio

from PyQt5 import QtWidgets

"""
Copyright (C) 2019 Denis Polygalov,
Laboratory for Circuit and Behavioral Physiology,
RIKEN Center for Brain Science, Saitama, Japan.

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, a copy is available at
http://www.fsf.org/
"""

"""
The SamTraker.py is a Semi-automatic mouse Tracker, - standalone
hybrid type (CLI + GUI) appliction for tracking single subject
in a recorded video stream. Output tracking data saved
as Matlab '*_tracking_data.mat' file.
"""

# TODO
# Add automatically generated timestamp to output mat file name?
# Add post-processing stage. Calculate distance traveled here?
# Add 'Subject ID' as user input? https://doc.qt.io/qt-5/qinputdialog.html
# Add pixel2cm constant input calculated based on first ROI selection?
# Allow user to terminate tracking WITH DATA SAVED during/after tracking failure handling?
# Save first accepted frame and first ROI drawn by user?

def process_video_file(s_fname_in, s_out_dir, s_out_fname_base, max_nframes=None):
    s_fname_out_mat = os.path.join(s_out_dir, s_out_fname_base + "_tracking_data.mat")
    if os.path.isfile(s_fname_out_mat):
        print("ERROR: output file already exist")
        sys.exit(-1)

    # the video source (movie) object
    oc_movie = CMuPaMovieCV((s_fname_in,))

    d_param = {}
    d_param['tracker_type'] = "KCF"
    d_param['aim_color'] = (60, 255, 255)
    d_param['aim_thickness'] = 2

    # extract video file parameters needed for detector
    d_param['frame_rate'] = oc_movie.df_info.loc[0, 'frame_rate']
    d_param['frames'] = oc_movie.df_info.loc[0, 'frames']

    i_frame_id = 0 # the frame counter
    t_ROI = None # current(!) ROI
    i_key_code = -1
    i_esc_key_code = 27
    oc_tracker = None # the video tracker instance
    na_tmp_img = None

    while(oc_movie.read_next_frame()):
        if not isinstance(na_tmp_img, np.ndarray):
            na_tmp_img = oc_movie.na_frame.copy()
        else:
            na_tmp_img[...] = oc_movie.na_frame[...]

        # yellow color frame-sized rectangle is used as indicator of first frame selection request
        cv.rectangle(na_tmp_img, (0,0), (na_tmp_img.shape[1], na_tmp_img.shape[0]), (0,255,255), 5)

        # Show current frame to user and ask for decision
        cv.imshow(s_out_fname_base, na_tmp_img)
        print("USER_INPUT_REQUEST: current frame #%i (a)ccept? (n)ext? (Esc)ape?" % i_frame_id)

        i_key_code = get_user_key_or_die(['a','n'])

        if i_key_code == ord('a'):
            break
        elif i_key_code == ord('n'):
            i_frame_id += 1
            continue
        else:
            raise ValueError("this should never happen")

    print("INFO: accepted frame #%i" % i_frame_id)

    b_is_frame_ready = True
    while b_is_frame_ready:
        if i_frame_id % 100 == 0: print("INFO: process frame #%i HINT: press 'r' to reset and assign new ROI" % (i_frame_id))

        if oc_tracker is None:
            d_param['initial_frame_number'] = i_frame_id
            print("USER_INPUT_REQUEST: select object to be tracked")
            d_param['initial_ROI'] = selectROI_or_die(s_out_fname_base, oc_movie.na_frame)
            print("INFO: selected ROI for the frame #%i: %s" % (i_frame_id, d_param['initial_ROI']))
            oc_tracker = CSingleSubjectTracker(
                oc_movie.na_frame.shape[0], # frame height
                oc_movie.na_frame.shape[1], # frame width
                oc_movie.na_frame.dtype,
                d_param
            )

        oc_tracker.process_frame(oc_movie.na_frame)
        cv.imshow(s_out_fname_base, oc_tracker.na_out)

        if not oc_tracker.is_tracking_succeed():
            print("WARNING: tracking failed at frame #%i" % i_frame_id)
            print("USER_INPUT_REQUEST: current frame #%i (a)ccept? (n)ext? (Esc)ape?" % i_frame_id)

            i_key_code = get_user_key_or_die(['a','n'])

            if i_key_code == ord('a'):
                print("USER_INPUT_REQUEST: accepted frame #%i. Select next ROI" % i_frame_id)
                t_ROI = selectROI_or_die(s_out_fname_base, oc_movie.na_frame)
                print("INFO: selected new ROI for the frame #%i: %s" % (i_frame_id, t_ROI))
                oc_tracker.restart_tracking(oc_movie.na_frame, t_ROI)
                b_is_frame_ready = oc_movie.read_next_frame() # WATCH OUT for number of calls
                i_frame_id += 1
                continue

            elif i_key_code == ord('n'):
                oc_tracker.skip_frame()
                b_is_frame_ready = oc_movie.read_next_frame() # WATCH OUT for number of calls
                i_frame_id += 1
                continue

            else: raise ValueError("this should never happen")

        # Wait for key press for 1ms
        i_key_code = cv.waitKey(1)
        i_key_code = i_key_code & 0xFF

        if i_key_code == ord('r'):
            print("USER_INPUT_REQUEST: ROI reset requested at frame #%i. Select new ROI" % i_frame_id)
            t_ROI = selectROI_or_die(s_out_fname_base, oc_movie.na_frame)
            print("INFO: selected new ROI for the frame #%i: %s" % (i_frame_id, t_ROI))
            oc_tracker.restart_tracking(oc_movie.na_frame, t_ROI)

        # Exit unconditionally if ESC key was pressed
        if i_key_code == i_esc_key_code: break

        b_is_frame_ready = oc_movie.read_next_frame() # WATCH OUT for number of calls
        i_frame_id += 1

        if max_nframes is not None and i_frame_id >= max_nframes: break
    #

    if i_key_code == i_esc_key_code:
        print("ERROR: interrupted by user. Tracking data is NOT saved!")
        sys.exit(-1)

    # seal output data into numpy arrays
    oc_tracker.finalize_tracking()
    # add extra parameters to be saved
    oc_tracker.d_STRACK['input_file_name'] = s_fname_in
    oc_tracker.d_STRACK['output_file_name'] = s_fname_out_mat
    oc_tracker.d_STRACK['hostname'] = platform.node()
    oc_tracker.d_STRACK['username'] = os.getlogin()
    oc_tracker.d_STRACK['datetime'] = str(datetime.datetime.now())
    # save output data into the mat file
    sio.savemat(s_fname_out_mat, oc_tracker.d_STRACK)
#

def get_user_key_or_die(l_key_choices):
    """
    Ask user to press one of keys listed in the l_key_choices
    or interrupt the whole script if 'q' or 'Esc' pressed.
    """
    while True:
        # Wait for key press indefinitely
        i_key_code = cv.waitKey(-1)
        i_key_code = i_key_code & 0xFF

        if i_key_code == 27 or i_key_code == ord('q'):
            print("ERROR: interrupted by user. No data saved.")
            sys.exit(-1)

        for _, ch in enumerate(l_key_choices):
            if i_key_code == ord(ch): return i_key_code

        print("ERROR: unknown key pressed")
        continue
    #
#

def selectROI_or_die(s_window_name, na_frame):
    """
    Ask user to draw the ROI on the frame.
    Providing acceptable ROI is obligatory for this script.
    """
    t_ROI = None
    while True:
        t_ROI = cv.selectROI(s_window_name, na_frame, False)
        if t_ROI[0] == 0 and all(i_elem == t_ROI[0] for i_elem in t_ROI):
            print("WARNING: selection was canceled or wrong")
        else:
            break
    if tuple is not type(t_ROI) or len(t_ROI) != 4:
        print("ERROR: something went seriously wrong. Exiting...")
        sys.exit(-1)
    return t_ROI
#

if __name__ == '__main__':
    s_base_dir, _ = os.path.split(os.getcwd())
    sys.path.append(s_base_dir)

    from mendouscopy.mupamovie import CMuPaMovieCV
    from mendouscopy.behavior import CSingleSubjectTracker

    i_max_nframes = None
    s_start_dir = os.path.expanduser('~')
    oc_app = QtWidgets.QApplication(sys.argv)
    s_file_filter = "Video Files(*.avi *.mp4 *.wmv *.mov *.mpg *.m4v *.mkv *.ts *.mjpg);; All Files(*.*)"

    while True:
        s_fname_in, _ = QtWidgets.QFileDialog.getOpenFileName(caption="Select input video file", directory=s_start_dir, filter=s_file_filter)

        if not os.path.isfile(s_fname_in):
            print("ERROR: input file is not accessible")
            sys.exit(-1)

        s_out_dir, s_fname = os.path.split(s_fname_in)
        if not os.path.isdir(s_out_dir):
            print("ERROR: output directory is not accessible")
            sys.exit(-1)

        s_out_fname_base, _ = os.path.splitext(s_fname)
        process_video_file(s_fname_in, s_out_dir, s_out_fname_base, max_nframes=i_max_nframes)
        cv.destroyAllWindows()

        oc_reply = QtWidgets.QMessageBox.question(
            None, 'Continue or Quit?',
            'Processing complete.\nDo you want to process another file?',
            QtWidgets.QMessageBox.Yes, QtWidgets.QMessageBox.No
        )

        if oc_reply == QtWidgets.QMessageBox.Yes:
            s_start_dir = s_out_dir
            continue
        else:
            break
    print()
#


import numpy as np
from numpy import random, exp, log, ones, zeros, sqrt, array, arange
from treelib import Node, Tree
import pandas as pd
from itertools import chain, combinations
#%%

def f1(x):
    x1,x2 = x
    if np.isin(x2,['a','b']):
        return 8 if x1 <= 5 else 2
    else:
        if x1 <= 3: return 1
        elif x1 <= 7: return 5
        else: return 8

def genData(n=800,seed=None):
    if seed is not None:
        random.seed(seed)
    x1val = np.arange(1,11)
    x2val = ['a','b','c','d']
    x1 = random.choice(x1val,size=n)
    x2 = random.choice(x2val,size=n)
    y = array(list(map(f1,tuple(zip(x1,x2))))) + 2*random.normal(size=n)
    return pd.DataFrame({'y':y,'x1':x1,'x2':x2})

def powerset(s):
    "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)"
    #s = set(iterable)
    S = chain.from_iterable(combinations(s, r) for r in range(1,len(s)))
    return list(S)

def get_xvar(df):
    n = df.iloc[:,1:].apply(lambda s:len(set(s)))
    return arange(1,df.shape[1])[n>1]

def get_st(df):
    y = df['y']
    n = len(y)
    s = (n-1)*y.var()
    t = n*a/(n+a)*(y.mean()-mu)**2
    return s+t

def get_pSplit(alpha,beta,depth):
    p = alpha/(1+depth)**beta
    return p/p.sum()

def grow(tree):
    global j
    #random.seed(98)
    leaf = list(filter(lambda node:len(node.xvar)>0,tree.leaves()))
    b = len(leaf)
    if b<1: raise ValueError('cannot grow anymore')
    pNode = random.choice(leaf)
    pid = pNode.identifier
    x = random.choice(pNode.xvar)
    dtype = DataTypes[x]
    df = pNode.data
    xVal = set(df.iloc[:,x])
    if dtype == 'O':
        S = powerset(xVal)
        s = S[random.choice(range(len(S)))]
        idx = df.iloc[:,x].isin(s)
    elif (dtype == 'i') | (dtype == 'f'):
        S = sorted(xVal)[:-1] #leave the last element out
        s = random.choice(S)
        idx = (df.iloc[:,x] <= s)
    nXS = len(pNode.xvar)*len(S)
    childDF = [df.loc[idx],df.loc[~idx]]
    tree1 = Tree(tree=tree,deep=True)
    node1 = tree1.create_node(str(j),j,parent=pid,data=childDF[0])
    node2 = tree1.create_node(str(j+1),j+1,parent=pid,data=childDF[1])
    st1 = [get_st(df) for df in childDF]
    tree1.stSum = tree.stSum-pNode.st+sum(st1)

    p1 = a*(a+len(df))/(a+array([len(df) for df in childDF])).prod()
    p2 = (tree.stSum/tree1.stSum)**npv
    rLike = sqrt(p1)*p2
    w2 = len(set([node.bpointer for node in tree1.leaves()]))
    rTransit = pPrune/pGrow*b*nXS/w2
    d = tree.level(pid)+1
    rStruct = alpha*(1-alpha/(1+d)**beta)**2/(d**beta-alpha)/nXS
    r = rLike*rTransit*rStruct
    if random.uniform() < r:
        j+=2
        pNode = tree1.get_node(pid)
        pNode.tag = str(pid)+'-'+str(x)+'-'+str(s)
#        pNode.data = None
        nodes = [node1,node2]
        for i in [0,1]:
            df = childDF[i]
            nodes[i].xvar = get_xvar(df)
            nodes[i].st = st1[i]
            nodes[i].data = df
        return tree1
    else:
        return tree

#%%
df = genData()
df['x3'] = 22
s = df['y'].var()
mu = df['y'].mean()
n,p = df.shape

v = 12; lamda = 4
vlamda = v*lamda
ig1 = v/2; ig2 = lamda*ig1
a = 1/3
lna = log(a)
npv = (n+v)/2

alpha = 0.95
beta = 1
DataTypes = df.dtypes.map(lambda x:x.kind)
#%%
tree = Tree()
root = tree.create_node('0',0,data=df)
root.st = get_st(root.data)
root.xvar = get_xvar(df)
tree.stSum = root.st+vlamda
j = 1
#%%
#actionSelect
pGrow = 1
pPrune = 0.5
action = 'grow'
leaf = tree.leaves()
#depth = array([len(i) for i in tree.paths_to_leaves()])-1

for t in range(100):
    tree = grow(tree)
    tree.show()


from __future__ import annotations

import asyncio
import inspect
from subprocess import PIPE
from typing import (
    Any,
    Callable,
    ClassVar,
    Coroutine,
    Dict,
    List,
    Optional,
    Tuple,
    Union,
)

import ray
import wrapt
from decorator import decorator

def check_enabled() -> bool:
    return RayRemote.ENABLED

class AbstractCallableDecorator:
    """
    An abstract class that should be subclassed to decorate different flavors of callables:

    - function
    - static method
    - class method
    - instance method
    - class with `__call__`

    Usage
    --------

    ```
    @AbstractCallableDecorator(...)
    def func(...):
        ...
    ```

    ```
    @AbstractCallableDecorator.transform(...)
    def func(...):
        ...
    ```

    ```
    @AbstractCallableDecorator.transform
    def func(...):
        ...
    ```

    """

    def _run(self, fn: Callable, *args: Any, **kwargs: Any):
        """Call the decorated function with its arguments."""
        return fn(*args, **kwargs)

    @classmethod
    def transform(cls, wrapped: Optional[Callable] = None, *args: Any, **kwargs: Any):
        """
        Create a wrapper by first create a decorator with provided arguments and then apply decorator to wrapped.
        """
        return cls(*args, **kwargs)(wrapped)

    @wrapt.decorator
    def __call__(self, wrapped, instance, args, kwargs):
        if not callable(wrapped):
            raise TypeError(f"{self.__class__.__name__} should decorate a callable")

        if instance is None:
            if inspect.isclass(wrapped):
                raise TypeError(f"{self.__class__.__name__} cannot decorate a class")
            elif inspect.isfunction(wrapped):
                return self._run(wrapped, *args, **kwargs)
            else:
                try:
                    # is callable
                    cls = getattr(wrapped, "__class__")
                    instance = wrapped
                    wrapped = cls.__call__
                except AttributeError:
                    return self._run(wrapped, *args, **kwargs)

        @decorator
        def wrapper(fn: Callable, *args: Any, **kwargs: Any):
            return fn(*args, **kwargs)

        return self._run(wrapper(wrapped), instance, *args, **kwargs)

class RayRemote(AbstractCallableDecorator):
    """
    Combining [`ray.remote`](https://docs.ray.io/en/master/package-ref.html#ray-remote) and invoke of remote function.

    Usage
    --------

    ```
    @RayRemote(...)
    def func(...):
        ...
    ```
    """

    ENABLED: ClassVar[bool] = True

    def __init__(
        self,
        wrapped: Optional[Callable] = None,
        *,
        num_returns: Optional[int] = None,
        num_cpus: Optional[float] = None,
        num_gpus: Optional[int] = None,
        resources: Optional[Dict[str, float]] = None,
        accelerator_type: Optional[Any] = None,
        max_calls: Optional[int] = None,
        max_restarts: Optional[int] = None,
        max_task_retries: Optional[int] = None,
        max_retries: Optional[int] = None,
        runtime_env: Optional[Dict[str, Any]] = None,
        retry_exceptions: Optional[bool] = None,
    ):
        """
        Args:
            num_returns (int): This is only for *remote functions*. It specifies
                the number of object refs returned by
                the remote function invocation.
            num_cpus (float): The quantity of CPU cores to reserve
                for this task or for the lifetime of the actor.
            num_gpus (int): The quantity of GPUs to reserve
                for this task or for the lifetime of the actor.
            resources (Dict[str, float]): The quantity of various custom resources
                to reserve for this task or for the lifetime of the actor.
                This is a dictionary mapping strings (resource names) to floats.
            accelerator_type: If specified, requires that the task or actor run
                on a node with the specified type of accelerator.
                See `ray.accelerators` for accelerator types.
            max_calls (int): Only for *remote functions*. This specifies the
                maximum number of times that a given worker can execute
                the given remote function before it must exit
                (this can be used to address memory leaks in third-party
                libraries or to reclaim resources that cannot easily be
                released, e.g., GPU memory that was acquired by TensorFlow).
                By default this is infinite.
            max_restarts (int): Only for *actors*. This specifies the maximum
                number of times that the actor should be restarted when it dies
                unexpectedly. The minimum valid value is 0 (default),
                which indicates that the actor doesn't need to be restarted.
                A value of -1 indicates that an actor should be restarted
                indefinitely.
            max_task_retries (int): Only for *actors*. How many times to
                retry an actor task if the task fails due to a system error,
                e.g., the actor has died. If set to -1, the system will
                retry the failed task until the task succeeds, or the actor
                has reached its max_restarts limit. If set to `n > 0`, the
                system will retry the failed task up to n times, after which the
                task will throw a `RayActorError` exception upon :obj:`ray.get`.
                Note that Python exceptions are not considered system errors
                and will not trigger retries.
            max_retries (int): Only for *remote functions*. This specifies
                the maximum number of times that the remote function
                should be rerun when the worker process executing it
                crashes unexpectedly. The minimum valid value is 0,
                the default is 4 (default), and a value of -1 indicates
                infinite retries.
            runtime_env (Dict[str, Any]): Specifies the runtime environment for
                this actor or task and its children. See
                :ref:`runtime-environments` for detailed documentation. This API is
                in beta and may change before becoming stable.
            retry_exceptions (bool): Only for *remote functions*. This specifies
                whether application-level errors should be retried
                up to max_retries times.
        """
        kwargs = {
            name: value
            for name, value in locals().items()
            if name != "self" and name != "kwargs" and name != "wrapped" and value is not None
        }

        self.__ray_wrapper = ray.remote(**kwargs) if kwargs else ray.remote

        if wrapped:
            raise ValueError(f"{self.__class__.__name__} should be initialized with arguments")

    def _run(self, fn: Callable, *args: Any, **kwargs: Any) -> ray.ObjectRef:
        try:
            ray_callable = self.__ray_wrapper(fn)
        except ValueError as error:
            if "async def" in str(error):

                @decorator
                def wrapper(fn: Callable, *args: Any, **kwargs: Any):
                    return fn(*args, **kwargs)

                ray_callable = self.__ray_wrapper(wrapper(fn))
            else:
                raise

        return ray_callable.remote(*args, **kwargs)

    def run_remote(self, fn: Callable, *args: Any, **kwargs: Any) -> ray.ObjectRef:
        """
        Run a callable in ray.

        This is short for `ray.remote(fn).remote(*args, **kwargs)`.
        """
        return self._run(fn, *args, **kwargs)

    @wrapt.decorator(enabled=check_enabled)
    def __call__(self, wrapped, instance, args, kwargs):
        return super(self.__class__, self).__call__(wrapped)(*args, **kwargs)

class AwaitCoroutine(AbstractCallableDecorator):
    """
    Convert a coroutine to a blocking synchronous operation.
    """

    def _run(self, fn, *args, **kwargs):
        return asyncio.get_event_loop().run_until_complete(fn(*args, **kwargs))

class RayAsyncActor:
    def __init__(self, coroutine: Coroutine):
        super().__init__()
        self.coroutine = coroutine

    @classmethod
    def transform(cls, wrapped: Optional[Coroutine] = None, **ray_options: Any):
        if ray_options:
            actor_constructor = ray.remote(cls).remote(**ray_options)
        else:
            actor_constructor = ray.remote(cls).remote

        if wrapped:
            return actor_constructor(wrapped)
        else:
            return actor_constructor

    async def __call__(self, *args: Any, **kwargs: Any):
        return await self.coroutine(*args, **kwargs)

class ShellActor:
    def __init__(self, *args: List[str], stdin=PIPE, stdout=PIPE, stderr=PIPE, **kwargs: Any):
        self.args = args
        self.stdin = stdin
        self.stdout = stdout
        self.stderr = stderr
        self.shell_kwargs = kwargs

    async def _start_async(self) -> None:
        self.process: asyncio.Process = await asyncio.create_subprocess_exec(
            *self.args,
            stdin=self.stdin,
            stdout=self.stdout,
            stderr=self.stderr,
            **self.shell_kwargs,
        )

    async def _communicate(
        self, _input: Optional[bytes] = None, encoding: Optional[str] = None
    ) -> Tuple[Union[str, bytes], Union[str, bytes]]:
        stdout, stderr = await self.process.communicate(_input)
        if encoding:
            return stdout.decode(), stderr.decode()
        else:
            return stdout, stderr

    async def run_async(
        self, _input: Optional[bytes] = None, encoding: Optional[str] = None
    ) -> Tuple[Union[str, bytes], Union[str, bytes]]:
        await self._start_async()
        return await self._communicate(_input, encoding)


"""
市场观察 爬虫
"""
import scrapy
import json
from riskweb.items import ArticleItem
import logging
import time
import re
logger = logging.getLogger(__name__)

class MediaMarketwatchSpider(scrapy.Spider):
    name = 'media_marketwatch'
    allowed_domains = ['marketwatch.com']

    def start_requests(self):
        # 基础网址
        BASEURL = 'https://www.marketwatch.com'
        #起始爬取网址
        topic_paths = [
            '/latest-news',
            '/markets',
            '/investing',
            '/personal-finance',
            '/economy-politics',
            '/economy-politics/federal-reserve',
            '/column/economic-report',
            '/column/capitol-report'
        ]

        # 发送请求
        for topic_path in topic_paths:
            yield scrapy.Request(BASEURL + topic_path, self.parse_dictionary, meta={
                'topicPath':topic_path
            }, dont_filter=False)

    def parse_dictionary(self, response):
        # 基础网址
        BASEURL = 'https://www.marketwatch.com'
        # 话题路径
        topicPath = response.meta['topicPath']

        # 解析动态请求返回的页面
        if 'partial=true' in response.url:
            article_urls = response.xpath('//div[contains(@class,"column")]//h3[@class="article__headline"]//a/@href').extract()
            for article_url in article_urls:
                yield scrapy.Request(article_url, self.parse_article, dont_filter=False)

            # 获取动态请求的参数值
            channelId = response.meta['channelId']
            position = response.meta['position']

            # 解析当前请求的最后一条消息的msgid值
            messageNumber = response.xpath(
                '//div[contains(@class,"column")]/div[contains(@class,"collection__elements")]/div[last()]/@data-msgid').extract_first()

        # 解析每个topic下的初始主页面
        else:
            container = response.xpath('//div[contains(@class,"region--primary")]')
            top_headlines = container.xpath('./div[contains(@class,"component--layout")]')
            more_headlines = container.xpath('./div[contains(@class,"more-headlines")]')
            article_urls = top_headlines.xpath(
                './div[contains(@class,"column")]//h3[@class="article__headline"]//a/@href').extract() \
                           + more_headlines.xpath(
                './div[contains(@class,"column")]//h3[@class="article__headline"]//a/@href').extract()
            for article_url in article_urls:
                yield scrapy.Request(article_url, self.parse_article, dont_filter=False)

            # 解析动态请求的参数值
            channelId = more_headlines.xpath('./div[contains(@class,"column")]/@data-channel-id').extract_first()
            position = more_headlines.xpath('./div[contains(@class,"column")]/@data-layout-position').extract_first()

            # 解析当前请求的最后一条消息的msgid值
            messageNumber = more_headlines.xpath(
                './div[contains(@class,"column")]/div[contains(@class,"collection__elements")]/div[last()]/@data-msgid').extract_first()

        # 特殊页面的请求（动态请求的参数不完全相同）
        if topicPath == '/column/capitol-report':
            pageNumber = 1
            if 'pageNumber' in response.meta.keys():
                pageNumber = response.meta['pageNumber'] + 1
            next_request_url = BASEURL + topicPath + '?pageNumber=' + str(pageNumber) + '&position=3.0&partial=true'

            yield scrapy.Request(
                next_request_url,
                self.parse_dictionary,
                meta={
                    'channelId': channelId,
                    'position': position,
                    'messageNumber': messageNumber,
                    'topicPath': topicPath,
                    'pageNumber':pageNumber,
                },
                dont_filter=False)

        # 生成普通页面的数据动态请求
        else:
            next_request_url = BASEURL + topicPath + '?messageNumber=' + str(messageNumber) + '&channelId=' + str(
                channelId) + '&position=' + str(position) + '&partial=true'

            yield scrapy.Request(
                next_request_url,
                self.parse_dictionary,
                meta={
                    'channelId': channelId,
                    'position': position,
                    'messageNumber': messageNumber,
                    'topicPath': topicPath,
                },
                dont_filter=False)

    def parse_article(self, response):
        # 调取参数
        item = ArticleItem()
        item['media_name'] = '市场观察'
        item['url'] = response.url

        # 特殊文章 video
        if '/video' in response.url:
            video_infos = response.xpath('//div[contains(@class,"video")]')
            item['title'] = video_infos.xpath('./div/h3/text()').extract_first().replace(
                r'\xa0', '').replace('\t', '').replace('\n', '').strip()
            item['topic'] = 'video'
            item['author'] = ''
            publish_date = video_infos.xpath('./div/p[@class="article__timestamp"]/text()').extract_first()

            publish_date_pattern = re.compile(
                r"(.+ \d+, \d{4} at \d+:\d+ ..) ET")  # Jun. 3, 2021 at 6:45 AM ET
            publish_date = re.findall(publish_date_pattern, publish_date)[0]
            item['publish_date'] = time.strftime("%Y/%m/%d %H:%M:%S",
                                                 time.strptime(publish_date,
                                                               "%b %d, %Y at %H:%M %p"))  # Jun. 3, 2021 at 6:45 AM ET

            raw_sentences = video_infos.xpath('./div/div[contains(@class,"article__description")]/p/text()').extract()
            raw_sentences = list(filter(lambda sentence: sentence.strip(), raw_sentences))
            item['content'] = '\n'.join(raw_sentences)

        else:
            article_infos = response.xpath('//div[@class="article__masthead"]')
            # 进入每篇文章获取标题
            item['title'] = article_infos.xpath('./h1[@class="article__headline"]/text()').extract_first().replace(
                r'\xa0', '').replace('\t', '').replace('\n', '').strip()
            # topic
            topics = response.xpath('//div[contains(@class,"breadcrumbs")]/ol/li/a/text()').extract()
            if len(topics) == 1:
                item['topic'] = topics[0]
            else:
                item['topic'] = '|'.join(topics)
            # author
            authors = article_infos.xpath('./div[contains(@class,"article__byline")]/div//h4//text()').extract()
            if len(authors) == 1:
                item['author'] = authors[0]
            else:
                item['author'] = ''.join(authors)
            # publish_date
            publish_date = article_infos.xpath('./time/text()').extract_first()
            # 正则匹配发布时间的有效数据
            if 'Updated' in publish_date:
                publish_date_pattern = re.compile(
                    r"Updated: (.+ \d+, \d{4} at \d+:\d+ .\..\.) .+")  # Last Updated: July 7, 2021 at 4:25 p.m. ET
            elif 'Published' in publish_date:
                publish_date_pattern = re.compile(
                    r"Published: (.+ \d+, \d{4} at \d+:\d+ .\..\.) .+")  # Published: July 8, 2021 at 12:04 a.m. ET
            else:
                publish_date.replace('\t', '').replace('\n', '').strip()
                publish_date_pattern = re.compile(
                    r"(.+ \d+, \d{4} at \d+:\d+ .\..\.) .+")  # July 6, 2021 at 7:18 p.m. ET

            publish_date = re.findall(publish_date_pattern, publish_date)[0]
            try:
                item['publish_date'] = time.strftime("%Y/%m/%d %H:%M:%S",
                                                     time.strptime(publish_date.replace('a.m.', 'am').replace('p.m.', 'pm'),
                                                                   "%B %d, %Y at %H:%M %p"))  # July 8, 2021 at 12:04 am
            except ValueError:
                item['publish_date'] = time.strftime("%Y/%m/%d %H:%M:%S",
                                                     time.strptime(
                                                         publish_date.replace('a.m.', 'am').replace('p.m.', 'pm'),
                                                         "%b %d, %Y at %H:%M %p"))  # Feb. 1, 2021 at 5:40 pm

            # content
            raw_sentences = response.xpath('//div[contains(@class,"article__body")]/p//text()').extract() \
                            + response.xpath('//div[contains(@class,"article__body ")]/div[contains(@class,"paywall")]//text()').extract()
            raw_sentences = list(filter(lambda sentence: sentence.strip(), raw_sentences))
            item['content'] = '\n'.join(raw_sentences)
        return item


from flask import Flask, request, jsonify
from flask_babel import Babel
from flask_cors import cross_origin

app = Flask(__name__)
babel = Babel(app)
supported = app.config.get('BABEL_SUPPORTED_LOCALES', ['en', 'ru', 'pt'])
default = app.config.get('BABEL_DEFAULT_LOCALE', 'en')

motto = {'en': 'Semantyca of your business', 'ru': 'Семантика вашего бизнеса', 'pt': 'Semantyca do seu negócio'}
subMotto = {'en': 'This is an inception. All you get is this web site', 'ru': 'Это начало. Все, что у нас есть, это этот сайт', 'pt': 'Este é um começo. Tudo que nos temos é este site'}
homeButton = {'en': 'Home', 'ru': 'Домой', 'pt': 'Para casa'}
tasksButton = {'en': 'Tasks', 'ru': 'Задачи', 'pt': 'Tarefas'}

@babel.localeselector
def get_locale():
    locale = request.accept_languages.best_match(supported, default)
    return 'pt'

@app.route('/home', methods=["GET"])
@cross_origin(origin='*')
def change_lang_default():
    return change_lang('en')

@app.route('/home/<lang>', methods=["GET"])
@cross_origin(origin='*')
def change_lang(lang):
    lang == request.view_args['lang']
    page = {
        'lang': lang,
        'name': 'Semantyca',
        'motto': motto.get(lang),
        'subMotto':  subMotto.get(lang),
        'menuHome':  homeButton.get(lang),
        'menuTasks': tasksButton.get(lang)

    }
    return jsonify(page)

@app.route('/login', methods=["POST"])
@cross_origin(origin='*')
def login():
    return 'Hello login!'

if __name__ == '__main__':
    app.run()


# coding: utf-8
import contextlib
import inspect
import logging
import os
import sys

from ._version import get_versions
__version__ = get_versions()['version']
del get_versions

@contextlib.contextmanager
def logging_restore(clear_handlers=False):
    '''
    Save logging state upon entering context and restore upon leaving.

    Parameters
    ----------
    clear_handlers : bool, optional
        If ``True``, clear active logging handlers while within context.

    Example
    -------

    Set logging level to ``DEBUG``, set logging level to ``INFO`` within
    context, and verify logging level is restored to ``DEBUG`` upon exiting
    context.

    >>> import logging_helpers as lh
    >>> import logging
    >>> logging.basicConfig(level=logging.DEBUG)
    >>> logging.debug('hello, world!')
    DEBUG:root:hello, world!
    >>> with lh.logging_restore():
    ...     logging.root.setLevel(logging.INFO)
    ...
    >>> logging.debug('hello, world!')
    DEBUG:root:hello, world!

    '''
    handlers = logging.root.handlers[:]
    level = logging.root.getEffectiveLevel()
    if clear_handlers:
        for h in handlers:
            logging.root.removeHandler(h)
    yield
    handlers_to_remove = logging.root.handlers[:]
    [logging.root.removeHandler(h) for h in handlers_to_remove]
    [logging.root.addHandler(h) for h in handlers]
    logging.root.setLevel(level)

def _L(skip=0):
    '''Shorthand to get logger for current function frame.'''
    return logging.getLogger(caller_name(skip + 1))

# Public Domain, i.e. feel free to copy/paste
# Considered a hack in Python 2
#
# Ported from [here][1].
# See [here][2] for modifications.
#
# [1]: https://gist.github.com/techtonik/2151727
# [2]: https://gist.github.com/techtonik/2151727#gistcomment-2333747
def caller_name(skip=2):
    """
    Get a name of a caller in the format module.class.method

    `skip` specifies how many levels of stack to skip while getting caller
    name. skip=1 means "who calls me", skip=2 "who calls my caller" etc.

    An empty string is returned if skipped levels exceed stack height

    .. versionchanged:: v0.4
        Look up the module name from the frame globals dictionary rather than
        by calling :func:`inspect.getmodule()`.  Calling
        :func:`inspect.getmodule()` is ~1000x slower compared to looking up the
        module name from the globals dictionary (~80 µs vs 90 ns).
    """
    def stack_(frame):
        framelist = []
        while frame:
            framelist.append(frame)
            frame = frame.f_back
        return framelist

    stack = stack_(sys._getframe(1))
    start = 0 + skip
    if len(stack) < start + 1:
        return ''
    parentframe = stack[start]

    # Look up module name from globals dictionary in frame rather than by
    # calling `inspect.getmodule()`.  Calling `inspect.getmodule()` is ~1000x
    # slower compared to looking up the module name from the globals dictionary
    # (~80 µs vs 90 ns).
    module_name = parentframe.f_globals['__name__']
    name = [module_name]

    # detect classname
    if 'self' in parentframe.f_locals:
        # I don't know any way to detect call from the object method
        # XXX: there seems to be no way to detect static method call - it will
        #      be just a function call
        name.append(parentframe.f_locals['self'].__class__.__name__)
    codename = parentframe.f_code.co_name
    if codename != '<module>':  # top level usually
        name.append(codename)  # function or a method
    return ".".join(name)