#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri May 17 11:53:14 2019

@author: zqy5086@AD.PSU.EDU
"""

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun May  5 17:59:38 2019

@author: zqy5086@AD.PSU.EDU
"""

import multiprocessing as mp
import numpy as np
import datetime
from six.moves import cPickle
import time
from sklearn.neighbors import KDTree
import matplotlib.pyplot as plt
#from gp import gp,gp_prep
from state_space import ball, gridsearch_construct, to_string,state

from car_dynamics import car_true,car,car_disturbance
#from safe_plot import progression_plot
#from safe_plot import progression
from state_space import State_Space,grid,d
from car_dynamics import car_execution_disturbance
from safety_control import safety_iteration_GP, SI, SI_GP
import pickle

from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C
def policy_search(center,X_safe,X_safe_tree,Xfree, control_history):
        ind1 = X_safe_tree.query(center.reshape(1,-1),1,dualtree=False, breadth_first=False)
        if center[2]<0:
            ind2 = X_safe_tree.query((center+np.array([0,0,2])).reshape(1,-1),1)
        else:
            ind2 = X_safe_tree.query((center-np.array([0,0,2])).reshape(1,-1),1)
        if ind1[0][0][0]<ind2[0][0][0]:
            ind=ind1
        else:
            ind=ind2
        coordinate = X_safe[ind[1][0][0]]
        key = to_string(coordinate)
        z = Xfree.table[key]
        if z.u is None:
            print(z.state)
        else:

            index=min(z.U,key=z.U.get)
            indices=[k for k in z.U if z.U[k]==z.U[index]]
            if len(indices)>1:
                new_ind=np.argmin([control_history[k] for k in indices])
                u=control_history[indices[new_ind]]
                control_history[new_ind]+=1
            else:
                u=z.U[index]
                z.U[index]+=1
                control_history[index]+=1

        return u,z,control_history

def progression(car_model,center,X_safe,X_safe_tree,Xfree,hp,ball_obs,Up):
    print('hi')
    t_end = 730.
    t_start = 0.
    t_step=0.01
    ZZ=[]
    ys=[]
    z1=None
    z2=None
    control_history=dict()
    for u in Up:
        control_history[u]=0
    while t_start<t_end:
        x = state(center)
        if not d(ball_obs,x):
            return 0,z1,z2,control_history#,U
        ys.append(center)

        u,zz,control_history=policy_search(center,X_safe,X_safe_tree,Xfree,control_history)
        ZZ.append((center,zz.state))
        z1,z2,_=np.vstack(ys).T
#        U.append(u)
        center=center+t_step*car_model(center,u)
        if center[0]<-0.01 or center[0]>1.01 or center[1]<-0.01 or center[1]>1.01:
            return 1, z1,z2,control_history

        t_start=t_start+t_step
    z11,z22,z=np.vstack(ys).T
    print(control_history)
    return 1, z11,z22,control_history #,U #,ZZ

def testing(i,X,Y):
    ZZ=[]
    for j in range(size):
        center = np.array([X[i][j],Y[i][j], angle])
        if len(X_safe)==0:
            ZZ.append(-1)
        else:
            z,z1,z2 = progression(car_model,center,X_safe,X_safe_tree,Xfree,hp,ball_obs,Up)
            ZZ.append(z)
    return (i,ZZ)
def collect_result(result):
        global results
        results.append(result)

def parent_update(s,need_update):
    for u in s.parents:
        for x in s.parents[u]:
            if not x.unsafe:
                index=np.where(x.u==u)
                if len(index[0])>0:
                    x.u=np.delete(x.u,index[0][0])
                if len(x.u)==0:
                    x.unsafe=True
#                    X_unsafe.append(x)
                    need_update.append(x)

if __name__=="__main__":
    save=True
    num=4
    size=50
    results=[]
    print(datetime.datetime.now())
    print('save is', save)
    P=5
    M = 0.005
    l = 0.005
    Time=0
    Up = np.linspace(-0.3,0.3,5)

    ratio=2.5

    obs_1 = State_Space(3,[0.47,0.52,0.25,0.75,-1,1])
    Xfree = grid([[0,1],[0,1],[-1,1]])
    Xfree.construct(P,Up)
    plot_progress=False
    plot_control=True

    left = obs_1.parameter[0]
    down = obs_1.parameter[2]
    width = obs_1.parameter[1]-obs_1.parameter[0]
    height = obs_1.parameter[3]-obs_1.parameter[2]

    X_unsafe=[]
    model=car
    car_model=car_disturbance
    theta = np.linspace(-1,1,2**(P-1)+1)
    angle = theta[-1]
    initial=np.array([1,0.8, angle])

    T_end=10
#    kernel = C(0.1, (1e-3, 1)) * RBF(0.001, (1e-5, 1e-2))
    kernel = C(1e-4, (1e-6, 1e-3)) * RBF(0.05, (1e-2, 1))
    Y, X = np.meshgrid(np.linspace(0, 1, size), np.linspace(0, 1, size))
    _,Z = np.meshgrid(np.linspace(0, 1, size), np.linspace(0, 1, size))
    ball_obs = ball(obs_1,0)
    leaf_size=200

    control_history=dict()
    for u in Up:
        control_history[u]=0

    print('computing backup controller')
    ######################################################################################################
    #######################################backup controller phase ########################################
    start = time.time()
    if save:
        for p in range(1,P+1):
            print(p)
            hp = 2**(-p)
            eps_p = (hp/(ratio*M*l))**(1/2)
            alpha_p=hp+l*eps_p*hp/2+M*l*eps_p**2
            ball_obs_1 = ball(obs_1,M*eps_p+hp)

            #table construction based on dynamics
            need_update=[]
            X_temp,state_temp,X_unsafe = gridsearch_construct(Xfree,p,ball_obs_1)
            X_temp_tree=KDTree(X_temp, leaf_size=leaf_size, metric='chebyshev')
#            cnt=0

            for s in state_temp:
#                print(cnt)
#                cnt+=1
                if not s.unsafe:
                    SI(model,s,Xfree,Up,eps_p,X_temp_tree,X_temp,alpha_p)
                    if s.unsafe:
#                        X_unsafe.append(s)
                        need_update.append(s)

            for x in need_update:
                parent_update(x,need_update)

        X_safe = [x.state for x in state_temp if not x.unsafe]
#        X_safe_control=[x.u for x in state_temp if not x.unsafe]

        if len(X_safe)>0:
            X_safe_tree=KDTree(X_safe, leaf_size=leaf_size, metric='manhattan')
#        with open("safe_tree_backup.save", "wb") as f:
#            cPickle.dump(X_safe_tree, f)
#        with open("X_safe_backup.save", "wb") as f:len(X_safe_)/(2**p+1)**3>0.2
#            cPickle.dump(X_safe, f)
#        with open("X_safe_control_backup.save","wb") as f:
#            cPickle.dump(X_safe_control,f)
#    else:
#        print('loading...')
#        with open("safe_tree_backup.save", "rb") as fp:   # Unpickling
#            X_safe_tree = cPickle.load(fp)
#        with open("X_safe_backup.save", "rb") as fp:   # Unpickling
#            X_safe = cPickle.load(fp)
#        with open("X_safe_control_backup.save","rb") as fp:
#            X_safe_control = cPickle.load(fp)
#        for x,u in zip(X_safe,X_safe_control):
#            key=to_string(x)
#            Xfree.table[key].u=u

    if plot_control:
            print('plotting control grid...')
            end = time.time()
            t = end-start
            Time = Time+t
            plt.figure()
            T=[]
            for angle in theta:
                X_temp_,state_temp,_ = gridsearch_construct(Xfree,P,fix=[2,angle])
                if save:
                    T.append([float(not x.unsafe) for x in state_temp])
                else:
                    for x in state_temp:
                        if x.u is None:
                            x.unsafe=True
                        else:
                            x.unsafe=False
                    T = np.array([float(not x.unsafe) for x in state_temp])
            T=np.stack(T)
            T=np.sum(T,axis=0)/len(theta)
            T[T==0]=-1
            X1 = [float(x.state[0]) for x in state_temp]
            X2 = [float(x.state[1]) for x in state_temp]
            center=initial
            z,z1,z2,control_record = progression(car_model,center,X_safe,X_safe_tree,Xfree,hp,ball_obs,Up)
            X_backup=X_safe
            X_backup_tree=X_safe_tree
            Xbackup=Xfree
            plt.scatter(X1,X2, c=T, vmin=0, vmax=1)
            plt.plot(z1,z2,color='red')
            plt.colorbar()
            rectangle=plt.Rectangle((left,down),width,height,fc='red')
            plt.gca().add_patch(rectangle)
            plt.xlabel('Time elapsed: '+str(round(Time*1000)/1000)+', '+ str(len(X_temp))+' points',fontsize=16)# subgrid construction time: '+str(round((end_-start)*1000)/1000),fontsize=16)
            plt.title(' Safe points: '+str(len(X_safe)),fontsize=16)

    t_end=T_end
    t_step=0.1
    t_start=0
    #initial
    car_x = initial

    record=[]
    train=[]
    train_f=[]
    record.append(car_x)

    success, record, train_new,train_f_new,car_x,control_history=car_execution_disturbance(car_x,t_end,t_step,X_safe_tree,X_safe,ball_obs,record,Xfree,control_history)
    car_trajectory=np.stack(record)

    if plot_progress:
        results=[]
        print('computing backup progression map')
        plt.figure()
        pool = mp.Pool(mp.cpu_count())
        for i in range(size):
    #        print(i)
        #    testing(i,X,Y)
            pool.apply_async(testing, args=(i,X,Y),callback=collect_result)
        pool.close()
        pool.join()
        results.sort(key=lambda x: x[0])
        for i in range(size):
            for j in range(size):
                Z[i][j]=results[i][1][j]
        plt.pcolormesh(X,Y,Z,vmin=-1, vmax=1)
#        plt.title('Initial Value: ['+str(initial[0])+','+str(initial[1])+','+str(initial[2])+']')

#        plt.plot(z1,z2)
        plt.colorbar()
        rectangle=plt.Rectangle((left,down),width,height,fc='red')
        plt.gca().add_patch(rectangle)
        plt.plot(car_trajectory[:,0],car_trajectory[:,1],color='blue')
#####################################################################################################################################3
############################################Learning dynamic phase####################################################################

    train=np.stack(train_new)
    train_f=np.stack(train_f_new)
#    mu_prep,cov_prep=gp_prep(kernel,train, train_f,sigma,car_true)
    gpr = GaussianProcessRegressor(kernel=kernel, normalize_y=True).fit(train, train_f)
    M_=np.max(train_f)
    M=max([M,M_])

    Xfree_ = grid([[0,1],[0,1],[-1,1]])
    Xfree_.construct(P,Up)
    switch=[]
    k_=0
    safe_index=0;
    safe_index_old=0;
    tol=0.1
    p_=1
   ##################################################################################################################################
    #################################################iterations of progression plots #################################################
    for k in range(num):
#        print(k)
#        filename='X_safe_GP_'+str(k)+'.save'
#        filename2='safe_tree_GP_'+str(k)+'.save'
#        filename3='X_safe_control_GP_'+str(k)+'.save'
        #safety controller
        print('computing safety controller...')
        ############################################################################################################
        ##################################################GP controller phase########################################
        start = time.time()
        Time=0
        t=0

        if save:
            for p in range(p_,P+1):
                print(k,p)
                hp = 2**(-p)
                eps_p = (hp/(ratio*M*l))**(1/2)
                ball_obs_1 = ball(obs_1,M*eps_p+hp)
                alpha_p=2*hp+l*eps_p*hp+M*l*eps_p**2
                #table construction based on dynamics
                need_update=[]
                X_temp,state_temp,X_unsafe = gridsearch_construct(Xfree_,p,ball_obs_1)
                X_temp_tree=KDTree(X_temp,leaf_size=leaf_size,metric='chebyshev')
                for s in state_temp:
                    if not s.unsafe:
                            SI_GP(s,Xfree_,Up,eps_p,X_temp_tree,X_temp,gpr,hp,M,l)
                            if s.unsafe:
#                                X_unsafe.append(s)
                                need_update.append(s)
                for x in need_update:
                    parent_update(x,need_update)

                X_safe_ = [x.state for x in state_temp if not x.unsafe]
                sp=len(X_safe_)/len(X_temp)

                if sp>safe_index:
                    print('switching')
                    safe_index=sp
                            ######################################################################################################################
        #################################################33#progression plot###################################################3
                    end = time.time()
                    t = end-start-t
                    if plot_control:
                        print('plotting controller plot...')
                        Time = Time+t
                        plt.figure()
                        theta = np.linspace(-1,1,2**(p-1)+1)
                        T=[]
                        for angle in theta:
                            X_temp_,state_temp,_ = gridsearch_construct(Xfree_,p,fix=[2,angle])
                            if save:
                                T.append([float(not x.unsafe) for x in state_temp])
                            else:
                                for x in state_temp:
                                    if x.u is None:
                                        x.unsafe=True
                                    else:
                                        x.unsafe=False
                                T = np.array([float(not x.unsafe) for x in state_temp])
                        T=np.stack(T)
                        T=np.sum(T,axis=0)/len(theta)
                        T[T==0]=-1
                        X1 = [float(x.state[0]) for x in state_temp]
                        X2 = [float(x.state[1]) for x in state_temp]
                        plt.scatter(X1,X2, c=T, cmap='viridis', vmin=0, vmax=1)
                        plt.plot(car_trajectory[:,0],car_trajectory[:,1],color='blue')
                        plt.colorbar()
                        rectangle=plt.Rectangle((left,down),width,height,fc='red')
                        plt.gca().add_patch(rectangle)
                        plt.xlabel('Time elapsed: '+str(round(Time*1000)/1000)+', '+ str(len(X_temp))+' points',fontsize=16)# subgrid construction time: '+str(round((end_-start)*1000)/1000),fontsize=16)
                        plt.title(' Safe points: '+str(len(X_safe_)),fontsize=16)

                    ball_obs = ball(obs_1,0)

                    print('in safe controller car execution...')
                    t_end=t

                    success, record, _,_,car_x,control_history=car_execution_disturbance(car_x,t_end,t_step,X_safe_tree,X_safe,ball_obs,record,Xfree,control_history)

                    if not success:
                        print('collision!!!!')
                    X_safe=X_safe_
                    X_safe_tree=KDTree(X_safe, leaf_size=leaf_size,metric='manhattan')
                    car_trajectory_=np.stack(record)
                    Xfree=Xfree_
                    switch.append(car_trajectory_[-1,:2])

                    print('car execution...')

                    t_end=T_end
                    success, record, train_new,train_f_new,car_x,control_history=car_execution_disturbance(car_x,t_end,t_step,X_safe_tree,X_safe,ball_obs,record,Xfree,control_history)
                    print('model updating...')
                    train=np.concatenate((train,train_new))
                    train_f=np.concatenate((train_f,train_f_new))
                    car_trajectory=np.stack(record)

                    if plot_progress:
                        print('plotting progression plot...')
                        results=[]
                        print('progression map...')
                        plt.figure()
                        pool = mp.Pool(mp.cpu_count())
                        for i in range(size):
                #            print(i)        ######################################################################################################################
        #################################################33#progression plot###################################################3
                            pool.apply_async(testing, args=(i,X,Y),callback=collect_result)
                        pool.close()
                        pool.join()
                        results.sort(key=lambda x: x[0])
                        for i in range(size):
                            for j in range(size):
                                Z[i][j]=results[i][1][j]
                        plt.pcolormesh(X,Y,Z,vmin=-1, vmax=1)
#                        plt.title('Initial Value: ['+str(initial[0])+','+str(initial[1])+','+str(initial[2])+']')
                    #    plt.plot(z1,z2)
                        plt.colorbar()
                        rectangle=plt.Rectangle((left,down),width,height,fc='red')
                        plt.gca().add_patch(rectangle)
                        plt.plot(car_trajectory[:,0],car_trajectory[:,1],color='blue')

                        plt.plot(car_trajectory_[:,0],car_trajectory_[:,1],color='black')
                        switch_points=np.stack(switch)
                        plt.scatter(switch_points[:,0],switch_points[:,1], color='black')

                        ########################################################################################
                        ################################ learning ###################################################
                    print('update uncertainty...')
                    gpr = GaussianProcessRegressor(kernel=kernel, normalize_y=True).fit(train, train_f)
                    M_=np.max(train_f)
                    M=max([M,M_])

                    if safe_index-safe_index_old<tol and safe_index>tol:
                        print('refine grid ...')
                        safe_index_old=safe_index
                        p_=p+1
                        break
                    else:
                        safe_index_old=safe_index
                        start=time.time()
                        p_=p
                        break
    print('safety test...')
    t_end=430
    success, record, _,_,car_x,control_history=car_execution_disturbance(car_x,t_end,t_step,X_safe_tree,X_safe,ball_obs,record,Xfree,control_history)
    car_trajectory=np.stack(record)

    plt.figure()
    plt.pcolormesh(X,Y,Z,vmin=-1, vmax=1)

    plt.colorbar()
    rectangle=plt.Rectangle((left,down),width,height,fc='red')
    plt.gca().add_patch(rectangle)
    plt.plot(car_trajectory[:,0],car_trajectory[:,1],color='blue')

    plt.plot(car_trajectory_[:,0],car_trajectory_[:,1],color='black')
    switch_points=np.stack(switch)
    plt.scatter(switch_points[:,0],switch_points[:,1], color='black')

#            with open(filename, "wb") as fp:   #Pickling
#                cPickle.dump(X_safe, fp)
#            with open(filename2, "wb") as fp:   #Pickling
#                cPickle.dump(X_safe_tree, fp)
#            with open(filename3,"wb") as fp:
#                cPickle.dump(X_safe_control,fp)
#        else:
#            print('loading...')
#            with open(filename, "rb") as fp:   # Unpickling
#                X_safe = cPickle.load(fp)
#            with open(filename2, 'rb') as f:
#                X_safe_tree = cPickle.load(f)
#            with open(filename3,'rb') as f:
#                X_safe_control=cPickle.load(f)
#            for x,u in zip(X_safe,X_safe_control):
#                key=to_string(x)
#                Xfree.table[key].u=u

#        plt.plot(car_trajectory[:,0],car_trajectory[:,1])
        #learn model

    #######################################################################################################################################
    #########################################################ends iterations#################################################################3
#    print('test trajectory')
#
#    left = obs_1.parameter[0]
#    down = obs_1.parameter[2]
#    width = obs_1.parameter[1]-obs_1.parameter[0]
#    height = obs_1.parameter[3]-obs_1.parameter[2]
#    if len(X_safe)>0:
#        X_safe_tree=KDTree(X_safe)
#    center=np.array([1,0.7,-angle])
#    z,z1,z2= progression(car_disturbance,center,X_safe,X_safe_tree,Xfree,hP,ball_obs)
#
#    plt.figure()
#    plt.plot(car_trajectory[:,0],car_trajectory[:,1])
#    plt.pcolormesh(X,Y,Z,vmin=-1, vmax=1)
#    plt.title('Initial Value: ['+str(initial[0])+','+str(initial[1])+','+str(initial[2])+']')
#    plt.colorbar()
#    rectangle=plt.Rectangle((left,down),width,height,fc='red')
#    plt.gca().add_patch(rectangle)
##    plt.plot(z1,z2)
#
#    plt.figure()
#    X_temp_,state_temp,_ = gridsearch_construct(Xfree,P,fix=[2,angle])
#    if save:
#        T = [float(not x.unsafe) for x in state_temp]
#    else:
#        for x in state_temp:
#            if x.u is None:
#                x.unsafe=True
#            else:
#                x.unsafe=False
#        T = [float(not x.unsafe) for x in state_temp]
#
#    X1 = [float(x.state[0]) for x in state_temp]
#    X2 = [float(x.state[1]) for x in state_temp]
#
#    plt.scatter(X1,X2, c=T)
#    plt.colorbar()
#    rectangle=plt.Rectangle((left,down),width,height,fc='red')
#    plt.gca().add_patch(rectangle)
#    plt.plot(z1,z2)
#


from __future__ import annotations
from typing import Iterator, NamedTuple, Iterable, Optional
from difflib import SequenceMatcher
import enum

from ezdxf.lldxf.tags import Tags
from ezdxf.lldxf.types import DXFVertex, DXFTag

__all__ = ["diff_tags", "print_diff", "OpCode"]

# https://docs.python.org/3/library/difflib.html

class OpCode(enum.Enum):
    replace = enum.auto()
    delete = enum.auto()
    insert = enum.auto()
    equal = enum.auto()

class Operation(NamedTuple):
    opcode: OpCode
    i1: int
    i2: int
    j1: int
    j2: int

CONVERT = {
    "replace": OpCode.replace,
    "delete": OpCode.delete,
    "insert": OpCode.insert,
    "equal": OpCode.equal,
}

def convert_opcodes(opcodes) -> Iterator[Operation]:
    for tag, i1, i2, j1, j2 in opcodes:
        yield Operation(CONVERT[tag], i1, i2, j1, j2)

def round_tags(tags: Tags, ndigits: int) -> Iterator[DXFTag]:
    for tag in tags:
        if isinstance(tag, DXFVertex):
            yield DXFVertex(tag.code, (round(d, ndigits) for d in tag.value))
        elif isinstance(tag.value, float):
            yield DXFTag(tag.code, round(tag.value, ndigits))  # type: ignore
        else:
            yield tag

def diff_tags(
    a: Tags, b: Tags, ndigits: Optional[int] = None
) -> Iterator[Operation]:
    if ndigits is not None:
        a = Tags(round_tags(a, ndigits))
        b = Tags(round_tags(b, ndigits))

    sequencer = SequenceMatcher(a=a, b=b)
    return convert_opcodes(sequencer.get_opcodes())

def print_diff(a: Tags, b: Tags, operations: Iterable[Operation]):
    t1: DXFTag
    t2: DXFTag
    print()
    for op in operations:
        if op.opcode == OpCode.insert:
            for t1 in b[op.j1 : op.j2]:
                print(f"insert a[{op.i1}:{op.i2}]: {t1}")
        elif op.opcode == OpCode.replace:
            for index, t1 in enumerate(a[op.i1 : op.i2], op.i1):
                print(f"replace a[{index}]: {t1}")
            for index, t2 in enumerate(b[op.j1 : op.j2], op.j1):
                print(f"     by b[{index}]: {t2}")
        elif op.opcode == OpCode.delete:
            for index, t1 in enumerate(a[op.i1 : op.i2], op.i1):
                print(f"delete a[{index}]: {t1}")


import requests
from bs4 import BeautifulSoup
import re
import mysql.connector
from sklearn import tree
myask1 = input('Enter 1 for all information, otherwise press enter')
erfan = mysql.connector.connect(
        host='localhost',
        user='root',
        passwd='',
        database=''
)
my_countery = ['us', 'spain', 'italy', 'India', 'france', 'germany', 'turkey', 'russia', 'iran', 'brazil','peru','chile','israel','japan'
                   ,'morocco','kuwait','finland','malaysia','egypt','australia','panama','belgium','ecuador','sweden','pakistan'
                   ,'chile','singapore','belarus','qatar','romania','ukraine','indonesia','denmark']
x2 = []
y2 = []
x = []
y = []
uq = 1
tedadnoghte = 1
for i in my_countery:
    my_list = []
    my_list3 = []
    my_list2 = []

    m = 0
    get = requests.get('https://www.worldometers.info/coronavirus/country/' + i + "/")
    one = BeautifulSoup(get.text, "html.parser")
    two = one.find_all('h1')
    three = one.find_all('div', attrs={'class': 'maincounter-number'})
    for ii in two:
        m = re.sub('\s+', '', ii.text).strip()
        my_list.append(m)
    for b in three:
        l = re.sub('\s+', '', b.text).strip()
        my_list2.append(l)
    g = len(my_list2)
    Cursor = erfan.cursor()

    a = str(my_list[0])
    b = str(my_list2[0])
    x.append(int(b.replace(',','')))
    c = str(my_list2[1])

    d = str(my_list2[2])
    x.append(int(d.replace(',','')))

    x.append(int(c.replace(',', '')))
    x2.append(x[0:2])
    y2.append(x[2])
    print(i, 'cheak')
    print('.' * tedadnoghte)
    Cursor.execute('INSERT INTO database name VALUE(\'%s\',\'%s\',\'%s\',\'%s\')' % (a, b, c, d))
    erfan.commit()
    my_list.clear()
    my_list2.clear()
    uq = uq + 1
    x.clear()
    tedadnoghte += 1
erfan = mysql.connector.connect(
    host='localhost',
    user='root',
    passwd='',
    database=''
    )
Cursor = erfan.cursor()
query = 'SELECT * FROM database name'
Cursor.execute(query)
for (name, coronaviruscases, deaths, recovered) in Cursor:
    a = str(name)
    b = str(coronaviruscases)
    c = str(deaths)
    d = str(recovered)
    if myask1 == '1':
        print('name = ', a, '/', 'corona virus cases = ', b, '/', 'deaths = ', c, '/', 'recovered = ', d)
cif = tree.DecisionTreeClassifier()
cif = cif.fit(x2,y2)
tedad_case_fayal = input('Enter the number of corona patients : ')
tedad_khob_shodegan = input('Enter the number of people who have fully recovered : ')

new_data = [[tedad_case_fayal,tedad_khob_shodegan]]
answer = cif.predict(new_data)
print(answer,'will probably die')
erfan.close()


import requests
import os

SHETTY_PRICES_ENDPOINTS = os.environ["SHETTY_PRICE_ENDPOINTS"]
SHETTY_USER_ENDPOINT = os.environ["SHETTY_USER_ENDPOINT"]

class DataManager:
    """"This class will handle all kind of task with SHETTY API"""

    def __init__(self):
        self.destination_data = {}

    def get_destination_data(self) -> list:
        """This method will retrieve all destination data from SHETTY API"""

        response = requests.get(url=SHETTY_PRICES_ENDPOINTS)
        data = response.json()
        self.destination_data = data["prices"]
        return self.destination_data

    def update_destination_codes(self):
        """This method will update missing IATA code on SHETTY API"""

        for city in self.destination_data:
            new_data = {
                "price": {
                    "iataCode": city["iataCode"]
                }
            }

            response = requests.put(url=f"{SHETTY_PRICES_ENDPOINTS}/{city['id']}", json=new_data)

    def get_customer_emails(self) -> list:
        """"This method will retrieve all emails from SHETTY API"""

        response = requests.get(url=SHETTY_USER_ENDPOINT)
        data = response.json()
        self.customer_data = data["users"]
        return self.customer_data


from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
from PyQt5.QtWebEngineWidgets import QWebEngineView
from PyQt5.QtPrintSupport import *
import AboutDialog
import InsertDialog
import DeleteDialog
import SearchDialog
import sys
import sqlite3
import time
import MainWindowDepartments
import MainWindowStudent
import LoginDialog as loginDialog
import ShowAllDepartment
import UniversitySearch

class MainWindowUniversity(QMainWindow):

    def __init__(self, *args, **kwargs):
        super(MainWindowUniversity, self).__init__(*args, **kwargs)

        self.conn = sqlite3.connect("info.db")
        self.c = self.conn.cursor()
        self.c.execute(
            "CREATE TABLE IF NOT EXISTS Universities (" +
            "uni_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE," +
            "uni_name TEXT NOT NULL UNIQUE)")
        self.c.close()

        file_menu = self.menuBar().addMenu("&File")

        self.setWindowTitle("Universities Management")
        self.setWindowState(Qt.WindowMaximized)
        self.setMinimumWidth(500)
        self.setMinimumHeight(250)
        self.main_container = QWidget(self)
        main_layout = QGridLayout(self.main_container)
#####################  Student Data  ###################
        self.tableWidget = QTableWidget()
        self.tableWidget.setAlternatingRowColors(True)
        self.tableWidget.setColumnCount(4)
        self.tableWidget.horizontalHeader().setCascadingSectionResizes(False)
        self.tableWidget.horizontalHeader().setSortIndicatorShown(False)
        self.tableWidget.verticalHeader().setVisible(False)
        self.tableWidget.verticalHeader().setStretchLastSection(False)
        self.tableWidget.setFixedWidth(800)
        self.tableWidget.setColumnWidth(0, 70)
        self.tableWidget.setColumnWidth(1, 495)
        self.tableWidget.setColumnWidth(2, 105)
        self.tableWidget.setColumnWidth(3, 125)
        self.tableWidget.setHorizontalHeaderLabels(
            ("ID", "University", "New ", "Departments"))

        main_layout.addWidget(self.tableWidget, 0, 0)
        self.setCentralWidget(self.main_container)

        toolbar = QToolBar()
        toolbar.setMovable(False)
        self.addToolBar(toolbar)

        statusbar = QStatusBar()
        self.setStatusBar(statusbar)

        btn_ac_adduser = QAction(
            QIcon("/home/salah/Desktop/Py-infoman-gui-project/icon/real-estate.png"), "Add University", self)
        btn_ac_adduser.triggered.connect(self.insert)
        btn_ac_adduser.setStatusTip("Add Student")
        toolbar.addAction(btn_ac_adduser)

        btn_ac_refresh = QAction(QIcon(
            "/home/salah/Desktop/Py-infoman-gui-project/icon/refresh .png"), "Refresh", self)
        btn_ac_refresh.triggered.connect(self.loaddata)
        btn_ac_refresh.setStatusTip("Refresh Table")
        toolbar.addAction(btn_ac_refresh)

        btn_ac_search = QAction(QIcon(
            "/home/salah/Desktop/Py-infoman-gui-project/icon/search.png"), "Search", self)
        btn_ac_search.triggered.connect(self.search)
        btn_ac_search.setStatusTip("Search User")
        toolbar.addAction(btn_ac_search)

        btn_ac_delete = QAction(QIcon(
            "/home/salah/Desktop/Py-infoman-gui-project/icon/trash.png"), "Delete", self)
        btn_ac_delete.triggered.connect(self.delete)
        btn_ac_delete.setStatusTip("Delete User")
        toolbar.addAction(btn_ac_delete)

        self.container = QWidget(self)
        self.container.setFixedWidth(20)
        toolbar.addWidget(self.container)

        self.container = QWidget(self)
        self.container.setFixedWidth(20)
        toolbar.addWidget(self.container)

        self.container = QWidget(self)
        self.container.setFixedWidth(650)
        toolbar.addWidget(self.container)

        self.container2 = QWidget(self)
        self.container2.setFixedWidth(400)
        layout = QGridLayout(self.container2)

        search_by_name = QLineEdit()
        search_by_name.setPlaceholderText("search")
        search_by_name.setFixedWidth(200)
        layout.addWidget(search_by_name, 0, 0)

        QBtn = QPushButton()
        QBtn.setIcon(
            QIcon("/home/salah/Desktop/Py-infoman-gui-project/icon/search.png"))
        QBtn.clicked.connect(
            lambda: self.search_by(search_by_name.text()))

        loout_Btn = QPushButton()
        loout_Btn.setIcon(
            QIcon("/home/salah/Desktop/Py-infoman-gui-project/icon/logout.png"))
        loout_Btn.setStatusTip("Logout")
        loout_Btn.clicked.connect(self.logout)

        space = QWidget(self)
        space.setFixedWidth(100)

        layout.addWidget(QBtn, 0, 1)
        layout.addWidget(space, 0, 2)
        toolbar.addWidget(self.container2)

        logout_Btn = QAction(
            QIcon("/home/salah/Desktop/Py-infoman-gui-project/icon/logout.png"), "Logout", self)
        logout_Btn.triggered.connect(self.logout)
        logout_Btn.setStatusTip("LOGOUT")
        toolbar.addAction(logout_Btn)

        btn_ac_close = QAction(
            QIcon("/home/salah/Desktop/Py-infoman-gui-project/icon/criss-cross.png"), "Closs App", self)
        btn_ac_close.triggered.connect(self.close_app)
        btn_ac_close.setStatusTip("Closs App")
        toolbar.addAction(btn_ac_close)

        show_deparments = QAction(
            QIcon("/home/salah/Desktop/Py-infoman-gui-project/icon/school.png"), "Show All Deparments", self)
        show_deparments.triggered.connect(self.show_all_departments)
        file_menu.addAction(show_deparments)

        adduser_action = QAction(
            QIcon("/home/salah/Desktop/Py-infoman-gui-project/icon/real-estate.png"), "Insert University", self)
        adduser_action.triggered.connect(self.insert)
        file_menu.addAction(adduser_action)

        searchuser_action = QAction(
            QIcon("/home/salah/Desktop/Py-infoman-gui-project/icon/search.png"), "Search university", self)
        searchuser_action.triggered.connect(self.search)
        file_menu.addAction(searchuser_action)

        deluser_action = QAction(
            QIcon("/home/salah/Desktop/Py-infoman-gui-project/icon/trash.png"), "Delete University", self)
        deluser_action.triggered.connect(self.delete)
        file_menu.addAction(deluser_action)

        rm_all_action = QAction(
            QIcon("/home/salah/Desktop/Py-infoman-gui-project/icon/criss-cross.png"), "Delete All Universities", self)
        rm_all_action.triggered.connect(self.delete_all)
        file_menu.addAction(rm_all_action)

        about_action = QAction(QIcon(
            "/home/salah/Desktop/Py-infoman-gui-project/icon/info.png"), "About Developer", self)
        about_action.triggered.connect(self.about)
        file_menu.addAction(about_action)

        close_action = QAction(QIcon(
            "/home/salah/Desktop/Py-infoman-gui-project/icon/close.png"), "Close", self)
        close_action.triggered.connect(self.close_app)
        file_menu.addAction(close_action)

    def loaddata(self):
        self.connection = sqlite3.connect("info.db")
        query = "SELECT * FROM Universities"
        result = self.connection.execute(query)
        self.tableWidget.setRowCount(0)
        for row_number, row_data in enumerate(result):
            uniName = ""
            self.tableWidget.insertRow(row_number)
            for column_number, data in enumerate(row_data):
                self.tableWidget.setItem(
                    row_number, column_number, QTableWidgetItem(str(data)))

            adddeptbut = QPushButton(self.tableWidget)
            adddeptbut.setText('Add')
            adddeptbut.clicked.connect(self.addDepartment)

            deptbut = QPushButton(self.tableWidget)
            deptbut.setText('Get')
            deptbut.clicked.connect(self.getDepartments)

            self.tableWidget.setCellWidget(row_number, 2, adddeptbut)
            self.tableWidget.setCellWidget(row_number, 3, deptbut)

        self.connection.close()

    def getDepartments(self):
        window = MainWindowDepartments.MainWindowDepartments(self.tableWidget.item(
            self.tableWidget.currentRow(), 0).text(), self.tableWidget.item(
            self.tableWidget.currentRow(), 1).text())
        window.show()
        window.loaddata(self.tableWidget.item(
            self.tableWidget.currentRow(), 0).text())

    def addDepartment(self):
        dlg = InsertDialog.InsertDepartment(self.tableWidget.item(
            self.tableWidget.currentRow(), 0).text())
        dlg.exec_()
        self.loaddata()

    def handlePaintRequest(self, printer):
        document = QTextDocument()
        cursor = QTextCursor(document)
        model = self.table.model()
        table = cursor.insertTable(
            model.rowCount(), model.columnCount())
        for row in range(table.rows()):
            for column in range(table.columns()):
                cursor.insertText(model.item(row, column).text())
                cursor.movePosition(QTextCursor.NextCell)
        document.print_(printer)

    def insert(self):
        dlg = InsertDialog.InsertUniversity()
        dlg.exec_()
        self.loaddata()

    def delete(self):
        dlg = DeleteDialog.DeleteUniversity()
        dlg.exec_()
        self.loaddata()

    def delete_all(self):
        qm = QMessageBox()
        ret = qm.warning(
            self, '', "All the departments will be removed !\n\n\n Are You sure?", qm.Yes | qm.No)
        if ret == qm.Yes:
            try:
                self.conn = sqlite3.connect("info.db")
                self.c = self.conn.cursor()
                self.c.execute("DELETE from Universities")
                self.conn.commit()
                self.c.close()
                self.conn.close()
                QMessageBox.information(
                    QMessageBox(), 'Successful', 'All the records are removed from the database.')

            except Exception as error:
                print(error)
                QMessageBox.warning(QMessageBox(), 'Error',
                                    'Could not delete the records from the database.')

            self.loaddata()

    def search(self):
        dlg = SearchDialog.SearchUniversity()
        dlg.exec_()

    def about(self):
        dlg = AboutDialog.AboutDialog()
        dlg.exec_()

    def show_all_departments(self):
        window = ShowAllDepartment.ShowAllDepartment(self)
        window.show()
        window.loaddata()

    def removwidget(self, main_layout, widef):
        main_layout.removeWidget(self.widef)
        try:
            self.widef.deleteLater()
        except Exception:
            print("error")

        self.widef = None

    def logout(self):
        self.close()
        passchick = loginDialog.LoginDialog()
        if(passchick.exec_() == QDialog.Accepted):
            choossedlg = loginDialog.LoginChoice()
            status = choossedlg.exec_()
            if(status == QDialog.Accepted):
                if(choossedlg.getChoice() == "university"):
                    self.show()

                elif(choossedlg.getChoice() == "Student"):
                    window = MainWindowStudent.MainWindowStudent()
                    window.show()
                    window.loaddata()

    def close_app(self):
        self.close()

    def search_by(self, parmeter):
        if(parmeter.strip() == ""):
            QMessageBox.warning(QMessageBox(), 'Error',
                                'Yu must enter at leasr one key word!')
        else:
            window = UniversitySearch.UniversitySearch(self)
            window.show()
            window.loaddata(parmeter)


import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--verbose', default=False, action='store_true')
args = parser.parse_args()

import numpy as np
import tensorflow as tf
from utils.layers import ChannelwiseAttention, SpatialAttention
from utils.models import make_attention_cnn
from utils.testing import test

vgg = tf.keras.applications.VGG16()
vgg.compile(loss='categorical_crossentropy', metrics=['acc'])

# Mixed precision needs to be off to match VGG16
attn_layer_chan = ChannelwiseAttention(init='ones')
attn_layer_spat = SpatialAttention(init='ones')
attn_cnn_channel_conv5 = make_attention_cnn(attn_layer_chan, 'conv5', False)
attn_cnn_channel_image = make_attention_cnn(attn_layer_chan, 'image', False)
attn_cnn_spatial_conv5 = make_attention_cnn(attn_layer_spat, 'conv5', False)
attn_cnn_spatial_image = make_attention_cnn(attn_layer_spat, 'image', False)

models = (
    vgg,
    attn_cnn_channel_conv5, attn_cnn_channel_image,
    attn_cnn_spatial_conv5, attn_cnn_spatial_image)

if args.verbose:
    for model in models:
        print(model.summary(), '\n')

accuracies = (
    test(vgg, 'image_generator'), test(vgg, 'image'),
    test(attn_cnn_channel_conv5, 'conv5'), test(attn_cnn_channel_image, 'image'),
    test(attn_cnn_spatial_conv5, 'conv5'), test(attn_cnn_spatial_image, 'image'))

close = [np.allclose(accuracies[0], accuracy) for accuracy in accuracies[1:]]
result = 'passed' if np.all(close) else 'failed'
print(f'\nSanity check {result}.')


from django.forms import ModelForm, TextInput, NumberInput
from .models import Product

class ProductForm(ModelForm):
    class Meta:
        model = Product
        fields = ['name', 'price', 'description', 'stock', 'image']
        widgets = {
            'price': NumberInput(attrs={'step': 0.01})
        }


from PyQt5.QtCore import Qt
from PyQt5.QtGui import QCursor, QColor

from qgis.gui import ( QgsMapTool,
                       QgsHighlight )
from qgis.core import ( QgsFields,
                        QgsRectangle,
                        QgsGeometry,
                        QgsFeatureRequest,
                        QgsRenderContext)

from qgis_agri import agriConfig, tr
from qgis_agri.log.logger import QgisLogger as logger
from qgis_agri.gui.layer_util import QGISAgriLayers

#
#-----------------------------------------------------------
class DifferenceSuoliTool(QgsMapTool):

    # --------------------------------------
    #
    # --------------------------------------
    def __init__(self,
                 iface,
                 layer,
                 suoliRefLayers=None,
                 suoliMinArea=agriConfig.TOLERANCES.AREA_MIN_VALID_SUOLO,
                 snapTolerance=agriConfig.TOLERANCES.SNAP_TOLERANCE,
                 suoliSnapLayers=None,
                 msgTitle='QGIS Agri'):

        """ Constructor """
        super().__init__( iface.mapCanvas() )
        self.name = 'DifferenceSuoliTool'
        self.iface = iface
        self.canvas = iface.mapCanvas()
        self.layer = layer
        self.suoliRefLayers = suoliRefLayers
        self.suoliMinArea = suoliMinArea
        self.snapTolerance = snapTolerance
        self.suoliSnapLayers = suoliSnapLayers
        self.highlightGeoms = []
        self.cursor = QCursor(Qt.CrossCursor)
        self.msgTitle = msgTitle
        self.initialize( layer, suoliRefLayers )

    # --------------------------------------
    #
    # --------------------------------------
    def initialize(self, layer, suoliRefLayers=None):
        self.layer = layer
        self.suoliRefLayers = suoliRefLayers

    # --------------------------------------
    #
    # --------------------------------------
    def activate(self):
        super().activate()
        self._deselect_all()
        self.canvas.setCursor(self.cursor)

    # --------------------------------------
    #
    # --------------------------------------
    def deactivate(self):
        super().deactivate()
        self._deselect_all()

    # --------------------------------------
    #
    # --------------------------------------
    def canvasPressEvent(self, e):
        """"Manage selection when clicking on the canvas."""

        self.deleteHighlightGeoms()

        ##if e.button() != Qt.LeftButton:
        ##    return

        # get selected feature
        selFeature = self._select_feature_by_point( e.pos() )
        if selFeature is None:
            return

        # update feature geometry by diffeerence
        self._doDifference( selFeature )

    # --------------------------------------
    #
    # --------------------------------------
    def canvasMoveEvent(self, e):
        self.deleteHighlightGeoms()
        # get feature under cursor
        feature = self._select_feature_by_point( e.pos() )
        if feature is None:
            return
        # highlight feature geometry
        self.highlightGeom( feature.geometry() )

    # --------------------------------------
    #
    # --------------------------------------
    def canvasReleaseEvent(self, e):
        pass

    # --------------------------------------
    #
    # --------------------------------------
    def keyPressEvent(self, e):
        pass

    # --------------------------------------
    #
    # --------------------------------------
    def keyReleaseEvent(self, e):
        pass

    # --------------------------------------
    #
    # --------------------------------------
    def isZoomTool(self):
        return False

    # --------------------------------------
    #
    # --------------------------------------
    def isTransient(self):
        return False

    # --------------------------------------
    #
    # --------------------------------------
    def isEditTool(self):
        return True

    # --------------------------------------
    #
    # --------------------------------------
    def highlightGeom(self, geometry, color=None, fillColor=None, width=1):
        # init
        color = color or QColor( 255,0,0,255 )
        fillColor = fillColor or self.canvas.selectionColor() or QColor( 255,0,0,100 )
        fillColor.setAlpha(100)
        # create highlight object
        h = QgsHighlight( self.canvas, geometry, self.layer )
        h.setColor( color )
        h.setWidth( width )
        h.setFillColor( fillColor )
        self.highlightGeoms.append( h )

    # --------------------------------------
    #
    # --------------------------------------
    def deleteHighlightGeoms(self):
        for h in self.highlightGeoms:
            self.canvas.scene().removeItem( h )
        self.highlightGeoms = []

    # --------------------------------------
    #
    # --------------------------------------
    def _deselect_all(self):
        self.deleteHighlightGeoms()
        if self.iface.mapCanvas().isCachingEnabled():
            self.layer.triggerRepaint()
        else:
            self.iface.mapCanvas().refresh()

    # --------------------------------------
    #
    # --------------------------------------
    def _select_feature_by_point(self, po):
        """Get feature by point"""

        # init
        x = po.x()
        y = po.y()

        canvas = self.canvas
        coo_tranf = canvas.getCoordinateTransform()
        w = canvas.mapUnitsPerPixel() * 3
        rect = QgsRectangle(
            coo_tranf.toMapCoordinates(x-w, y-w),
            coo_tranf.toMapCoordinates(x+w, y+w)
        )
        rect.normalize()

        featRequest = QgsFeatureRequest()\
                             .setFilterRect( rect )\
                             .setFlags( QgsFeatureRequest.ExactIntersect | QgsFeatureRequest.NoGeometry )\
                             .setNoAttributes()

        ##if self.selectionFilter is not None:
        ##    featRequest.setFilterExpression( str( self.selectionFilter ) )

        # get features
        selFeature = None
        featuresIter = self.layer.getFeatures( featRequest )
        featuresId = [ feature.id() for feature in featuresIter ]
        featuresIter.close()
        featuresId.reverse()

        # filter not visible features by style
        renderer = self.layer.renderer().clone()
        ctx = QgsRenderContext()
        renderer.startRender( ctx, QgsFields() )

        for fid in featuresId:
            feature = self.layer.getFeature( fid )
            if renderer.willRenderFeature( feature, ctx ):
                selFeature = feature
                break

        renderer.stopRender( ctx )

        return selFeature

    # --------------------------------------
    #
    # --------------------------------------
    def _doDifference(self, inFeature):
        """ Update feature geometry by difference with others """

        # init
        inGeometry = QgsGeometry( inFeature.geometry() )

        # list all difference layers
        cutLayers = [ self.layer ]
        cutLayers.extend(x for x in self.suoliRefLayers if x not in cutLayers)

        # loop over all difference layers
        nFeatFound = 0
        for layer in cutLayers:
            # get features that insersect the input feature
            outFeats_ids = QGISAgriLayers.getFeaturesIdByGeom( layer,
                                                               inGeometry,
                                                               expression=None )

            # loop features for intersection\difference
            for fid in outFeats_ids:
                if fid == inFeature.id():
                    continue

                # get feature geometry
                feat = layer.getFeature( fid )
                featGeom = QgsGeometry( feat.geometry() )

                # check if there are intersections
                if not inGeometry.intersects( featGeom ):
                    continue

                # create difference feature
                inGeometry = inGeometry.difference( featGeom )
                if inGeometry.isEmpty():
                    logger.msgbar( logger.Level.Warning,
                           tr( "Il suolo selezionato tende ad annullarsi: controllare le sovrapposizioni di geometrie" ),
                           title=self.msgTitle )
                    return

                # increment counter
                nFeatFound += 1

        # check if difference found
        if nFeatFound == 0:
            return

        # start edit command
        self.layer.beginEditCommand( 'DifferenceSuoliTool' )
        try:
            # update feature geometry
            inFeature.setGeometry( inGeometry )

            # repair geometry
            QGISAgriLayers.repair_feature(self.layer,
                                           inFeature,
                                           attr_dict=None,
                                           splitMultiParts=True,
                                           autoSnap=False,
                                           suoliMinArea=self.suoliMinArea,
                                           suoliSnapTolerance=self.snapTolerance,
                                           suoliSnapLayerUris=None,
                                           addTopologicalPoints=True)

            # end edit command
            self.layer.endEditCommand()

        except Exception as e:
            self.layer.destroyEditCommand()
            raise e


import socket
import ssl
import json
import datetime

class TcpJson:
    def __init__(self, host, port, is_ssl):
        self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        if is_ssl :
            print(Ts() + "SSL connecting to " + host + ":" + str(port))
            self._sock = ssl.wrap_socket(self._sock)
        else :
            print("connecting to " + host + ":" + str(port))

        self._sock.connect((host,port))

    def __del__(self):
        self._sock.close()

    ## TX JSON with unsigned 4-bytes length header
    def Send(self, jsn):
        tx = json.dumps(jsn).encode('utf-8')            # serialize to utf-8 bytes
        txlen = len(tx)                                 # calculate payload length
        tx = txlen.to_bytes(4, byteorder = 'big') + tx  # prepend 32 bit length header
        print(Ts() + 'TXRAW  : ', end='')
        print(tx, flush=True)
        self._sock.send(tx)

    ## RX JSON with unsigned 4-bytes length header
    def Recv(self):
        data = self._sock.recv(4) # receive only 4 bytes first (32 bit length header)
        rxlen = int.from_bytes(data, byteorder='big') # convert to integer
        print(rxlen)
        rx = self._sock.recv(rxlen)      # receive a full json binary and decode to string
        jsn = json.loads(rx.decode())   # decode json to python object
        print(Ts() + 'RXRAW  : ', end='')
        print(data + rx)
        return jsn

def Ts():
    return '{:%Y-%m-%d %H:%M:%S} '.format(datetime.datetime.now())


import pickle
from random import randint

import torch
from peft import LoraConfig, TaskType, get_peft_model, prepare_model_for_kbit_training
from sklearn.metrics import precision_recall_fscore_support, accuracy_score, auc, precision_recall_curve
from transformers import TrainingArguments, AutoConfig, \
    AutoModelForSequenceClassification, AutoTokenizer, BitsAndBytesConfig, DataCollatorWithPadding

EPOCHS = 6 # TODO: CHANGE THIS
max_length = 4096
output_dir = f"change_me"  # TODO: CHANGE THIS
num_labels = 2
model_id = "meta-llama/Llama-2-13b-hf"
# model_id = "stanford-crfm/BioMedLM"

# MIMIC-IV DATA
# each pickle in the following format: [[[124325,
#   0,
#   ['Mood disorders',
#    'Diabetes mellitus with complications',
#    'Other circulatory disease'],
#   [1, 1, 1]]]
# For more details, see the documentation on GitHub.
train_pickle_file_path = '/sise/home/benshoho/projects/Med-BERT/Fine-Tunning-Tutorials/data/mimic-iv/chronic_kidney_disease_descriptions_train.pickle'  # TODO: CHANGE THIS
validation_pickle_file_path = '/sise/home/benshoho/projects/Med-BERT/Fine-Tunning-Tutorials/data/mimic-iv/chronic_kidney_disease_descriptions_validation.pickle'  # TODO: CHANGE THIS
test_pickle_file_path = '/sise/home/benshoho/projects/Med-BERT/Fine-Tunning-Tutorials/data/mimic-iv/chronic_kidney_disease_descriptions_test.pickle'  # TODO: CHANGE THIS

# set of descriptions for the mimic-iv dataset according the descriptions of the CCS categories.
mimic_iv_description_codes_path = '/sise/home/benshoho/projects/Med-BERT/Pretraining Code/Data Pre-processing Code/mimic_iv_descriptions_set.types'  # TODO: CHANGE THIS
with open(mimic_iv_description_codes_path, 'rb') as file:
    mimic_iv_description_codes = pickle.load(file)

def print_trainable_parameters(model):
    """
    Prints the number of trainable parameters in the model.
    """
    trainable_params = 0
    all_param = 0
    for _, param in model.named_parameters():
        all_param += param.numel()
        if param.requires_grad:
            trainable_params += param.numel()
    print(
        f"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}"
    )

# Load the tokenizer
tokenizer = AutoTokenizer.from_pretrained(model_id)
tokenizer.pad_token = tokenizer.eos_token

config = LoraConfig(
    r=32,
    lora_alpha=32,
    lora_dropout=0.1,
    bias="none",
    task_type=TaskType.SEQ_CLS
)

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)

model = AutoModelForSequenceClassification.from_pretrained(model_id, quantization_config=bnb_config,
                                                           config=AutoConfig.from_pretrained(model_id,
                                                                                trust_remote_code=True,
                                                                                num_labels=num_labels),
                                                           trust_remote_code=True)

model.gradient_checkpointing_enable()
model = prepare_model_for_kbit_training(model)

print_trainable_parameters(model)
model = get_peft_model(model, config)
print_trainable_parameters(model)

model.resize_token_embeddings(len(tokenizer))
model.config.pad_token_id = model.config.eos_token_id

def compute_metrics(p):
    p.predictions = torch.from_numpy(p.predictions)
    p.predictions = torch.softmax(p.predictions, dim=1)

    preds = np.argmax(p.predictions.cpu().numpy(), axis=1)
    labels = p.label_ids
    accuracy = accuracy_score(labels, preds)

    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average=None)
    precision1, recall1, thresholds = precision_recall_curve(labels, p.predictions[:, 1], pos_label=1)
    auc_precision_recall = auc(recall1, precision1)
    positive_confidences = p.predictions[:, 1]

    return {
        'accuracy': accuracy,
        'precision_class_0': precision[0],
        'precision_class_1': precision[1],
        'recall_class_0': recall[0],
        'recall_class_1': recall[1],
        'f1_class_0': f1[0],
        'f1_class_1': f1[1],
        'aucpr': auc_precision_recall,
        'pos_scores': positive_confidences.tolist(),
        'true_labels': labels.tolist(),
    }

import pickle
from datasets import Dataset

class CustomDataset:
    def __init__(self, pickle_file_path):
        with open(pickle_file_path, "rb") as file:
            data = pickle.load(file)

        # Create empty lists to store the dataset
        patient_ids = []
        labels = []
        diagnoses = []
        visits_num = []
        # Extract data from the loaded pickle
        for item in data:
            patient_ids.append(item[0])
            labels.append(item[1])
            diagnoses_list = item[2]
            diagnoses.append(diagnoses_list)
            visits_num.append(item[3])

        self.data_dict = {
            "patient_id": patient_ids,
            "classification_label": labels,
            "diagnoses": diagnoses,
            "visits_num": visits_num
        }

    def to_dict(self):
        return self.data_dict

dataset_type = 'mimic-iv' if 'mimic' in train_pickle_file_path else 'eicu-crd'
print(f'dataset_type={dataset_type}')

train_dataset = CustomDataset(train_pickle_file_path)
test_dataset = CustomDataset(test_pickle_file_path)
validation_dataset = CustomDataset(validation_pickle_file_path)

train_dict = train_dataset.to_dict()
test_dict = test_dataset.to_dict()
validation_dict = validation_dataset.to_dict()

train_dataset = Dataset.from_dict(train_dict)
test_dataset = Dataset.from_dict(test_dict)
validation_dataset = Dataset.from_dict(validation_dict)

def add_mimic_new_tokens_from_diagnosis_strings():
    new_tokens = set()
    for diagnosis_str in mimic_iv_description_codes:
        new_tokens.add(diagnosis_str)

    tokens_to_add = list(new_tokens - set(tokenizer.get_vocab()))
    prev_num_of_tokens = len(tokenizer)
    tokenizer.add_tokens(tokens_to_add)
    tokenizer.add_tokens(['empty_pad'], special_tokens=True)
    new_num_of_tokens = len(tokenizer)
    model.resize_token_embeddings(len(tokenizer))
    print(f'first 20 added tokens are= {list(new_tokens)[:20]}')
    print(f'prev tokens number= {prev_num_of_tokens}, new tokens number= {new_num_of_tokens}')

# If you want to use eicu-crd and add tokens to the vocab of pretrained tokenizer, use this:

# def add_eicu_crd_new_tokens_from_diagnosis_strings():
#     new_tokens = set()
#     for diagnosis_str in icd_code_to_icd_string_dict.values():
#         diagnoses_part_of_diagnosis_str = diagnosis_str.split('|')
#         # for words:
#         # for diagnoses_part in diagnoses_part_of_diagnosis_str:
#         #     new_tokens = new_tokens.union(diagnoses_part.split(" "))
#         for diagnoses_part in diagnoses_part_of_diagnosis_str:
#             new_tokens.add(diagnoses_part)
#     # consider to remove special tokens such ( etc..

#     tokens_to_add = list(new_tokens - set(tokenizer.get_vocab()))
#     prev_num_of_tokens = len(tokenizer)
#     tokenizer.add_tokens(tokens_to_add)
#     tokenizer.add_tokens(['empty_pad'], special_tokens=True)
#     new_num_of_tokens = len(tokenizer)
#     model.resize_token_embeddings(len(tokenizer))
#     print(f'first 20 added tokens are= {list(new_tokens)[:20]}')
#     print(f'prev tokens number= {prev_num_of_tokens}, new tokens number= {new_num_of_tokens}')

add_mimic_new_tokens_from_diagnosis_strings()
# add_eicu_crd_new_tokens_from_diagnosis_strings()

prompt_template = """
Your task is to determine whether a patient is likely to have a specific Chronic kidney disease based on their diagnosis descriptions provided below.
Each diagnosis description is separated by a comma.

**Patient Diagnosis Descriptions:**

{diagnoses}
"""

def template_dataset(sample):
    sample["text"] = prompt_template.format(diagnoses=sample["diagnoses"],
                                            eos_token=tokenizer.eos_token)
    return sample

# apply prompt template per sample
train_dataset = train_dataset.map(template_dataset)
print(f'example sample from train:\n {train_dataset[0]}')
validation_dataset = validation_dataset.map(template_dataset)

print(train_dataset[randint(0, len(train_dataset))]["diagnoses"])

# apply prompt template per sample
test_dataset = test_dataset.map(template_dataset)
lm_train_dataset = train_dataset.map(
    lambda sample: {
        "input_ids": tokenizer(sample["text"], truncation=True, max_length=max_length).input_ids,
        "attention_mask": tokenizer(sample["text"], truncation=True, max_length=max_length).attention_mask,
        "labels": sample["classification_label"]
    },
    batched=True,
    batch_size=64,
    remove_columns=list(train_dataset.features)
)
lm_validation_dataset = validation_dataset.map(
    lambda sample: {
        "input_ids": tokenizer(sample["text"], truncation=True, max_length=max_length).input_ids,
        "attention_mask": tokenizer(sample["text"], truncation=True, max_length=max_length).attention_mask,
        "labels": sample["classification_label"]
    },
    batched=True,
    batch_size=64,
    remove_columns=list(validation_dataset.features)
)
lm_test_dataset = test_dataset.map(
    lambda sample: {
        "input_ids": tokenizer(sample["text"], truncation=True, max_length=max_length).input_ids,
        "attention_mask": tokenizer(sample["text"], truncation=True, max_length=max_length).attention_mask,
        "labels": sample["classification_label"]
    },
    batched=True,
    batch_size=64,
    remove_columns=list(test_dataset.features)
)

from sklearn.utils.class_weight import compute_class_weight
import numpy as np

labels = lm_train_dataset['labels']

# Calculate class weights for an imbalanced dataset
class_weights = compute_class_weight('balanced', classes=np.unique(labels), y=labels)
class_weights = torch.tensor(class_weights, dtype=torch.float)
print(f'class_weights= {class_weights}')

print(f"Train dataset size: {len(train_dataset)}")
print(f"Val dataset size: {len(validation_dataset)}")
print(f"Test dataset size: {len(test_dataset)}")

from transformers import Trainer

training_args = TrainingArguments(
    output_dir=output_dir,
    evaluation_strategy='steps',
    save_strategy='steps',
    eval_steps=1000,
    save_steps=1000,
    num_train_epochs=EPOCHS,
    per_device_train_batch_size=8,
    per_device_eval_batch_size=4,
    auto_find_batch_size=True,
    logging_steps=30,
    learning_rate=2e-5,
    optim="adamw_torch",
    save_total_limit=20,
    logging_dir='./logs',
    load_best_model_at_end=True,
    metric_for_best_model="eval_aucpr",
    greater_is_better=True,
    dataloader_num_workers=8,
)

model.config.use_cache = False
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=lm_train_dataset,
    eval_dataset=lm_validation_dataset,
    compute_metrics=compute_metrics,
    data_collator=DataCollatorWithPadding(tokenizer=tokenizer)
)

trainer.train()
trainer.save_model(output_dir)

test_results = trainer.evaluate(eval_dataset=lm_test_dataset)

print(f'see outputs in= {output_dir}')
trainer.save_metrics("test", test_results)


import json
import sys

import spacy

import judicious
import proselint

nlp = spacy.load('en')
judicious.register("http://127.0.0.1:5000")

def error_str_to_json(err_string):
    return json.loads(
        proselint.tools.errors_to_json(
            proselint.tools.lint(err_string)
            )
        )

def sentence_context(text, position):
    """The sentence in the text that includes the given position."""
    doc = nlp(text)
    counter = 0
    for sentence in doc.sents:
        counter += len(sentence.text)
        if counter >= position:
            return sentence.text

def error_is_valid(text, error):
    """Call out to Judicious and figure out if error is valid."""
    return judicious.prosewash(
        text=sentence_context(text, error['start']),
        error_message=error['message'],
    )

def main(text=None):
    if text is None:
        with open(sys.argv[1], "r") as f:
            text = f.read()
    errors = error_str_to_json(text)['data']['errors']

    valid_errors = [error for error in errors
                    if error_is_valid(text, error)]
    json_output = error_str_to_json(text)
    json_output['data']['errors'] = valid_errors

    print(json_output)

if __name__ == "__main__":
    main()


import http

from django.contrib.auth.models import User
from django.shortcuts import render, redirect

from epe import settings
from .models import Classroom
from epe.utils import *
from django.core.exceptions import PermissionDenied, ObjectDoesNotExist

def list(request):
    if not request.user.is_authenticated:
        return redirect('%s?next=%s' % (settings.LOGIN_REDIRECT_URL, request.path))
    if request.user.is_staff:
        classrooms = Classroom.objects.all()
    else:
        classrooms = request.user.classrooms.all()
    return render(request, 'classroom_list.html', {'classrooms': classrooms})

def detail(request, id):
    if not request.user.is_authenticated:
        return redirect('%s?next=%s' % (settings.LOGIN_REDIRECT_URL, request.path))
    try:
        classroom = Classroom.objects.get(code=id)
    except ObjectDoesNotExist:
        return render(request, 'errors/404.html', {'error': 'El Aula no existe'})
    if not request.user.is_staff and request.user not in classroom.teachers.all():
        return render(request, 'errors/403.html', {'error': 'Usted no está esta acreditado como profesor en esta aula'})
    return render(request, 'classroom_detail.html', {'classroom': classroom, 'devices': classroom.devices.all().filter(token__isnull=False), 'documents': classroom.documents.all()})

def create(request):
    if not request.user.is_authenticated:
        return redirect('%s?next=%s' % (settings.LOGIN_REDIRECT_URL, request.path))
    if not request.user.is_staff:
        return render(request, 'errors/403.html', {'error': 'Usted no está acreditado para crear aulas'})
    classroom = Classroom()
    if request.POST:
        code = request.POST['code']
        name = request.POST['name']
        teachers = request.POST.getlist('teachers')
        classroom.code = code
        classroom.name = name
        if not code:
            return render(request, 'classroom_create.html', {'classroom': classroom, 'teachers': User.objects.all(), 'error': "Código del aula no introducido"})
        if not name:
            return render(request, 'classroom_create.html', {'classroom': classroom, 'teachers': User.objects.all(), 'error': "Nombre del aula no introducido"})
        if Classroom.objects.filter(code=code).exists():
            return render(request, 'classroom_create.html', {'classroom': classroom, 'teachers': User.objects.all(), 'error': "Código del aula ya existe"})
        classroom.save()
        for teacher in teachers:
            classroom.teachers.add(User.objects.get(id=teacher))
        request.session['info'] = "Aula agregada"
        return render(request, "classroom_detail.html", {'classroom': classroom, 'info': "Aula agregada"})
    return render(request, 'classroom_create.html', {'teachers': User.objects.all()})

def update(request, id):
    if not request.user.is_authenticated:
        return redirect('%s?next=%s' % (settings.LOGIN_REDIRECT_URL, request.path))
    if not request.user.is_staff:
        return render(request, 'errors/403.html', {'error': 'Usted no está esta acreditado para modificar aulas'})
    classroom = Classroom.objects.get(code=id)
    if request.POST:
        name = request.POST['name']
        teachers = request.POST.getlist('teachers')
        classroom.teachers.clear()
        for teacher in teachers:
            classroom.teachers.add(User.objects.get(id=teacher))
        if not name:
            return render(request, 'classroom_update.html', {'classroom': classroom, 'teachers': User.objects.all(), 'error': "Nombre del aula no introducido"})
        classroom.name = name
        classroom.save()
        return render(request, 'classroom_detail.html', {'classroom': classroom, 'teachers': User.objects.all(), 'info': "Aula " + classroom.name + " actualizada correctamente"})
    return render(request, 'classroom_update.html', {'classroom': classroom, 'teachers': User.objects.all()})

def delete(request, id):
    if not request.user.is_authenticated:
        return redirect('%s?next=%s' % (settings.LOGIN_REDIRECT_URL, request.path))
    if not request.user.is_staff:
        return render(request, 'errors/403.html', {'error': 'No esta acreditado para borrar aulas'})
    classroom = Classroom.objects.get(code=id)
    if classroom.devices.all().filter(token__isnull=False).count() > 0:
        return render(request, 'classroom_detail.html', {'classroom': classroom, 'devices': classroom.devices.all().filter(token__isnull=False), 'error': "El aula " + classroom.name + " tiene dispositivos asociados"})
    classroom.delete()
    classrooms = Classroom.objects.all()
    return render(request, 'classroom_list.html', {'classrooms': classrooms, 'info': "Aula " + classroom.name + " borrada"})


import numpy
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot
pyplot.rcParams.update({'font.size':40})
from matplotlib.colors import LogNorm
from velociraptor import load
import unyt
from velociraptor.tools.lines import binned_median_line as bml
import hist

cat = load('/cosma6/data/dp004/dc-borr1/snap7/new_randomness_runs/adiabatic/Run_0/halo_0007.properties')
distance = unyt.unyt_array(numpy.loadtxt('/cosma5/data/durham/dc-murr1/adi_gas_nearest_halo_distance.txt'), units = 'Mpc')
spread = unyt.unyt_array(numpy.loadtxt('/cosma5/data/durham/dc-murr1/adiabatic_gas_spread.txt'), units = 'Mpc')
radius = unyt.unyt_array(numpy.loadtxt('/cosma5/data/durham/dc-murr1/adi_gas_nearest_halo_radius.txt'), units = 'Mpc')
#frac_spread = spread / radius
#frac_distance = distance / radius
dm_halos = numpy.loadtxt('/cosma5/data/durham/dc-murr1/adi_gas_neighbour_halos.txt').astype(int)
mass = unyt.unyt_array(numpy.loadtxt('/cosma5/data/durham/dc-murr1/adi_gas_nearest_halo_mass.txt'), units = 'msun')

where = numpy.where(dm_halos >= 0)[0]
spread = spread[where]
distance = distance[where]
mass = mass[where]

hist.plot_hist(distance,
            spread,
            'Distance to nearest halo [Mpc]',
            'Spread metric [Mpc]',
            '/cosma5/data/durham/dc-murr1/adi_gas_neighbour_spread_distance.png')

one = numpy.where(numpy.logical_and(numpy.amin(mass) < mass, mass < numpy.percentile(mass, 25)))[0]
two = numpy.where(numpy.logical_and(numpy.percentile(mass, 25) < mass, mass < numpy.percentile(mass, 50)))[0]
three = numpy.where(numpy.logical_and(numpy.percentile(mass, 50) < mass, mass < numpy.percentile(mass, 75)))[0]
four = numpy.where(numpy.logical_and(numpy.percentile(mass, 75) < mass, mass < numpy.amax(mass)))[0]

spread_one = spread[one]
distance_one = distance[one]
spread_two = spread[two]
distance_two = distance[two]
spread_three = spread[three]
distance_three = distance[three]
spread_four = spread[four]
distance_four = distance[four]

hist.plot_hist(
    distance_one,
    spread_one,
    'Distance to nearest halo [Mpc]',
    'Spread metric [Mpc]',
    '/cosma5/data/durham/dc-murr1/adi_gas_neighbour_spread_distance_0_25.png'
)

hist.plot_hist(
    distance_two,
    spread_two,
    'Distance to nearest halo [Mpc]',
    'Spread metric [Mpc]',
    '/cosma5/data/durham/dc-murr1/adi_gas_neighbour_spread_distance_25_50.png'
)

hist.plot_hist(
    distance_three,
    spread_three,
    'Distance to nearest halo [Mpc]',
    'Spread metric [Mpc]',
    '/cosma5/data/durham/dc-murr1/adi_gas_neighbour_spread_distance_50_75.png'
)

hist.plot_hist(
    distance_four,
    spread_four,
    'Distance to nearest halo [Mpc]',
    'Spread metric [Mpc]',
    '/cosma5/data/durham/dc-murr1/adi_gas_neighbour_spread_distance_75_100.png'
)


from sklearn.linear_model import LinearRegression
import os
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
# regressor = LinearRegression()
# regressor.fit(X,y)

# y_pred = regressor.predict(X_next)

# para cada csv de stats na pasta de extractions faz uma regressão linear para as colunas Media, Min e Max

for i, file in enumerate(os.listdir('data/extractions')):
    # verifica se o arquivo começa com stats
    if file.startswith('stats'):
        df = pd.read_csv('data/extractions/' + file)
        # pega a esperecialidade
        esp = file.split('_')[1].split(".")[0]

        # faz a regressão linear para cada coluna
        media = None
        minimo = None
        maximo = None

        for col in ['Media', 'Min', 'Max']:
            # pega os valores de X e y
            X = df['Ano'].values
            y = df[col].values

            # cria um novo regressor
            regressor = LinearRegression()
            # treina o regressor
            regressor.fit(X.reshape(-1, 1), y)

            # pega o valor de X para o próximo ano
            X_next = np.array([2024])
            # faz a predição
            y_pred = regressor.predict(X_next.reshape(-1, 1))
            # printa o resultado
            print(f'{esp} {col} {y_pred[0]}')
            if col == 'Media':
                media = y_pred[0]
            elif col == 'Min':
                minimo = y_pred[0]
            elif col == 'Max':
                maximo = y_pred[0]

                # inclui a predição no dataframe
        new_row = {'Ano': 2024, "Media": media, "Min": minimo, "Max": maximo}
        df = df.append(new_row, ignore_index=True)
        # salva o dataframe
        df.to_csv('data/extractions/regression_stats_' +
                  esp + ".csv", index=False)

        df['Ano'] = df['Ano'].astype(int)
        df.plot(x='Ano', y=['Media', 'Max', 'Min'])

        #  faz o passo do eixo x ser 1
        plt.xticks(np.arange(df['Ano'].min(), df['Ano'].max()+1, 1.0))

        plt.savefig(f"data/charts/regression_{esp}.png")

        plt.clf()


from typing import List

class Solution:
    def maximalRectangle(self, matrix: List[List[str]]) -> int:
        if len(matrix) == 0:
            return 0

        ans = 0
        rows = len(matrix)
        cols = len(matrix[0])
        heights = [0] * (cols + 1)

        for i in range(rows):
            for j in range(cols):
                heights[j] = (heights[j] + 1) * int(matrix[i][j])

            # largest rectangle in histogram
            stack = [-1]
            for j in range(cols + 1):
                while heights[stack[-1]] > heights[j]:
                    width = j - stack[-2] - 1
                    height = heights[stack[-1]]
                    ans = max(ans, height * width)
                    stack.pop()

                stack.append(j)

        return ans


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from os import path

import logging
import sqlite3
from datetime import date, datetime

# from thewave.marketdata.tickerlist import TickerList
import numpy as np
import pandas as pd

DATABASE_DIR = path.realpath(__file__).\
    replace('rewave/marketdata/stooq.pyc','/marketdata/Stooq.db').\
    replace("rewave\\marketdata\\stooq.pyc","marketdata\\Stooq.db").\
    replace('rewave/marketdata/stooq.py','/marketdata/Stooq.db').\
    replace("rewave\\marketdata\\stooq.py","marketdata\\Stooq.db")

class StooqToDB:

    def __init__(self, root = "/", frequency='daily'):
        self.initialize_db()

        self.root = root

        self.frequency = frequency

    def initialize_db(self):
        with sqlite3.connect(DATABASE_DIR) as connection:
            cursor = connection.cursor()
            #cursor.execute('CREATE TABLE IF NOT EXISTS History (ticker varchar(20), date date, open FLOAT, high FLOAT, low FLOAT, close FLOAT, volume FLOAT, ex_dividend FLOAT,adj_open FLOAT, adj_high FLOAT, adj_low FLOAT, adj_close FLOAT, adj_volume FLOAT, PRIMARY KEY (ticker, date));')

            EXCHANGE_CREATE_STATEMENT = ('CREATE TABLE IF NOT EXISTS exchange (id INTEGER PRIMARY KEY AUTOINCREMENT,'
                                          'abbrev VARCHAR(32) NOT NULL,'
                                          'name VARCHAR(255) NOT NULL,'
                                          'city VARCHAR(255) NOT NULL,'
                                          'country VARCHAR(255) NOT NULL,'
                                          'currency VARCHAR(32) NOT NULL,'
                                          'timezone_offset time NOT NULL,'
                                          'created_date date_time NOT NULL,'
                                          'last_updated_date VARCHAR(255) NOT NULL);')

            DATA_VENDOR_CREATE_STATEMENT = ('CREATE TABLE IF NOT EXISTS data_vendor (id INTEGER PRIMARY KEY AUTOINCREMENT,'
                                         'name VARCHAR(64) NOT NULL,'
                                         'web_site_url VARCHAR(255) NOT NULL,'
                                         'support_email VARCHAR(255) NOT NULL,'
                                         'created_date date_time NOT NULL,'
                                         'last_updated_date VARCHAR(255) NOT NULL);')

            SYMBOL_CREATE_STATEMENT = ('CREATE TABLE IF NOT EXISTS symbol (id INTEGER PRIMARY KEY AUTOINCREMENT,'
                                        'exchange_id INTEGER,'
                                        'ticker VARCHAR(32) NOT NULL,'
                                        'instrument VARCHAR(64),'
                                        'name VARCHAR(255) NOT NULL,'
                                        'sector VARCHAR(255),'
                                        'currency varchar(32),'
                                        'created_date date_time NOT NULL,'
                                        'last_updated_date VARCHAR(255) NOT NULL,'
                                        'FOREIGN KEY(exchange_id) REFERENCES exchange(id) );')

            DAILY_PRICE_CREATE_STATEMENT = ('CREATE TABLE IF NOT EXISTS daily_price (id INTEGER PRIMARY KEY AUTOINCREMENT,'
                                           'data_vendor_id INTEGER,'
                                           'symbol_id INTEGER,'
                                           'price_date DATETIME NOT NULL,'
                                           'open FLOAT,'
                                           'high FLOAT,'
                                           'low FLOAT,'
                                           'close FLOAT,'
                                           'volume FLOAT,'
                                           'ex_dividend FLOAT,'
                                           'split_ratio FLOAT,'
                                           'adj_open FLOAT,'
                                           'adj_high FLOAT,'
                                           'adj_low FLOAT,'
                                           'adj_close FLOAT,'
                                           'adj_volume FLOAT,'
                                           'created_date date_time NOT NULL,'
                                           'last_updated_date VARCHAR(255) NOT NULL,'
                                           'FOREIGN KEY(data_vendor_id) REFERENCES data_vendor(id),'
                                           'FOREIGN KEY(symbol_id) REFERENCES symbol(id) );')

            cursor.execute(EXCHANGE_CREATE_STATEMENT)
            cursor.execute(DATA_VENDOR_CREATE_STATEMENT)
            cursor.execute(SYMBOL_CREATE_STATEMENT)
            cursor.execute(DAILY_PRICE_CREATE_STATEMENT)
            connection.commit()

    def set_ticker_id(self, ticker):
        connection = sqlite3.connect(DATABASE_DIR)

    def get_ticker_id(self, ticker):
        connection = sqlite3.connect(DATABASE_DIR)

    def update_ticker(self, ticker):
        connection = sqlite3.connect(DATABASE_DIR)
        logging.info("DB Update for ticker: %s " % (ticker))

        min_date = None
        max_date = None
        try:
            cursor = connection.cursor()
            min_date = cursor.execute('SELECT MIN(date) FROM daily_price WHERE ticker=?;', (ticker,)).fetchall()[0][0]
            max_date = cursor.execute('SELECT MAX(date) FROM daily_price WHERE ticker=?;', (ticker,)).fetchall()[0][0]
        finally:
            connection.commit()

        try:
            cursor = connection.cursor()
            #min_date = cursor.execute('SELECT MIN(date) FROM History WHERE ticker=?;', (ticker,)).fetchall()[0][0]
            #max_date = cursor.execute('SELECT MAX(date) FROM History WHERE ticker=?;', (ticker,)).fetchall()[0][0]

            if min_date==None or max_date==None:
                logging.info("DB REQUEST CREATION for ticker: %s " % (ticker))
                self.fill_ticker(ticker, cursor)
            else:
                now_ts = pd.Timestamp(datetime.now())
                max_ts = pd.Timestamp(max_date)
                if (max_date == None) or (now_ts - max_ts).days >1:
                    logging.info("DB REQUEST UPDATE for ticker: %s from: %s" % (ticker, max_date))
                    ticker_data = self._quandl_request.data(ticker, {'start_date': max_date})
                    for tick in ticker_data:
                        cursor.execute('INSERT OR IGNORE INTO History (date, ticker, open, high, low, close, '
                                       'volume, ex_dividend, '
                                       'adj_open, adj_high, adj_low, adj_close, adj_volume) '
                                       'VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)',
                                       (tick[0], ticker, tick[1], tick[2], tick[3], tick[4],
                                        tick[5], tick[6],
                                        tick[8], tick[9], tick[10], tick[11], tick[12]))

            # if there is no data
        finally:
            connection.commit()
            connection.close()

if __name__ == "__main__":
    StooqToDB()


#!/usr/bin/python3
""" ... """

def load_from_json_file(filename):
    """  creates an Object from a “JSON file”: """
    with open(filename, mode="r", encoding="utf-8") as file:
        import json
        my_object = json.load(file)
        file.close()
        return my_object