try:
    import vrep
except:
    print ('--------------------------------------------------------------')
    print ('"vrep.py" could not be imported. This means very probably that')
    print ('either "vrep.py" or the remoteApi library could not be found.')
    print ('Make sure both are in the same folder as this file,')
    print ('or appropriately adjust the file "vrep.py"')
    print ('--------------------------------------------------------------')
    print ('')

import time
import numpy as np
import params
import yaml
import argparse
from   utilities import *
from math import sqrt

global angErrHist
global distErrHist

angErrHist = []
distErrHist = []

print ('Program started')

# 3. Routine to get the Distance and angular errors for the go-to-goal PID controller:
def g2g_xy_errors(clientID, pioneerHandle, goal):

    # a: Stream current position and orientation:
    res , pioneerPosition = vrep.simxGetObjectPosition(clientID, pioneerHandle, -1 , vrep.simx_opmode_buffer)
    res , pioneerOrientation = vrep.simxGetObjectOrientation(clientID, pioneerHandle, -1 , vrep.simx_opmode_buffer)

    xp = pioneerPosition[0]
    yp = pioneerPosition[1]
    thetap = pioneerOrientation[2]

    # # VREP orientation returns a value between -pi to +pi. Trnasform this to the range o to 2*pi
    # if thetap < 0:
    #
    #     thetap = 2*np.pi + thetap
    #
    # print ("Thetap -> " , thetap)

    xg = goal[0]
    yg = goal[1]

    diff_x = xg - xp
    diff_y = yg - yp

    # b. Compute orientation error:
    targetOrientation = np.arctan2( diff_y , diff_x)

    # # Arctan2 returns a value between -pi to +pi. Trnasform this to the range o to 2*pi
    # if targetOrientation < 0:
    #
    #     targetOrientation = 2*np.pi + targetOrientation
    #
    # print ("Target -> " , thetap)

    errAng = targetOrientation - thetap

    # Restrict to -pi and +pi:
    errAng = np.mod((errAng+ np.pi), 2*np.pi) - np.pi

    if abs(errAng) < params.angErrThresh:

        reachedOrientation = True
    else:
        reachedOrientation = False

    # c. Compute distance error:
    errDist = np.sqrt( (xg - xp)**2 + (yg - yp)**2)

    if errDist < params.distErrThresh:

        reachedGoal = True
    else:

        reachedGoal = False

    # Return the values:
    return errAng, errDist, reachedOrientation , reachedGoal

# 4. Function to map unicycle commands to differential drive commands:
def diff_drive_map(v,omega, trackWidth,wheelRadius):

    # TO DO: Document the formulas in detail here:

    # limit the meximun velocity
    meximumVelocity = 6

    # 1. Left wheel angular velocity:
    omegaLeft = (v + 0.5*trackWidth*omega)/wheelRadius
    if omegaLeft <= meximumVelocity:
        omegaLeft = (v + 0.5*trackWidth*omega)/wheelRadius
        # omegaLeft = sqrt(x**2+y**2)/t
    else:
        omegaLeft = meximumVelocity

    # 2. Right wheel angular velocity:
    omegaRight = (v - 0.5*trackWidth*omega)/wheelRadius
    if omegaRight <= meximumVelocity:
        omegaRight = (v - 0.5*trackWidth*omega)/wheelRadius
        # omegaRight = sqrt(x**2+y**2)/t
    else:
        omegaRight = meximumVelocity

    return omegaLeft , omegaRight

# 5. Go to Goal PID control from a 2-D differential drive robot:
def g2g_controller_PID(clientID, pioneerHandle, pioneerLeftMotorHandle, pioneerRightMotorHandle, goal):

    global angErrHist , distErrHist

    # Set the sleep time in While function
    sleepTime = 0.2

    # a. Setup internal parameters of the PID controller:
    reachedGoal = False
    reachedOrientation = False
    wheelRadius = 0.0975   # meters - From VREP model of left wheel
    trackWidth = 0.4       # meters - This is approximate
    v= 0
    omega = 0

    timer = 0

    # Continue this as long as the Robot does not reach its goal:
    while not reachedGoal:

        errAng, errDist, reachedOrientation , reachedGoal = g2g_xy_errors(clientID, pioneerHandle, goal)

        # print (" Time = " , timer , " sec" , end= " -> ")
        # print ("Angular Error =  " , errAng, end= " -> ")
        # print (" Distance Error = " , errDist)

        # Append error values to the cache variabes:
        angErrHist.append(errAng)
        distErrHist.append(errDist)

        # Orientation correction:
        if not reachedOrientation:

            # PID control law for Orientation:
            v = 0

            if len(angErrHist) > params.integralWindow:
                omega = (-params.kpTheta*errAng - params.kdTheta*(errAng - angErrHist[-2]) - params.kiTheta*sum(angErrHist[-params.integralWindow:]))

            elif len(angErrHist) > 2:

                omega = (-params.kpTheta*errAng - params.kdTheta*(errAng - angErrHist[-2]) - params.kiTheta*sum(angErrHist))

            else:

                omega =  (-params.kpTheta*errAng)

            omegaLeft , omegaRight  = diff_drive_map(v,omega,trackWidth,wheelRadius)

            res = vrep.simxSetJointTargetVelocity(clientID, pioneerLeftMotorHandle, omegaLeft, vrep.simx_opmode_oneshot_wait)
            res = vrep.simxSetJointTargetVelocity(clientID, pioneerRightMotorHandle, omegaRight, vrep.simx_opmode_oneshot_wait)

        # Distance correction:
        else:

            # Clear the angular error history cache:
            angErrHist = []

            omega = 0.0

            if  len(distErrHist) > params.integralWindow:
                v = 8*(params.kpDist*errDist + params.kdDist*(errDist - distErrHist[-2]) + params.kiDist*sum(distErrHist[-params.integralWindow:]))

            elif len(distErrHist) > 2:

                v = 8*(params.kpDist*errDist + params.kdDist*(errDist - distErrHist[-2]) + params.kiDist*sum(distErrHist))

            else:
                v = 8*(params.kpDist*errDist)

            omegaLeft , omegaRight  = diff_drive_map(v,omega,trackWidth, wheelRadius)

            res = vrep.simxSetJointTargetVelocity(clientID, pioneerLeftMotorHandle, omegaLeft, vrep.simx_opmode_oneshot_wait)
            res = vrep.simxSetJointTargetVelocity(clientID, pioneerRightMotorHandle, omegaRight, vrep.simx_opmode_oneshot_wait)

        # Set the frequency of the controller:
        time.sleep(sleepTime)

    # Notify the user:
    #print(" Robot reached its Goal!")

    # Clear the error history caches:
    angErrHist = []
    distErrHist = []

    v= v/2
    omega = 0

    omegaLeft , omegaRight  = diff_drive_map(v,omega,trackWidth,wheelRadius)

    # Deactivate joint actuations:
    res = vrep.simxSetJointTargetVelocity(clientID, pioneerLeftMotorHandle, omegaLeft, vrep.simx_opmode_oneshot_wait)
    res = vrep.simxSetJointTargetVelocity(clientID, pioneerRightMotorHandle, omegaRight, vrep.simx_opmode_oneshot_wait)

    return reachedGoal

# ---------------------------------- MAIN SCRIPT BEGINS HERE -----------------------------------------------------------------------------#

vrep.simxFinish(-1) # just in case, close all opened connections
clientID=vrep.simxStart('127.0.0.1',19997,True,True,5000,5) # Connect to V-REP
if clientID!=-1:

    print ('Connected to remote API server')

    # 2.a: Handle the Pioneer:
    res , pioneerHandle = vrep.simxGetObjectHandle(clientID, "Pioneer_p3dx", vrep.simx_opmode_blocking)
    res,  pioneerLeftMotorHandle = vrep.simxGetObjectHandle(clientID, "left", vrep.simx_opmode_blocking)
    res,  pioneerRightMotorHandle = vrep.simxGetObjectHandle(clientID, "right", vrep.simx_opmode_blocking)

    # Get the position of the Pioneer for the first time in streaming mode: Still have to understand why this is done!
    res , pioneerPosition = vrep.simxGetObjectPosition(clientID, pioneerHandle, -1 , vrep.simx_opmode_streaming)
    res , pioneerOrientation = vrep.simxGetObjectOrientation(clientID, pioneerHandle, -1 , vrep.simx_opmode_streaming)

    # Deactivate joint actuations:Make sure Pioneer is stationary:
    res = vrep.simxSetJointTargetVelocity(clientID, pioneerLeftMotorHandle, 0, vrep.simx_opmode_streaming)
    res = vrep.simxSetJointTargetVelocity(clientID, pioneerRightMotorHandle, 0, vrep.simx_opmode_streaming)

    ### Step 3: Start the Simulation: Keep printing out status messages!!!
    res = vrep.simxStartSimulation(clientID, vrep.simx_opmode_oneshot_wait)

    if res == vrep.simx_return_ok:

        print ("---!!! Started Simulation !!! ---")

    ### Step 3: Perception:  TO BE DONE:

    # # First call:
    # res,resolution,image=vrep.simxGetVisionSensorImage(clientID, quadCameraHandle,0,vrep.simx_opmode_streaming)
    # print ('firts call',res)
    # time.sleep(5)
    # res,resolution,image=vrep.simxGetVisionSensorImage(clientID, quadCameraHandle,0,vrep.simx_opmode_buffer)
    # print ('second call',res)
    #
    # if res == vrep.simx_return_ok:
    #
    #     print (" successful first image!")
    #
    # time.sleep(5)
    #
    # while (vrep.simxGetConnectionId(clientID) != -1):
    #
    #      res, resolution, image = vrep.simxGetVisionSensorImage(clientID, quadCameraHandle, 0, vrep.simx_opmode_buffer)
    #
    #      print (res)
    #
    #      if res == vrep.simx_return_ok:
    #          print (len(image))

    # # 3. Setup for vectorization:
    # setup_vec()

    # 4. Run the RRT algorithm and directly plot the path:
    startTime = time.time()
    # RRT_path = RRT(start, goal)
    endTime = time.time()

    # Paht for robots
    revPath = []
    output_time=[]
    scale = 1

    # 5. Get the true path from start to goal:
    with open("output.yaml", 'r') as output_file:
        data = yaml.load(output_file, Loader=yaml.FullLoader)
        agent1 = data['agent1']
        agent0 = data['agent0']

    # set agent
    for element in agent0:
        x = element['x'] * scale
        y = element['y'] * scale
        t = element['t']
        coordinates = [x, y]
        element_time = [t]
        output_time.append(element_time)
        revPath.append(coordinates)

    # revPath = reverse_path(RRT_path)
    # revPath = [[10.0,0.0],[10.0, -10.0],[6,-4],[8,7],[-4,-4.0],[0,0]]

    timeRRT= endTime - startTime

    print (" Time taken time is : " , timeRRT ,  "secs")
    print (" The RRT path is : ")
    print (revPath)
    # ----------------------------------------------------- #

    # Start a timer:
    sTime = time.time()

    pathPointCount = 0

    # Loop to traverse RRT Path:
    for immGoal in revPath:

        # Run the Go-to-goal control function:
        reached = g2g_controller_PID(clientID, pioneerHandle, pioneerLeftMotorHandle, pioneerRightMotorHandle, immGoal)

        if reached:

            # Increment the path point count:
            pathPointCount +=1

            print (" Reached node # " , pathPointCount )
            print (" ---------------------------------")

        else:

            print(" There was an ERROR! Please investigate!!!")

    eTime = time.time()

    travelTime = eTime - sTime

    # Display successful path planning message!:
    print ("--- Robot has Successfully reached its final Goal --- ")
    print (" The toal time taken is : " , travelTime , " secs")
    print ("The robot position is : ", pioneerPosition)

    # Set the joint velocities back to zero:
    res = vrep.simxSetJointTargetVelocity(clientID, pioneerLeftMotorHandle, 0, vrep.simx_opmode_streaming)
    res = vrep.simxSetJointTargetVelocity(clientID, pioneerRightMotorHandle, 0, vrep.simx_opmode_streaming)

    # Stop the simulation:
    res = vrep.simxStopSimulation(clientID, vrep.simx_opmode_oneshot_wait)

    if res == vrep.simx_return_ok:

        print (" --- !!!Stopped Simulation!!! ---")

    # Now close the connection to V-REP:
    vrep.simxFinish(clientID)

else:

    print ('Failed connecting to remote API server')

print (' --- Python Remote API Program ended ---')


#! /usr/bin/env python3

import random
import os
import sys
import tensorflow as tf
from tensorflow import keras
import numpy as np
import matplotlib.pyplot as plt

def plot_image(i, predictions_array, true_label, img):
  predictions_array, true_label, img = predictions_array, true_label[i], img[i]
  plt.grid(False)
  plt.xticks([])
  plt.yticks([])

  plt.imshow(img, cmap=plt.cm.binary)

  predicted_label = np.argmax(predictions_array)
  if predicted_label == true_label:
    color = 'blue'
  else:
    color = 'red'

  plt.xlabel("test {} {:2.0f}% ({})".format(i,
                                100*np.max(predictions_array),
                                class_names[true_label]),
                                color=color)

def plot_value_array(i, predictions_array, true_label):
  predictions_array, true_label = predictions_array, true_label[i]
  plt.grid(False)
  plt.xticks(range(10))
  plt.yticks([])
  thisplot = plt.bar(range(10), predictions_array, color="#777777")
  plt.ylim([0, 1])
  predicted_label = np.argmax(predictions_array)

  thisplot[predicted_label].set_color('red')
  thisplot[true_label].set_color('blue')

print('tensorflow version is', tf.__version__)

# the module gives easy access to the test data
# first run this will download to a local file
# subsequent runs it will be much quicker
#fashion_mnist = keras.datasets.fashion_mnist
#(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()
#class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
#               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

number_mnist = keras.datasets.mnist
(train_images, train_labels), (test_images, test_labels) =  number_mnist.load_data()
class_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']

# see what the data looks like
show_data = (len(sys.argv) > 1)
if show_data:
   plt.figure(figsize=(10,10))
   for i in range(25):
       plt.subplot(5,5,i+1)
       plt.xticks([])
       plt.yticks([])
       plt.grid(False)
       plt.imshow(train_images[i], cmap=plt.cm.binary)
       plt.xlabel(class_names[train_labels[i]])
   plt.show()

train_images = train_images / 255.0
test_images  = test_images  / 255.0

if os.path.exists('model.json') and os.path.exists('model.h5'):
   with open('model.json') as file:
      json = file.read()
   model = keras.models.model_from_json(json)
   model.load_weights('model.h5')
   print('loaded model from disk')

   model.compile(optimizer='adam',
                 loss='sparse_categorical_crossentropy',
                 metrics=['accuracy'])
else:
   model = keras.Sequential([
       keras.layers.Flatten(input_shape=(28, 28)),
       keras.layers.Dense(128, activation='relu'),
       keras.layers.Dense(10, activation='softmax')
   ])

   model.compile(optimizer='adam',
                 loss='sparse_categorical_crossentropy',
                 metrics=['accuracy'])

   # this is the training step that takes all the time
   model.fit(train_images, train_labels, epochs=5)

   with open('model.json', 'w') as file:
      file.write(model.to_json())
   model.save_weights('model.h5')
   print('saved model to files model.json and model.h5')

# either way now we have a model with weights; write them out to csv
i = 0
for a in model.get_weights():
   fname = 'model.weights'+str(i)+'.csv'
   i += 1
   with open(fname, 'w') as csv:
      if len(a.shape)==2:
         rows, cols = a.shape
         for r in range(rows):
            for c in range(cols):
               csv.write(str(a[r][c]) + ',')
            csv.write("\n")
      else:
         rows = a.shape[0]
         for r in range(rows):
            csv.write(str(a[r]) + "\n")

test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)
print('\nTest loss:    ', test_loss)
print('Test accuracy:', test_acc)

# this applies the model to all the test images
predictions = model.predict(test_images)

num_rows = 10
num_cols = 6
num_images = num_rows*num_cols
plt.figure(figsize=(2*2*num_cols, 2*num_rows))
for ii in range(num_images):
  i = random.randint(0,len(test_images)-1)
  plt.subplot(num_rows, 2*num_cols, 2*ii+1)
  plot_image(i, predictions[i], test_labels, test_images)
  plt.subplot(num_rows, 2*num_cols, 2*ii+2)
  plot_value_array(i, predictions[i], test_labels)
plt.tight_layout()
plt.show()


import cv2 as cv
import numpy as np

# reading input image
img = cv.imread('D:/DIP/Assignment_1/a1_2019701006/input_data/quantize.jpg')

# function for Quantized Image
def BitQuantizeImage (img,k):
    g = np.array((np.floor(img/(2**(8-k))) * 255/(2**k-1)),dtype='uint8')
    return g

g=BitQuantizeImage (img,3)
cv.imshow('Quantized Image',g)
cv.waitKey(0)


from vector import Vector

"""Using the Vector Operations to represent a colour, stored in RGB dimentions"""

class Colour(Vector):

    @classmethod
    def from_hex(cls, hexcolour="#000000"):
        r = int(hexcolour[1:3], 16) / 255.0
        g = int(hexcolour[3:5], 16) / 255.0
        b = int(hexcolour[5:7], 16) / 255.0
        return cls(r, g, b)


"Niftools Vertex Correction"
"Dirty script to duplicate a nif and overwrite its vert colors"

import site
import bpy
from pyffi.formats.nif import NifFormat

class NifDataHolder:
    """ A python singleton """

    class __impl:
        """ Implementation of the singleton interface """
        def __init__(self):
            self.data = null

        def getinstancedata(self):
            return self.data

        def setinstancedata(self,data):
            self.data = data

    # storage for the instance reference
    __instance = None

    def __init__(self):
        """ Create singleton instance """
        # Check whether we already have an instance
        if Singleton.__instance is None:
            # Create and remember instance
            Singleton.__instance = Singleton.__impl()

        # Store instance reference as the only member in the handle
        self.__dict__['_Singleton__instance'] = Singleton.__instance

    def __getattr__(self, attr):
        """ Delegate access to implementation """
        return getattr(self.__instance, attr)

    def __setattr__(self, attr, value):
        """ Delegate access to implementation """
        return setattr(self.__instance, attr, value)

class NifReader(bpy.types.Operator):
    "This is a dirty script used to change vert color"

    '''Bevels selected edges of the mesh'''
    bl_idname = "vertexcolor.nifread"
    bl_label = "Duplicate mesh and overwrite the vert color"

    def execute(self, context):
        filein = bpy.context.scene.vertexcolor.fileinput
        hexvalue = bpy.context.scene.vertexcolor.hexwidget

        # open file for binary reading
        print("Importing %s" % filein)
        niffile = open(filein, "rb")
        self.data = NifFormat.Data()

        # check if nif file is valid
        self.data.inspect(niffile)
        if self.data.version >= 0:
            # it is valid, so read the file
            print("NIF file version: 0x%08X" % self.data.version)
            print("Reading file")
            self.data.read(niffile)
        elif self.data.version == -1:
            print("Unsupported NIF version.")
        else:
            print("Not a NIF file.")

        for root in self.data.roots:
            for block in root.tree():
                if isinstance(block, NifFormat.NiTriShape):
                    if(block.data.has_vertex_colors):
                        vertexcol = block.data.vertex_colors[0]
                        bpy.context.scene.vertexcolor.hexwidget = [vertexcol._items[0]._value,
                                                                   vertexcol._items[1]._value,
                                                                   vertexcol._items[2]._value]

        if(bpy.context.scene.vertexcolor.fileoutput == ""):
            print("check run ok")
            input = bpy.context.scene.vertexcolor.fileinput
            bpy.context.scene.vertexcolor.fileoutput = input[ :-4] + "_copy.nif"

        niffile.close()
        print("Finished Reading file :D")

        nif = NifDataHolder()
        nif.setinstancedata(self.data)
        return{'FINISHED'}

class NifWriter(bpy.types.Operator):
    "This will write the loaded file"

    bl_idname = "vertexcolor.nifwrite"
    bl_label = "Output file with overwritten vert color"

    def execute(self, context):

        nif = NifDataHolder()
        if(nif.getinstancedata() == null):
            bpy.context.scene.vertexcolor.helpmsg
        else:
            self.data = nif.getinstancedata()
            for root in self.data.roots:
                for block in root.tree():
                    if isinstance(block, NifFormat.NiTriShape):
                        if(block.data.has_vertex_colors):
                            for vertexcol in block.data.vertex_colors:
                                vertexcol._items[0]._value = bpy.context.scene.vertexcolor.hexwidget[0]
                                vertexcol._items[1]._value = bpy.context.scene.vertexcolor.hexwidget[1]
                                vertexcol._items[2]._value = bpy.context.scene.vertexcolor.hexwidget[2]
                            print(block.data.vertex_colors)
            stream = open(bpy.context.scene.vertexcolor.fileoutput, "wb")
            try:
                self.data.write(stream)
                print("Writing File")
            finally:
                stream.close()

        return{'FINISHED'}

class Popup(bpy.types.Operator):
    bl_idname = "vertexcolor.helpmsg"
    bl_label = "Useful info messages"

    test = bpy.props.StringProperty(name="warning", default="Please load the nif before trying to save")

    def execute(self, context):
        wm = context.window_manager
        return wm.invoke_props_dialog(self)


import multiprocessing

import pickle
import logging

logger = logging.getLogger(__name__)

from gensim.models import Word2Vec, KeyedVectors
from gensim.models.callbacks import CallbackAny2Vec

logging.getLogger("gensim").setLevel(
    logging.WARNING
)  # avoid seeing all logs from gensim.

from tqdm import tqdm

import numpy as np
import pandas as pd

from src.processing import list_shuffler
from autosklearn.classification import AutoSklearnClassifier
from autosklearn.metrics import roc_auc

np.random.seed(0)
tqdm.pandas()

class LossLogger(CallbackAny2Vec):
    """Gensim callback to output loss at each epoch"""

    def __init__(self):
        self.epoch = 1
        self.losses = []

    def on_epoch_end(self, model):
        loss = model.get_latest_training_loss()
        self.losses.append(loss)
        if self.epoch == 1:
            logger.info(f"Epoch: {self.epoch}    Loss: {loss}")
        else:
            logger.info(
                f"Epoch: {self.epoch}    Loss: {loss - self.loss_previous_step}"
            )
        self.epoch += 1
        self.loss_previous_step = loss

class ModelSaver(CallbackAny2Vec):
    """Gensim callback to save model every log_frequency epochs"""

    def __init__(
        self, d2v_object, rated_embeddings_path, w2v_model_path, log_frequency=5
    ):
        self.epoch = 1
        self.log_frequency = log_frequency
        self.d2v_object = d2v_object
        self.rated_embeddings_path = rated_embeddings_path
        self.w2v_model_path = w2v_model_path

    def on_epoch_begin(self, model):
        pass

    def on_epoch_end(self, model):
        if self.epoch % self.log_frequency == 0:
            logger.info("Saving model weights and embeddings.")
            self.d2v_object.w2v_model = model
            self.d2v_object.wv = model.wv
            self.d2v_object.save_rated_vec(self.rated_embeddings_path)
            self.d2v_object.save_w2v_model(self.w2v_model_path)
        self.epoch += 1

class D2V_Recommender:
    def __init__(
        self,
        embedding_size=100,
        window=3,
        min_count=1,
        workers=multiprocessing.cpu_count() - 1,
        num_epochs=50,
        sample=0,  # do not downsample
    ):
        self.embedding_size = embedding_size
        self.window = window
        self.min_count = min_count
        self.workers = workers
        self.num_epochs = num_epochs
        self.sample = sample

        self.wv = None  # access to embeddings with self.wv["rated_user_id"]
        self.mean_embeddings = (
            None  # access to embeddings with self.mean_embeddings["rater_user_id"]
        )
        self.data_dict = None  # dict of arrays with X_train, X_test, y_train, y_test

    def fit_rated_embeddings(
        self, d2v_train, w2v_model_path, rated_embeddings_path, resume_training=False
    ):
        """
        Fit and save Word2Vec model to embed rated users based on dating behavior of raters.

        :param d2v_train: A pd.Series of list of rated_ids (string) that were co-liked by a rater
        """
        # Prepare the data iterator
        d2v_train_iterator = self.build_data_iterator(d2v_train)

        # Initiate the model
        loss_logger = LossLogger()
        model_saver = ModelSaver(self, rated_embeddings_path, w2v_model_path)
        self.w2v_model = Word2Vec(
            size=self.embedding_size,
            window=self.window,
            min_count=self.min_count,
            workers=self.workers,
            sample=self.sample,
            sg=1,  # skip-gram
            hs=0,
            negative=5,
            callbacks=[loss_logger, model_saver],
            seed=0,
        )
        model = self.w2v_model
        if resume_training:
            logger.info("Resuming previous training.")
            model = self.load_w2v_model(w2v_model_path)
            model.build_vocab(d2v_train_iterator, update=True)
        elif model.train_count == 0:
            model.build_vocab(d2v_train_iterator)

        # train and save final model
        model.train(
            d2v_train_iterator,
            total_examples=model.corpus_count,
            epochs=self.num_epochs,
            compute_loss=True,
        )

        self.w2v_model = model
        self.wv = model.wv
        self.save_rated_vec(rated_embeddings_path)
        self.save_w2v_model(w2v_model_path)

    def fit_rater_embeddings(self, input_train, save_path=False):
        """
        Calculate an embedding for raters as the mean of embeddings of users they liked.

        :param df_: a pd.Series of list of rated_ids (string) that were co-liked by a rater
        """
        A_col, B_col, m_col = input_train.columns
        train_ = input_train.copy()
        train_ = train_[train_[m_col] > 0]  # select only those who matched

        # save the average embedding of matched people for all raters
        # TODO: optimize in one operation
        train_[B_col] = train_[B_col].apply(self.get_single_rated_vec)
        train_ = train_.dropna(
            subset=[B_col]
        )  # avoid considering the rated people appearing only once.
        train_ = train_.groupby(A_col)[B_col].apply(np.mean)
        train_.index = train_.index.astype(str)
        train_ = train_.to_frame()
        self.mean_embeddings = train_
        if save_path:
            self.save_rater_vec(save_path)

    def prepare_X_y_dataset(self, train_, test_, data_dict_path=False):
        """
        Calculate the embeddings euclidian distance (X) aligned with potential likes (y).

        :param train_: rater_id, rated_id, match (0/1) pandas Dataset
        :param test_: rater_id, rated_id, match (0/1) pandas Dataset
        :return data_dict: dictionnary of X_test, X_train, y_test, y_train np arrays.
        """
        A_col, B_col, m_col = train_.columns

        def get_vec_diff(rater_rated):
            rater, rated = rater_rated
            vec1 = self.get_single_rater_vec(rater)
            vec2 = self.get_single_rated_vec(rated)
            if vec1 is not None and vec2 is not None:
                diff = (vec1 - vec2)[0]
                dist = np.sqrt(np.sum(diff * diff, axis=0))
                return dist
            else:
                return None

        train_[A_col] = train_[[A_col, B_col]].progress_apply(
            lambda x: get_vec_diff(x), axis=1
        )
        logger.info("Finished getting distance for test dataset")
        train_ = train_.dropna()
        X_train = train_[A_col].values
        X_train = np.stack(X_train)
        y_train = train_[m_col].values
        del train_
        test_[A_col] = (
            test_[[A_col, B_col]]
            .progress_apply(lambda x: get_vec_diff(x), axis=1)
            .values
        )
        logger.info("Finished getting distance for test dataset")
        test_ = test_.dropna()
        X_test = test_[A_col].values
        X_test = np.stack(X_test)
        y_test = test_[m_col].values
        del test_

        logger.info(
            f"After skipping unseen rated users: Train size = {y_train.shape} Test size = {y_test.shape}"
        )
        data_dict = {
            "X_train": X_train,
            "X_test": X_test,
            "y_train": y_train,
            "y_test": y_test,
        }

        self.data_dict = data_dict

        if data_dict_path:
            self.save_data_dict(data_dict_path)

    def build_data_iterator(self, data):
        """ Create an iterator of which an iter is random passes on the data"""

        class shuffle_generator:
            def __init__(self, data):
                self.data = data

            def __iter__(self):
                self.data.apply(np.random.shuffle)
                return shuffle_generator_iter(self.data)

        class shuffle_generator_iter:
            def __init__(self, data):
                self.i = 0
                self.data = data
                self.data_length = len(data)

            def __iter__(self):
                # Iterators are iterables too.
                # Adding this functions to make them so.
                return self

            def __next__(self):
                if self.i < 5 * self.data_length:
                    # Shuffle at the end of the data
                    if self.i % self.data_length == 0:
                        self.data.apply(np.random.shuffle)
                    i = self.i
                    self.i += 1
                    return self.data[i % self.data_length]  # a list
                else:
                    raise StopIteration()

        return shuffle_generator(data)

    def get_single_rated_vec(self, rated_id):
        """ Get embedding vector of rated user of id rated_id"""
        try:
            return self.wv[str(rated_id)]
        except KeyError:
            # The rated user did not appear in the training dataset
            return None

    def get_single_rater_vec(self, rater_id):
        """ Get embedding vector of rater user of id rater_id"""
        try:
            return self.mean_embeddings.loc[str(rater_id)].values
        except KeyError:
            return None

    def save_rated_vec(self, wordvectors_path):
        wordvectors_path.parent.mkdir(parents=True, exist_ok=True)
        self.wv.save(str(wordvectors_path))

    def load_rated_vec(self, wordvectors_path):
        self.wv = KeyedVectors.load(str(wordvectors_path), mmap="r")
        return self.wv

    def save_w2v_model(self, w2v_model_path):
        w2v_model_path.parent.mkdir(parents=True, exist_ok=True)
        self.w2v_model.save(str(w2v_model_path))

    def load_w2v_model(self, w2v_model_path):
        self.w2v_model = Word2Vec.load(str(w2v_model_path), mmap="r")
        return self.w2v_model

    def save_rater_vec(self, rater_embeddings_path):
        # saving as a numpy array for later loading of embeddings
        rater_embeddings_path.parent.mkdir(parents=True, exist_ok=True)
        np.save(rater_embeddings_path, arr=self.mean_embeddings.reset_index().values)

    def load_rater_vec(self, rater_embeddings_path):
        arr = np.load(rater_embeddings_path, allow_pickle=True)
        self.mean_embeddings = pd.DataFrame(index=arr[:, 0].astype(str), data=arr[:, 1])
        return self.mean_embeddings

    def save_data_dict(self, data_dict_path):
        with open(data_dict_path, "wb") as handle:
            pickle.dump(self.data_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)

    def load_data_dict(self, data_dict_path):
        with open(data_dict_path, "rb") as handle:
            self.data_dict = pickle.load(handle)
        return self.data_dict


import re
import json
from uuid import uuid4
from datetime import datetime
from apt.cache import Cache
from .piplist import getpip

def tocyclonedx(pattern = None,dopip=False):
    """
    Routine to convert apt information to CycloneDx.
    """
    pkgs = [ ]
    deps = []

    sbom = {
        "bomFormat": "CycloneDX",
        "specVersion": "1.4",
        "serialNumber": "urn:uuid:" + str(uuid4()),
        "version": 1
        }
    meta = {
        "timestamp": re.sub(r'[.].*$','',str(datetime.now().isoformat())) + 'Z',
        "tools" : [ {
            "vendor" : "Eliot Lear",
            "name" : "apt2sbom",
            } ],
        "component" : {
            "type" : "device",
            "name" : "ubuntu-derived-system",
            "version" : "1"
            },
        "licenses" : [ { "license" : {
            "id" : "BSD-3-Clause"
            }
        } ]
      }
    sbom['metadata']= meta

    cache=Cache()

    for pkg in cache:
        if not pkg.is_installed:
            continue
        if pattern:
            if not re.match(pattern,pkg.name):
                continue
        ver = pkg.installed
        pack = { }
        pack['type'] = 'application'
        pack["name"]=pkg.name
        rec_info=ver.record
        pack["bom-ref"] = pkg.name
        pack["version"] = ver.version
        pack["purl"] = "pkg:deb/ubuntu/" + pkg.name + "@" + ver.version +\
            "?arch=" + ver.architecture
        pack["supplier"] = { "name" : rec_info['Maintainer'] }
        if not ver.uri is None and not ver.uri == "":
            pack['supplier']['url']= [ ver.uri ]

        hashes= []
        try:
            hashes.append({ "alg" : 'SHA-256',
                       'content' : ver.sha256 })
        except SystemError:
            pass
        try:
            hashes.append({ "alg" : 'SHA-1',
                       'content' : ver.sha1 })
        except SystemError:
            pass

        try:
            hashes.append({ "alg" : 'MD5',
                       'content' : ver.md5 })
        except SystemError:
            pass

        if hashes:
            pack['hashes'] = hashes

        if ver.homepage != '':
            pack['externalReferences'] = [ {
                "url" : ver.homepage,
                "type" : "website"
            } ]

        if ver.dependencies:
            dep = { "ref" : pack["bom-ref"] }
            dees = []

            for dep_ent in ver.dependencies:
                tname=re.sub(':any$','',dep_ent[0].name)
                if tname in cache and cache[tname].is_installed and\
                   not tname in dees:
                    dees.append(tname)

            if dees:
                dep["dependsOn"] = dees
                deps.append(dep)

        pkgs.append(pack)

    if dopip:
        pips = getpip()
        for pip in pips:
            if pattern:
                if not re.match(pattern,pip['Name']):
                    continue
            pack={}
            pack["name"]=pip["Name"] + ".pip"
            pack['type']="application"
            pack["version"] = pip["Version"]
            pack["purl"] = "pkg:pypi/" + re.sub('_','-',pack["name"].lower()) +\
                "@" + pack["version"]
            pack["bom-ref"] = pack["name"]
            pack["supplier"] = {
                "name" : pip["Author"]
                }
            try:
                pack["externalReferences"] = [ {
                    "url" : pip["homepage"],
                    "type" : "website"
                    }]
            except KeyError:
                pass
            pkgs.append(pack)

    sbom['components'] = pkgs
    sbom['dependencies'] = deps
    return json.dumps(sbom)


from pandas import DataFrame
from pandas import read_csv
from smtplib import SMTP
from smtplib import SMTPConnectError
from smtplib import SMTPServerDisconnected
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from pretty_html_table import build_table
from socket import gaierror
from io import StringIO
from sys import exc_info
from log import Log

class SendMail():
    def __init__(self):
        pass

    def set_config(self, cfg_mail, cfg_smtp):

        # return [style, sender_email, receiver_email]
        self.style = cfg_mail[0]
        self.sender_email = cfg_mail[1]
        self.receiver_email1 = cfg_mail[2]

        # return [smtp_server, port]
        self.smtp_server = cfg_smtp[0]
        self.port = cfg_smtp[1]

    def set_content(self, content):

        style = self.style
        content_out_html_final = ""

        for sheet in content:
            sheet = StringIO(sheet)
            content_out = read_csv(sheet, sep=",")
            content_out = DataFrame(content_out)

            if(content_out.empty != True):
                content_out_text = content_out.to_string()
                content_out_text = content_out_text.replace("NaN", "Brak wpisu")

                content_out_html = build_table(content_out, style)
                content_out_html_final = content_out_html_final + content_out_html + "\n
\n"

        self.text = f"""\
                Zajrzyj do pliku excel z dostępami. Zaraz jakieś wygasną !

                {content_out_text}

                ----------------------------
                Powered by Majster
                """

        self.html = f"""\

                    Zajrzyj do pliku excel z dostępami.

                        Zaraz jakieś wygasną !

                {content_out_html_final}

----------------------------

                Powered by Majster
                """

    def send_mail(self, temat):

        log_error = Log().log_error

        sender_email = self.sender_email
        receiver_email1 = self.receiver_email1
        text = self.text
        html = self.html
        smtp_server = self.smtp_server
        port = self.port

        message = MIMEMultipart("alternative")
        message["Subject"] = temat
        message["From"] = sender_email

        part1 = MIMEText(text, "plain")
        part2 = MIMEText(html, "html")

        message.attach(part1)
        message.attach(part2)

        try:
            server = SMTP(smtp_server, port)
            server.ehlo_or_helo_if_needed()

            for receiver_email in receiver_email1:
                message["To"] = receiver_email
                server.sendmail(sender_email, receiver_email, message.as_string())
                receiver_email = None

            server.close()
            return True

        except SMTPConnectError as err:
            msg = "Błąd wysyłania maila. SMTP Connection Error."
            print(msg)
            log_error(err)
            return False

        except SMTPServerDisconnected as err:
            msg = "Błąd wysyłania maila. SMTP Auth Error."
            print(msg)
            log_error(f"SMTP Error: {err}\nSprawdź dane wysyłającego")
            return False

        except TimeoutError as err:
            msg = "Błąd wysyłania maila. Timeout Error."
            print(msg)
            log_error(f"SMTP Error: {err}\nSprawdź port. Czy ruch jest otwarty?\n")
            return False

        except gaierror as err:
            msg = "Błąd wysyłania maila. SMTP Address Error."
            print(msg)
            log_error(f"SMTP Error: {err}\nSprawdź nazwę serwera SMTP w configu")
            return False

        except:
            print(exc_info()[0])
            log_error(exc_info()[0])
            return False

    def __del__(self):
        pass


""" 주사위 굴리기 2
<문제 조건>
- NxM인 크기의 지도
- 지도의 오른쪽은 동쪽, 위쪽은 북쪽

<주사위 이동>
1. 동쪽으로 한 칸 굴러감 (이동방향에 칸이 없으면 반대로 한 칸 굴러감)
2. 주사위가 도착한 칸에 해당하는 숫자만큼의 점수 획득
3. 주사위 아랫면의 정수 A와 (x, y)의 정수 B를 비교해 이동 방향 결정
    - A > B: 90도로 시계 방향 회전
    - A = B: 변화 없음
    - A < B: 90도로 반시계 방향 회전
** 점수 획득 방법: 동서남북으로 연속해서 이동할 때 (x,y)좌표의 값이 나오는 개수에 (x,y) 좌표의 값을 곱한다.

"""
import sys
from collections import deque
input = sys.stdin.readline

N, M, K = map(int, input().split(' ')) # 세로, 가로, 이동 횟수

board = []
for n in range(N):
    arr = list(map(int, input().split(' '))) # 지도에 쓰여 있는 수
    board.append(arr)

vertical = deque([2, 1, 5, 6])
horizontal = deque([4, 1, 3])

dice_x, dice_y = 0, 0 # 처음에 주사위는 외쪽 위에 위치해 있다.
dice_dir = 0
# DX, DY = [0, 0, 1, -1], [-1, 1, 0, 0] # 동-서-남-북
# 동-남-서-북
DX, DY = [0, 1, 0, -1], [1, 0, -1, 0]
def roll_dice():
    global dice_x, dice_y, dice_dir, horizontal, vertical
    nx, ny = dice_x + DX[dice_dir], dice_y + DY[dice_dir]
    if (0 <= nx < N and  0 <= ny < M):
        dice_x, dice_y = nx, ny
    else:
        if dice_dir == 0 or dice_dir == 1:
            dice_dir += 2
        else:
            dice_dir -= 2
        nx, ny = dice_x + DX[dice_dir], dice_y + DY[dice_dir]
        if (0 <= nx < N and 0 <= ny < M):
            dice_x, dice_y = nx, ny

    if dice_dir == 0: # 동쪽으로 굴려야 하는 경우
        bottom = vertical.pop()
        move = horizontal.pop()
        vertical.append(move)
        horizontal.appendleft(bottom)
        vertical[1] = horizontal[1]
    elif dice_dir == 2: # 서쪽으로 굴려야 하는 경우
        move = horizontal.popleft()
        bottom = vertical.pop()
        vertical.append(move)
        horizontal.append(bottom)
        vertical[1] = horizontal[1]
    elif dice_dir == 3: # 북쪽으로 굴려야 하는 경우
        move = vertical.popleft()
        vertical.append(move)
        horizontal[1] = vertical[1]
    else: # 남쪽으로 굴려야 하는 경우
        move = vertical.pop()
        vertical.appendleft(move)
        horizontal[1] = vertical[1]

score = 1
def dfs(x, y, cnt, visited):
    global score
    for dx, dy in zip(DX, DY):
        nx, ny = x + dx, y + dy

        if (0 <= nx < N and 0 <= ny < M) and board[x][y] == board[nx][ny] and visited[nx][ny] == False:
            visited[nx][ny] = True
            dfs(nx, ny, cnt + 1, visited)
            visited[nx][ny] = False
    return cnt

def calculate_score():
    # score = 1
    # 주사위가 도착한 칸에 대해서 점수 계산

    score = 1
    q = deque([[dice_x, dice_y, 0]])
    visited = [[False for _ in range(M)] for _ in range(N)]
    visited[dice_x][dice_y] = True

    while q:
        x, y, dist = q.popleft()
        for dx, dy in zip(DX, DY):
            nx, ny = x + dx, y + dy
            if (0 <= nx < N and 0 <= ny < M) and (visited[nx][ny] == False) and board[nx][ny] == board[dice_x][dice_y]:
                q.append([nx, ny, dist+1])
                score += 1
                visited[nx][ny] = True

    # print(f"SCORE : {score}")

    return score * board[dice_x][dice_y]

def change_dir():
    global dice_dir
    B = board[dice_x][dice_y]
    A = vertical[-1]
    # print(f"A: {A} B: {B}")
    if (A > B): # 시계방향으로 변경
        dice_dir = (dice_dir + 1) % 4
    elif (A < B): # 반시계방향으로 변경
        dice_dir = (dice_dir + 3)  % 4
    ## 같을 때는 변화가 없음.

answer= 0
for trial in range(K):
    score = 1
    roll_dice()
    change_dir()
    answer += calculate_score()
#     change_dir()

    # print(horizontal, vertical)
    # print(dice_x, dice_y)
    # print(dice_dir)
print(answer)


import os,sys,argparse
# import fileinput,time
import glob
import re,bisect
import pandas as pd
import numpy as np
from scipy import stats
import matplotlib
# matplotlib.use('Agg')
import matplotlib.pyplot as plt
matplotlib.rcParams['font.size']=14
import seaborn as sns
sns.set(font_scale=1.)
sns.set_style("whitegrid", {'axes.grid' : False})
import scipy
import scipy.optimize
sns.set_style("ticks")
matplotlib.rcParams["font.sans-serif"] = ["Arial"]
from matplotlib import gridspec

def mark_pvalue(compr_pos,positions,box_vals):
    s,p = stats.ttest_ind(box_vals[compr_pos[0]],box_vals[compr_pos[1]],nan_policy='omit')
    y, h, col = np.percentile(np.append(box_vals[compr_pos[0]],box_vals[compr_pos[1]]),95)*.95 ,1.05, 'k'
    y2 = np.percentile(np.append(box_vals[compr_pos[0]],box_vals[compr_pos[1]]),5)*0.99
    x1,x2 = positions[compr_pos[0]],positions[compr_pos[1]]
    indicator = 'down' if s>0 else 'up'
    p_label='{} \n {:.1e}'.format(indicator,p)
    if p_label[-2]=='0':
        p_label = p_label[:-2]+p_label[-1]
    if p>=0.05:
        p_label = 'n.s.'
    if compr_pos[2] == 't':
        plt.plot([x1*1.03, x1*1.03, x2*0.97, x2*0.97], [y, y*h, y*h, y], lw=1, c=col)
        plt.text((x1+x2)*.5, y*h, p_label, ha='center', va='bottom', color=col,fontsize=12)
    else:
        plt.plot([x1*1.03, x1*1.03, x2*0.97, x2*0.97], [y2, y2*.91, y2*.91, y2], lw=1, c=col)
        plt.text((x1+x2)*.5, y2*.95, p_label, ha='center', va='top', color=col,fontsize=12)

outdir = 'f2_compr_statistics_box'
os.makedirs(outdir,exist_ok=True)

indirs = ['CP_TFBS_vs_TFMS','CP_TFBS_with_motif_vs_TFMS','CP_TFBS_without_motif_vs_TFMS']
group_info = {'CP':'log10-distance CP t-stats', 'SE':'fisher_exact_s'}
or_thre = {'CP':9,'SE':6}
label_name = {'CP':'log2 T-statistics','SE':'log2 Odds Ratio'}

xticklabels = ['All peaks','Peaks w/ motif','Peaks w/o motif']
positions = [0,1,2]
colors = ['grey','grey','grey']

for or_key in ['CP','SE'][:]:
    col = group_info[or_key]
    box_vals = []
    for indir in indirs[:]:
        infile = '../f2_TFMS_TFBS_CP/{}/per_TF_per_Celltype_{}.csv'.format(indir,indir)
        df = pd.read_csv(infile,index_col = 0)
        vals = np.log2(df[col].replace(0,np.nan)).dropna().values
        box_vals.append(vals)

    ## box with scatter
    plt.figure(figsize=(3,3))
    g = plt.boxplot(box_vals,positions=positions,widths = .6,patch_artist=True,\
                boxprops=dict(color='k',facecolor='w',fill=None,lw=1),\
                medianprops=dict(color='grey'),showfliers=False)

    for patch, color in zip(g['boxes'], colors):
        patch.set_facecolor(color)

    # scatter_X = []
    # for position_id in np.arange(len(positions)):
    #     scatter_x = np.random.normal(positions[position_id],0.07,len(box_vals[position_id]))
    #     plt.scatter(scatter_x,box_vals[position_id],color=colors[position_id],s=20,zorder=0,alpha=0.99,rasterized=True)

    # positions=[0,1,2]
    for compr_pos in [[0,1,'b'], [1,2,'b'],[0,2,'t']]:
        mark_pvalue(compr_pos,positions,box_vals)

    plt.axes().set_xticklabels(xticklabels,rotation=30, ha='right',fontsize=16,color='k')
    plt.ylabel(label_name[or_key],fontsize=16)
    plt.title(or_key)
    plt.savefig('{}/{}.pdf'.format(outdir,or_key),bbox_inches='tight',pad_inches=0.02,transparent=True)
    plt.show()
    plt.close()


from dash import html, dcc, Input, Output, State, html
import dash_bootstrap_components as dbc
from dash import page_registry, page_container

dropdown_ninez_english = dbc.Row([
    dbc.Col(
        dbc.DropdownMenu(
        children=[
            dbc.DropdownMenuItem("Schooling", header=True),
            dbc.DropdownMenuItem("Pre-primary enrollment rate (5 Year old)", href="/ninez_english/enrollment-rate-pre-primary-5"),
            dbc.DropdownMenuItem("Pre-primary enrollment rate (3-5 Year old)", href="/ninez_english/enrollment-rate-pre-primary-35"),
            dbc.DropdownMenuItem("Primary net enrollment rate", href="/ninez_english/net-enrollment-primary"),
            dbc.DropdownMenuItem("Secondary net enrollment rate", href="/ninez_english/net-enrollment-secondary"),
            dbc.DropdownMenuItem("Tertiary net enrollment rate", href="/ninez_english/net-enrollment-tertiary"),
            dbc.DropdownMenuItem("Primary completion rate", href="/ninez_english/primary-completion-rate"),
            dbc.DropdownMenuItem("Secondary completion rate", href="/ninez_english/secondary-completion-rate"),
            dbc.DropdownMenuItem(divider=True),
            dbc.DropdownMenuItem("Activities and time use", header=True),
            dbc.DropdownMenuItem("Participation in care activities", href="/ninez_english/participation-care-activities"),
            dbc.DropdownMenuItem("Weekly hours allocated to care activities", href="/ninez_english/hours-care-activities"),
            dbc.DropdownMenuItem("Participation in childcare activities", href="/ninez_english/participation-childcare-activities"),
            dbc.DropdownMenuItem("Weekly hours allocated to childcare activities", href="/ninez_english/hours-childcare-activities"),
            dbc.DropdownMenuItem("Youth neither in school nor economically active", href="/ninez_english/youth-neither-school-active"),
            dbc.DropdownMenuItem(divider=True),
            dbc.DropdownMenuItem("Test Scores", header=True),
            dbc.DropdownMenuItem("Average score in standardized tests for 3rd grade students", href="/ninez_english/score-3rd"),
            dbc.DropdownMenuItem("Average score in standardized tests for 6th grade students", href="/ninez_english/score-6th"),
            dbc.DropdownMenuItem("Average score in standardized tests for 15 year-old students", href="/ninez_english/score-15"),
            dbc.DropdownMenuItem("Ratio of standardized tests scores for 3rd grade students", href="/ninez_english/ratio-3rd-english"),
            dbc.DropdownMenuItem("Ratio of standardized tests scores for 6th grade students", href="/ninez_english/ratio-6th-english"),
            dbc.DropdownMenuItem("Ratio of standardized tests scores for 15 year-old students", href="/ninez_english/ratio-15-english"),
            dbc.DropdownMenuItem("Percentage of women among the bottom 10% test scores in 3rd grade", href="/ninez_english/women-10-low-3rd"),
            dbc.DropdownMenuItem("Percentage of women among the bottom 10% test scores in 6th grade", href="/ninez_english/women-10-low-6th"),
            dbc.DropdownMenuItem("Percentage of women among the bottom 10% test scores for 15 year-old students", href="/ninez_english/women-10-low-15"),
            dbc.DropdownMenuItem("Percentage of women among the top 10% test scores in 3rd grade", href="/ninez_english/women-10-high-3rd"),
            dbc.DropdownMenuItem("Percentage of women among the top 10% test scores in 6th grade", href="/ninez_english/women-10-high-6th"),
            dbc.DropdownMenuItem("Percentage of women among the top 10% test scores for 15 year-old students", href="/ninez_english/women-10-high-15"),
            dbc.DropdownMenuItem("Percentage of women among 15 year-old functionally-illiterate students", href="/ninez_english/women-illiterate-functionally"),
        ],
        size="lg",
        nav=True,
        in_navbar=True,
        label="Indicators",
        className="ms-0",
        toggle_style={"color": "#460074"},
        align_end=False,
        style={'width':'100%'}

        )
    )
],
className="g-0 ms-auto flex-nowrap mt-5 mt-md-0",
align="center",
)

navbar_ninez_english = dbc.Navbar(
    dbc.Container(
        [
            html.A(
                # Use row and col to control vertical alignment of logo / brand
                dbc.Row(
                    [
                        dbc.NavbarToggler(id="navbar-toggler_ninez_english", n_clicks=0),
                        dbc.Collapse(
                            dropdown_ninez_english,
                            className="ml-auto",
                            id="navbar-collapse_ninez_english",
                            is_open=False,
                            navbar=True,
                        ),
                    ],
                    align="center",
                    className="g-0",
                ),
                href="/ninez_english/enrollment-rate-pre-primary-5",
                style={"textDecoration": "none"},
            ),
        ],
    fluid=True),
    #outline=True,
    color="light",
    dark=True,
    expand=True,
)

def generate_dropdown(options):
    dropdown = dcc.Dropdown(
        options=[{'label': option, 'value': option} for option in options],
        value="Argentina",
        id="all-pages-dropdown-pais-ninez-english",
        persistence=True,
        persistence_type = 'memory',
        multi=True,
        className="bg-light",
    )
    return dropdown

# Armar loop:
mark_values = {2000:'2000',2001:'2001',2002:'2002',
                2003:'2003',2004:'2004',2005:'2005',
                2006:'2006',2007:'2007',2008:'2008',
                2009:'2009',2010:'2010',2011:'2011',
                2012:'2012',2015:'2015',2016:'2016',
                2013:'2013',2014:'2014',2015:'2015',
                2016:'2016',2017:'2017',2018:'2018',
                2019:'2019',2020:'2020',2021:'2021'}

ranger_slider_year_ninez_english = dcc.RangeSlider(
        min=2000,
        max=2021,
        value=[2000,2021],
        marks=mark_values,
        step=1,
        persistence=True,
        persistence_type = 'memory',
        id="all-pages-ranger-slider-year-ninez-english"
)


"""
我们的任务 , worker,broker 都需要Celery去协调  所以我们需要创建一个Celery对象

"""

from celery import Celery

# import os
# os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mall.settings")

# 进行Celery允许配置
# 为celery使用django配置文件进行设置
import os

if not os.getenv('DJANGO_SETTINGS_MODULE'):
    os.environ['DJANGO_SETTINGS_MODULE'] = 'mall.settings'

# 以上信息加载到 app创建前面

# selery也是需要使用到django项目中的配置信息的

# 创建celery对象
# 第一步
# 第一个参数,main 一般以celery的文件夹为名字,不要重复
app = Celery(main='celery_tasks')

# 第二步
# app设置broker
# app.config_from_object设置配置文件的路径
app.config_from_object('celery_tasks.config')

# 第三步
# celery可以自动检测任务,检测任务的路径就是任务包的路径
# autodiscover_tasks第一个参数是列表
# 列表中的元素 是任务包的路径
# 路径是从celery_tasks开始就可以
app.autodiscover_tasks(['celery_tasks.sms', 'celery_tasks.email', 'celery_tasks.html'])

# worker 是通过指令来执行的
# celery -A celery实例对象的文件路径   worker -l info

# 这个指令需要在虚拟环境中执行
# celery -A celery_tasks worker -l info

# celery -A celery_tasks.main worker -l info


import copy
from datetime import datetime

from abc import ABC, abstractmethod
from forensic.common.stream.stream import DataStruct, dynamic, BinaryReader, uint16le, uint32le, bytesbe

class CodesysV3FileFormatAbstract(ABC):
    @abstractmethod
    def read_tag(self, br: BinaryReader):
        pass

    @abstractmethod
    def read_list_of_tags(self, tag_data: bytes):
        pass

class CodesysBaseTag(DataStruct):
    tid = dynamic

    def _null_ending_bytes(self, data: bytes) -> bytes:
        return data[:data.find(b'\x00')]

class CodesysUnknownTag(CodesysBaseTag):
    data = dynamic

class CodesysListOfTags(CodesysBaseTag):
    tid = 0x81
    sub_tags = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        self.sub_tags = cff.read_list_of_tags(data)

class CodesysApplicationName(CodesysBaseTag):
    tid = 0x10
    name = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        self.name = self._null_ending_bytes(data)

class CodesysListOfCodeAreas(CodesysBaseTag):
    tid = 0x82
    sub_areas = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        self.sub_areas = cff.read_list_of_tags(data)

class CodesysCodeAreaHeader(CodesysBaseTag):
    tid = 0x21
    data = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        self.data = data

class CodesysCodeAreaData(CodesysBaseTag):
    tid = 0x22
    data = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        self.data = data

class CodesysCodeArea(CodesysBaseTag):
    tid = 0xA0
    area_type = dynamic
    pointer = dynamic
    code = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        br_code_area = BinaryReader(data)
        tag_header = cff.read_tag(br_code_area)

        if isinstance(tag_header, CodesysCodeAreaHeader):
            br_pointer = BinaryReader(tag_header.data)
            self.area_type = br_pointer.read(uint16le)
            self.pointer = br_pointer.read(uint32le)

            tag_code = cff.read_tag(br_code_area)

            if isinstance(tag_code, CodesysCodeAreaData):
                br_code = BinaryReader(tag_code.data)
                code_size = br_code.read(uint32le)
                self.code = br_code.read(bytesbe(code_size))

class CodesysProjectInfo(CodesysBaseTag):
    tid = 0x87
    attributes = dynamic
    unknown_tags = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        self.attributes = {}

        labels = {1: 'name',
                  2: 'project_version',
                  3: 'project_build_datetime',
                  4: 'author',
                  5: 'description',
                  6: 'ide_build_version',
                  8: 'codesys_version'}

        for sub_tag in cff.read_list_of_tags(data):
            if sub_tag.tid in labels:
                if sub_tag.tid == 3:
                    self.attributes[labels[sub_tag.tid]] = datetime.fromtimestamp(
                        int.from_bytes(sub_tag.data, byteorder='little'))
                else:
                    self.attributes[labels[sub_tag.tid]] = self._null_ending_bytes(sub_tag.data)

            else:
                self.attributes[f'unknown_tag {sub_tag.tid}'] = sub_tag.data

class CodesysFunctionSymbols(CodesysBaseTag):
    tid = 0x88
    symbols = dynamic
    unknown_tags = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        self.symbols = {}
        self.unknown_tags = []

        temp_symbol = None
        for sub_tag in cff.read_list_of_tags(data):
            if sub_tag.tid == 1:
                temp_symbol = {}
                br_sub_tag = BinaryReader(sub_tag.data)
                temp_symbol['type'] = br_sub_tag.read(uint16le)
                temp_symbol['pointer'] = br_sub_tag.read(uint32le)
                temp_symbol['raw'] = br_sub_tag.read(bytesbe(br_sub_tag.remaining_data()))
            elif sub_tag.tid == 2:
                if temp_symbol is None:
                    temp_symbol = {}
                temp_symbol['name'] = self.null_ending_bytes(sub_tag.data)
                self.symbols[temp_symbol['pointer']] = copy.deepcopy(temp_symbol)
                temp_symbol = None
            else:
                self.unknown_tags.append({'unknown tag': sub_tag.tid, 'raw': sub_tag.data})

class CodesysFooterAddresses(CodesysBaseTag):
    tid = 0x86
    sub_tags = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        self.sub_tags = cff.read_list_of_tags(data)

class CodesysInitJumpTable(CodesysBaseTag):
    tid = 0x60
    pointer_type = dynamic
    pointer = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        br = BinaryReader(data)
        self.pointer_type = br.read(uint16le)
        self.pointer = br.read(uint32le)

class CodesysSymbols(CodesysBaseTag):
    tid = 0x85
    symbols = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        self.symbols = cff.read_list_of_tags(data)

class CodesysSymbol(CodesysBaseTag):
    tid = 0x50
    pointer_type = dynamic
    pointer = dynamic
    zeros = dynamic
    name = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        br_symbol = BinaryReader(data)
        self.pointer_type = br_symbol.read(uint16le)
        self.pointer = br_symbol.read(uint32le)
        self.zeros = br_symbol.read(uint32le)
        self.name = self._null_ending_bytes(br_symbol.read(bytesbe(br_symbol.remaining_data())))

class CodesysSymbolExtended(CodesysBaseTag):
    tid = 0x88
    pointer_type = dynamic
    pointer = dynamic
    resolve_hash = dynamic
    index = dynamic
    raw = dynamic
    name = dynamic
    unknown_tags = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        self.unknown_tags = []

        for sub_tag in cff.read_list_of_tags(data):
            if sub_tag.tid == 1:
                br_sub_tag = BinaryReader(sub_tag.data)
                self.pointer_type = br_sub_tag.read(uint16le)
                self.pointer = br_sub_tag.read(uint32le)
                self.resolve_hash = br_sub_tag.read(uint32le)
                self.index = br_sub_tag.read(uint32le)
                self.raw = br_sub_tag.read(bytesbe(br_sub_tag.remaining_data()))
            elif sub_tag.tid == 2:
                self.name = self._null_ending_bytes(sub_tag.data)
            else:
                self.unknown_tags.append({'unknown tag': sub_tag.tid, 'raw': sub_tag.data})

class CodesysEntryPoint(CodesysBaseTag):
    tid = 0x61
    pointer_type = dynamic
    pointer = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        br = BinaryReader(data)
        self.pointer_type = br.read(uint16le)
        self.pointer = br.read(uint32le)

class CodesysInternalTableSymbolStart(CodesysBaseTag):
    tid = 0x6f
    pointer_type = dynamic
    pointer = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        br = BinaryReader(data)
        self.pointer_type = br.read(uint16le)
        self.pointer = br.read(uint32le)

class CodesysInternalTableSymbolEnd(CodesysBaseTag):
    tid = 0x71
    pointer_type = dynamic
    pointer = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        br = BinaryReader(data)
        self.pointer_type = br.read(uint16le)
        self.pointer = br.read(uint32le)

class CodesysInitApplication(CodesysBaseTag):
    tid = 0x63
    pointer_type = dynamic
    pointer = dynamic

    def __init__(self, cff: CodesysV3FileFormatAbstract, data: bytes):
        br = BinaryReader(data)
        self.pointer_type = br.read(uint16le)
        self.pointer = br.read(uint32le)


import unittest
from unittest.mock import MagicMock, patch

from bson import ObjectId

from extensions.authorization.models.action import AuthorizationAction
from sdk.common.utils.convertible import ConvertibleClassValidationError
from sdk.phoenix.audit_logger import AuditLog, Metadata
from sdk.phoenix.config.server_config import PhoenixServerConfig, Server, Project

def sample_log():
    return {
        AuditLog.ACTION: AuthorizationAction.UpdateUser.value,
        AuditLog.USER_ID: "601919b5c03550c421c075eb",
        AuditLog.TARGET_ID: "5ed803dd5f2f99da73684412",
        AuditLog.REQUEST_OBJECT: {"phoneNumber": "+37253551722"},
        AuditLog.RESPONSE_OBJECT: {"id": "5ed803dd5f2f99da73684412"},
        AuditLog.METADATA: {
            Metadata.IP_ADDRESS: "0.0.0.0",
            Metadata.USER_AGENT: "",
            Metadata.HOST: "",
        },
    }

class MockInject:
    project = Project(id="p1")
    server = Server(project=project)
    instance = MagicMock(return_value=PhoenixServerConfig(server=server))

@patch("sdk.phoenix.audit_logger.inject", MockInject)
class AuditLogToDictTestCase(unittest.TestCase):
    def test_success_audit_log_to_dict(self):
        audit_log = AuditLog.from_dict(sample_log())
        self.assertIsNotNone(audit_log)

    def test_failure_missing_required_fields(self):
        required_fields = {AuditLog.ACTION, AuditLog.METADATA}

        for field in required_fields:
            log_dict = sample_log()
            log_dict.pop(field)
            with self.assertRaises(ConvertibleClassValidationError):
                AuditLog.from_dict(log_dict)

    def test_failure_non_serializable_data(self):
        serializable_fields = {AuditLog.REQUEST_OBJECT, AuditLog.RESPONSE_OBJECT}

        for field in serializable_fields:
            log_dict = sample_log()
            log_dict[field] = ObjectId
            with self.assertRaises(ConvertibleClassValidationError):
                AuditLog.from_dict(log_dict)


from functools import cache
from typing import List

class Solution:
    """
    1. p选正号的数的和
    2. s所有数组的总和
    3. s-p选负号的数的和
    => p-(s-p)==t
    => 2p==s+t
    => p==(s+t)/2
    """

    def findTargetSumWays2(self, nums: List[int], target: int) -> int:
        n, s = len(nums), sum(nums)
        target += s
        if target < 0 or target % 2 != 0:
            return 0
        target //= 2

        @cache
        def dfs(i, c):
            if i < 0:
                return 1 if c == 0 else 0
            if c < nums[i]:
                return dfs(i - 1, c)
            return dfs(i - 1, c) + dfs(i - 1, c - nums[i])

        return dfs(n - 1, target)

    def findTargetSumWays3(self, nums: List[int], target: int) -> int:
        n, s = len(nums), sum(nums)
        target += s
        if target < 0 or target % 2 != 0:
            return 0
        target //= 2
        dp = [[0] * (target + 1) for _ in range(n + 1)]
        dp[0][0] = 1
        for i, x in enumerate(nums):
            for c in range(target + 1):
                if c < x:
                    dp[i + 1][c] = dp[i][c]
                else:
                    dp[i + 1][c] = dp[i][c] + dp[i][c - x]
        return dp[n][target]

    def findTargetSumWays4(self, nums: List[int], target: int) -> int:
        n, s = len(nums), sum(nums)
        target += s
        if target < 0 or target % 2 != 0:
            return 0
        target //= 2
        dp = [[0] * (target + 1) for _ in range(2)]
        dp[0][0] = 1
        for i, x in enumerate(nums):
            for c in range(target + 1):
                if c < x:
                    dp[(i + 1) % 2][c] = dp[i % 2][c]
                else:
                    dp[(i + 1) % 2][c] = dp[i % 2][c] + dp[i % 2][c - x]
        return dp[n % 2][target]

    def findTargetSumWays(self, nums: List[int], target: int) -> int:
        n, s = len(nums), sum(nums)
        target += s
        if target < 0 or target % 2 != 0:
            return 0
        target //= 2
        dp = [[0] * (target + 1)]
        dp[0] = 1
        for x in nums:
            for c in range(target, x - 1, -1):
                dp[c] = dp[c] + dp[c - x]
        return dp[target]


"""
This module handles txtorcon debug messages.
"""

from twisted.python import log as twlog

__all__ = ['txtorlog']

DEBUG = False

txtorlog = twlog.LogPublisher()

if DEBUG:
    stdobserver = twlog.PythonLoggingObserver('txtorcon')
    fileobserver = twlog.FileLogObserver(open('txtorcon.log', 'w'))

    txtorlog.addObserver(stdobserver.emit)
    txtorlog.addObserver(fileobserver.emit)


import re
from django import template
from django.utils.safestring import mark_safe
register = template.Library()

@register.filter(name='highlight')
def highlight(text, word):
    #return mark_safe(text.replace(word, "%s" % word))
    pattern = r'(?i)(' + word + ')'
    regex = re.compile(pattern)
    text = regex.sub(r"\1", text)

    return mark_safe(text)


import openpyxl as xl
import shutil

def write(text):
    print(text, end='')

# MAYBE make a backup...
# shutil.copyfile('doge.xlsx', 'doge.xlsx.bak')

wb = xl.load_workbook('doge.xlsx')

# print(wb)

print(wb.sheetnames)

ws = wb["DogSheet"]
print(ws)

names = wb.sheetnames
ws = wb[names[0]]
print(ws)

ws = wb.worksheets[0]
print(ws)

cell = ws["A1"]
print(cell)
print(cell.value)

cell.value = "PAUL WUZ HERE"

# BADDDDDDD
# cell = "YEAH PAUL WUZ HERE AGAIN"
# print(cell)

cell = ws["A1"]
print(cell.value)

wb.save('doge.xlsx')

range = ws["C5:C8"]
print(range)
range = ws["D5:G9"]
print(range)
range = ws["D5:D9"]
print(range)

print(ws.rows)

print("ROWS")
for row in ws.rows:
    print(row)

print(ws.columns)

print("COLS")
for col in ws.columns:
    print(col)

prods = ws["C5:C8"] # a range is ALWAYS tuple of rows, each row a tuple of cells
for row in prods:
    for cell in row:
        write(cell.value + '\t')
print()

sales = ws["D5:G9"]
for row in sales:
    for cell in row:
        write(str(cell.value) + '\t')
    print()

col_letters = 'DEFG'
row_numbers = '5678'

for let in col_letters:
    # print(let)
    tot = 0
    for num in row_numbers:
        # print(num)
        # print(let+num)
        # OOPS!! NEED TO CHECK FOR A "NONE" VALUE
        addr = let+num
        val = ws[addr].value
        tot += val
    avg = tot / 4
    print(f"{let}: {avg}")


import cv2
import numpy as np

src = cv2.imread('data/lena.jpg')

dst1 = cv2.boxFilter(src, ddepth= -1, ksize=(30,30))
dst2 = cv2.bilateralFilter(src, d= -1, sigmaColor=200, sigmaSpace=10)

cv2.imshow('src', src)
cv2.imshow('dst1', dst1)
cv2.imshow('dst2', dst2)
cv2.waitKey()
cv2.destroyAllWindows()


import pyowm
import json
from discord.ext import commands as viking
from functools import partial

with open('config/config.json') as cfg:
    config = json.load(cfg)

owm_api_key = config['owm_api_key']

class Weather:
    def __init__(self, viking):
        self.viking = viking

    @viking.command()
    async def forecast(self, ctx, *, name):
        """*forecast <location>

        A command that will return the forecast of a specified location.
        """

        owm = pyowm.OWM(owm_api_key)

        observation = await self.viking.loop.run_in_executor(None, partial(owm.weather_at_place, name))
        weather = await self.viking.loop.run_in_executor(None, partial(observation.get_weather))
        location = await self.viking.loop.run_in_executor(None, partial(observation.get_location))
        get_temperature = await self.viking.loop.run_in_executor(None, partial(weather.get_temperature, unit='celsius'))
        get_wind = await self.viking.loop.run_in_executor(None, partial(weather.get_wind))

        await ctx.send('**Location:** {}'.format(location.get_name()))
        await ctx.send('**Temperature:** {}'.format(get_temperature['temp']) + u'\N{DEGREE SIGN}C')
        await ctx.send('**Humidity:** {}'.format(weather.get_humidity()) + '%')
        await ctx.send('**Wind Speed:** {}'.format(get_wind['speed']) + ' m/s')
        await ctx.send('**Description:** {}'.format(weather.get_detailed_status()))

def setup(viking):
    viking.add_cog(Weather(viking))


from pathlib import Path
import pandas as pd
import os
import shutil

data_dir = Path("data/guangzhou-daily-use-original/")
target_dir = Path("data/guangzhou-daily-use/")
target_data_dir = target_dir/"test"

os.makedirs(target_data_dir, exist_ok=True)

metadata = pd.read_csv(data_dir/"UTTRANSINFO.txt", sep="\t")

with open(target_dir/"test.tsv", "w+") as target_metadata:
    target_metadata.write("path\tsentence\n")
    for _, row in metadata.iterrows():
        path = row["UTTRANS_ID"]
        speaker = row["SPEAKER_ID"]
        sentence = row["TRANSCRIPTION"]
        target_metadata.write(path + "\t" + sentence + "\n")
        shutil.copy(data_dir/"WAV"/speaker/path, target_data_dir)


import pprint
import os
import re
import shutil
import sys
import tempfile
from io import StringIO

from copy import deepcopy
from unittest import TestCase as OrigTestCase
from vsc.install.headers import nicediff
from vsc.install.methodinspector import MethodInspector
from mock import patch

class TestCase(OrigTestCase):
    """Enhanced test case, provides extra functionality (e.g. an assertErrorRegex method)."""

    longMessage = True # print both standard messgae and custom message

    LOGCACHE = {}

    ASSERT_MAX_DIFF = 100
    DIFF_OFFSET = 5 # lines of text around changes

    # pylint: disable=arguments-differ
    def assertEqual(self, first, second, msg=None):
        """Make assertEqual always print useful messages"""

        try:
            super().assertEqual(first, second)
        except AssertionError as err:
            if msg is None:
                msg = str(err)
            else:
                msg = f"{msg}: {err}"

            if isinstance(first, str):
                txta = first
            else:
                txta = pprint.pformat(first)
            if isinstance(second, str):
                txtb = second
            else:
                txtb = pprint.pformat(second)

            diff = nicediff(txta, txtb, offset=self.DIFF_OFFSET)
            if len(diff) > self.ASSERT_MAX_DIFF:
                limit = f' (first {self.ASSERT_MAX_DIFF} lines)'
            else:
                limit = ''

            raise AssertionError(f"{msg}:\nDIFF{limit}:\n{''.join(diff[:self.ASSERT_MAX_DIFF])}")

    def setUp(self):
        """Prepare test case."""
        super().setUp()

        self.max_diff = None
        self.long_message = True

        self.orig_sys_stdout = sys.stdout
        self.orig_sys_stderr = sys.stderr

        self.orig_sys_argv = sys.argv
        sys.argv = deepcopy(self.orig_sys_argv)

        self.orig_workdir = os.getcwd()
        self.tmpdir = tempfile.mkdtemp()

    def convert_exception_to_str(self, err):
        """Convert an Exception instance to a string."""
        msg = err
        if hasattr(err, 'msg'):
            msg = err.msg
        elif hasattr(err, 'message'):
            msg = err.message
            if not msg:
                # rely on str(msg) in case err.message is empty
                msg = err
        elif hasattr(err, 'args'):  # KeyError in Python 2.4 only provides message via 'args' attribute
            msg = err.args[0]
        else:
            msg = err
        try:
            res = str(msg)
        except UnicodeEncodeError:
            res = msg.encode('utf8', 'replace')

        return res

    def assertErrorRegex(self, error, regex, call, *args, **kwargs):
        """
        Convenience method to match regex with the expected error message.
        Example: self.assertErrorRegex(OSError, "No such file or directory", os.remove, '/no/such/file')
        """
        try:
            call(*args, **kwargs)
            str_kwargs = ['='.join([k, str(v)]) for (k, v) in kwargs.items()]
            str_args = ', '.join(list(map(str, args)) + str_kwargs)
            self.assertTrue(False, f"Expected errors with {call.__name__}({str_args}) call should occur")
        except error as err:
            msg = self.convert_exception_to_str(err)
            if isinstance(regex, str):
                regex = re.compile(regex)
            self.assertTrue(regex.search(msg), f"Pattern '{regex.pattern}' is found in '{msg}'")

    def mock_stdout(self, enable):
        """Enable/disable mocking stdout."""
        sys.stdout.flush()
        if enable:
            sys.stdout = StringIO()
        else:
            sys.stdout = self.orig_sys_stdout

    def mock_stderr(self, enable):
        """Enable/disable mocking stdout."""
        sys.stderr.flush()
        if enable:
            sys.stderr = StringIO()
        else:
            sys.stderr = self.orig_sys_stderr

    def get_stdout(self):
        """Return output captured from stdout until now."""
        return sys.stdout.getvalue()

    def get_stderr(self):
        """Return output captured from stderr until now."""
        return sys.stderr.getvalue()

    def mock_logmethod(self, logmethod_func):
        """
        Intercept the logger logmethod. Use as
            mylogger = logging.getLogger
            mylogger.error = self.mock_logmethod(mylogger.error)
        """
        def logmethod(*args, **kwargs):
            if hasattr(logmethod_func, 'func_name'):
                funcname = logmethod_func.func_name
            elif hasattr(logmethod_func, 'im_func'):
                funcname = logmethod_func.im_func.__name__
            elif hasattr(logmethod_func, '__name__'):
                funcname = logmethod_func.__name__
            else:
                raise ValueError(f"Unknown logmethod {dir(logmethod_func)}")
            logcache = self.LOGCACHE.setdefault(funcname, [])
            logcache.append({'args': args, 'kwargs': kwargs})
            logmethod_func(*args, **kwargs)

        return logmethod

    def reset_logcache(self, funcname=None):
        """
        Reset the LOGCACHE
        @param: funcname: if set, only reset the cache for this log function
                (default is to reset the whole chache)
        """
        if funcname:
            self.LOGCACHE[funcname] = []
        else:
            self.LOGCACHE = {}

    def count_logcache(self, funcname):
        """
        Return the number of log messages for funcname in the logcache
        """
        return len(self.LOGCACHE.get(funcname, []))

    def gen_inspector(self, *args, **kwargs):
        """
        Convenience method to generate MethodInspector instance.
        All args are passed to the MethodInspector
        """
        return MethodInspector(*args, **kwargs)

    def create_patch(self, *args, **kwargs):
        """
        Create patch and return mocked whatever. Patch is added to tearDown
        All args are passed to patch
        """
        patcher = patch(*args, **kwargs)
        thing = patcher.start()
        self.addCleanup(patcher.stop)
        return thing

    def tearDown(self):
        """Cleanup after running a test."""
        self.mock_stdout(False)
        self.mock_stderr(False)
        self.reset_logcache()
        sys.argv = self.orig_sys_argv
        os.chdir(self.orig_workdir)
        shutil.rmtree(self.tmpdir)

        super().tearDown()

# backwards incompatible change
class VSCImportTest(TestCase):
    """
    catchall for old tests.
    """
    def test_deprecated_fail(self):
        """
        VSCImportTest is now deprecated and will always fail, use
            from vsc.install.commontest import CommonTest
        """
        self.assertTrue(False, msg='Use "from vsc.install.commontest import CommonTest" instead of VSCImportTest.')


import re
import requests

def caption_image(image_data):
    # Replace with the appropriate API URL
    api_url = "http://localhost:2000/caption_image"
    res = requests.post(api_url, files={"image": image_data})
    res_json = res.json()
    extracted_text = res_json["text"]
    caption = res_json["caption"]
    print("(debug) Used image captioning. Caption: ", caption)

    # Remove repeating character sequences like "##oooooooooooooooooooooooooooooooooooooooooooooooo"
    caption = re.sub(r'(.)\1{10,}', '', caption)

    # If such sequence was at the beginning of the caption, it could leave leading white space, remove it
    caption = caption.lstrip()

    # Limit the caption to 100 characters
    # caption = caption[:100]

    return caption, extracted_text


import tensorflow as tf
import numpy as np

#from mpl_toolkits.mplot3d import Axes3D
#import matplotlib.pyplot as plt

import random

import signal
import sys
import time
import math
import csv
import os
import argparse
from datetime import datetime
from functools import reduce

print("\nPair Identification")
print("============================\n")

# Step 1: Input parameters
###################################################################################################

# Default parameters
# X, Y, Z bins
XBins = 32
YBins = 32
ZBins = 64

# File names
FileName = "PairIdentification.p1.sim.gz"
GeometryName = "$(MEGALIB)/resource/examples/geomega/GRIPS/GRIPS.geo.setup"

# Depends on GPU memory and layout

#Set later
BatchSize = 128

MaxEvents = 100000

# Determine derived parameters

OutputDataSpaceSize = ZBins

XMin = -43
XMax = 43

# XMin = -5
# XMax = +5

YMin = -43
YMax = 43

# YMin = -5
# YMax = +5

ZMin = 13
ZMax = 45

#Set in stone later
TestingTrainingSplit = 0.8

OutputDirectory = "Results"

parser = argparse.ArgumentParser(description='Perform training and/or testing of the pair identification machine learning tools.')
parser.add_argument('-f', '--filename', default='PairIdentification.p1.sim.gz', help='File name used for training/testing')
parser.add_argument('-m', '--maxevents', default='10000', help='Maximum number of events to use')
parser.add_argument('-s', '--testingtrainigsplit', default='0.1', help='Testing-training split')
parser.add_argument('-b', '--batchsize', default='128', help='Batch size')

args = parser.parse_args()

if args.filename != "":
  FileName = args.filename

if int(args.maxevents) > 1000:
  MaxEvents = int(args.maxevents)

if int(args.batchsize) >= 16:
  BatchSize = int(args.batchsize)

if float(args.testingtrainigsplit) >= 0.05:
  TestingTrainingSplit = float(args.testingtrainigsplit)

if os.path.exists(OutputDirectory):
  Now = datetime.now()
  OutputDirectory += Now.strftime("_%Y%m%d_%H%M%S")

os.makedirs(OutputDirectory)

###################################################################################################
# Step 2: Global functions
###################################################################################################

# Take care of Ctrl-C
Interrupted = False
NInterrupts = 0
def signal_handler(signal, frame):
  global Interrupted
  Interrupted = True
  global NInterrupts
  NInterrupts += 1
  if NInterrupts >= 2:
    print("Aborting!")
    sys.exit(0)
  print("You pressed Ctrl+C - waiting for graceful abort, or press  Ctrl-C again, for quick exit.")
signal.signal(signal.SIGINT, signal_handler)

# Everything ROOT related can only be loaded here otherwise it interferes with the argparse
from EventData import EventData

# Load MEGAlib into ROOT so that it is usable
import ROOT as M
M.gSystem.Load("$(MEGALIB)/lib/libMEGAlib.so")
M.PyConfig.IgnoreCommandLineOptions = True

###################################################################################################
# Step 3: Create some training, test & verification data sets
###################################################################################################

# Read the simulation file data:
DataSets = []

# Load geometry:
Geometry = M.MDGeometryQuest()
if Geometry.ScanSetupFile(M.MString(GeometryName)) == True:
  print("Geometry " + GeometryName + " loaded!")
else:
  print("Unable to load geometry " + GeometryName + " - Aborting!")
  quit()

Reader = M.MFileEventsSim(Geometry)
if Reader.Open(M.MString(FileName)) == False:
  print("Unable to open file " + FileName + ". Aborting!")
  quit()

print("\n\nStarted reading data sets")
NumberOfDataSets = 0
while NumberOfDataSets < MaxEvents:
  Event = Reader.GetNextEvent()
  if not Event:
    break

  if Event.GetNIAs() > 0:
    Data = EventData()
    if Data.parse(Event) == True:
      if Data.hasHitsOutside(XMin, XMax, YMin, YMax, ZMin, ZMax) == False:
        DataSets.append(Data)
        NumberOfDataSets += 1
        if NumberOfDataSets % 500 == 0:
            print("Data sets processed: {}".format(NumberOfDataSets))

print("Info: Parsed {} events".format(NumberOfDataSets))

# Split the data sets in training and testing data sets

TestingTrainingSplit = 0.75

numEvents = len(DataSets)

numTraining = int(numEvents * TestingTrainingSplit)

TrainingDataSets = DataSets[:numTraining]
ValidationDataSets = DataSets[numTraining:]

# For testing/validation split
# ValidationDataSets = TestingDataSets[:int(len(TestingDataSets)/2)]
# TestingDataSets = TestingDataSets[int(len(TestingDataSets)/2):]

print("###### Data Split ########")
print("Training/Testing Split: {}".format(TestingTrainingSplit))
print("Total Data: {}, Training Data: {}, Validation Data: {},Testing Data: {}".format(numEvents, len(TrainingDataSets), len(ValidationDataSets), len(TestingDataSets)))
print("##########################")

###################################################################################################
# Step 4: Setting up the neural network
###################################################################################################

#TODO: Tweak/optimize model
# Is there a better loss function?
#Make more efficient for larger data sets

print("Info: Setting up neural network...")

print("Info: Setting up 3D CNN...")
conv_model = tf.keras.models.Sequential(name='Pair Identification CNN')
conv_model.add(tf.keras.layers.Conv3D(filters=64, kernel_size=3, strides=2, input_shape=(XBins, YBins, ZBins, 1)))
# conv_model.add(tf.keras.layers.MaxPooling3D((2,2,1)))
conv_model.add(tf.keras.layers.LeakyReLU(alpha=0.25))
conv_model.add(tf.keras.layers.BatchNormalization())
conv_model.add(tf.keras.layers.Conv3D(filters=96, kernel_size=3, strides=1, activation='relu'))
conv_model.add(tf.keras.layers.BatchNormalization())
# conv_model.add(tf.keras.layers.MaxPooling3D((2,2,1)))
conv_model.add(tf.keras.layers.Flatten())
conv_model.add(tf.keras.layers.Dense(3*OutputDataSpaceSize, activation='relu'))
conv_model.add(tf.keras.layers.BatchNormalization())
print("Conv Model Summary: ")
print(conv_model.summary())

print("Info: Setting up Gamma Energy Model...")
base_model = tf.keras.models.Sequential(name='Gamma Model')
base_model.add(tf.keras.layers.Dense(int(1.5*OutputDataSpaceSize), activation='relu', input_shape=(1,)))
base_model.add(tf.keras.layers.BatchNormalization())
print("Base Model Summary: ")
print(base_model.summary())

print("Info: Setting up Combined NN...")
combinedInput = tf.keras.layers.concatenate([conv_model.output, base_model.output])
combinedLayer = tf.keras.layers.Dense(OutputDataSpaceSize, activation='softmax')(combinedInput)
combined_model = tf.keras.models.Model([conv_model.input, base_model.input], combinedLayer)
print("Combined Model Summary: ")
print(combined_model.summary())

combined_model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])

# Session configuration
print("      ... configuration ...")
Config = tf.ConfigProto()
Config.gpu_options.allow_growth = True

# Create and initialize the session
print("      ... session ...")
Session = tf.Session(config=Config)
Session.run(tf.global_variables_initializer())

print("      ... listing uninitialized variables if there are any ...")
print(tf.report_uninitialized_variables())

print("      ... writer ...")
writer = tf.summary.FileWriter(OutputDirectory, Session.graph)
writer.close()

# Add ops to save and restore all the variables.
print("      ... saver ...")
Saver = tf.train.Saver()

K = tf.keras.backend
K.set_session(Session)

###################################################################################################
# Step 5: Training and evaluating the network
###################################################################################################

#TODO: attempt multiple models based on energy level
#TODO: Try a dual model setup for high and low energy setups: 10-20 and then 21+
# TODO: Add more robust model performance evaluation
#TODO: Modularize tensor set up and try sequence for training

BatchSize = 128

print("Initializing Tensors...")

class tensor_generator(tf.keras.utils.Sequence):

    def __init__(self, event_data, batch_size):
        self.event_data = event_data
        self.batch_size = batch_size

    def __len__(self):
        return int(len(self.event_data)/self.batch_size)

    def __getitem__(self, idx):
        pos_tensor = make_positional_tensor(self.event_data, idx, self.batch_size)
        gamma_tensor = make_gamma_tensor(self.event_data, idx, self.batch_size)
        label_tensor = make_label_tensor(self.event_data, idx, self.batch_size)

        if len(pos_tensor) != len(gamma_tensor) and len(pos_tensor) != len(label_tensor):
            raise Exception("Bad tensor sizes")

        return ([pos_tensor, gamma_tensor], label_tensor)

def make_positional_tensor(event_data, idx, batch_size):
    tensor = np.zeros(shape=(batch_size, XBins, YBins, ZBins, 1))
    for i in range(batch_size):
        Event = event_data[i + idx*batch_size]
        for j in range(len(Event.X)):
            XBin = int( (Event.X[j] - XMin) / ((XMax - XMin) / XBins) )
            YBin = int( (Event.Y[j] - YMin) / ((YMax - YMin) / YBins) )
            ZBin = int( (Event.Z[j] - ZMin) / ((ZMax - ZMin) / ZBins) )
            if XBin >= 0 and YBin >= 0 and ZBin >= 0 and XBin < XBins and YBin < YBins and ZBin < ZBins:
                tensor[i][XBin][YBin][ZBin][0] = Event.E[j]
    return tensor

def make_gamma_tensor(event_data, idx, batch_size):
    tensor = np.zeros(shape=(batch_size, 1))
    for i in range(batch_size):
        event = event_data[i + idx*batch_size]
        tensor[i][0] = event.GammaEnergy
    return tensor

def make_label_tensor(event_data, idx, batch_size):
    tensor = np.zeros(shape=(batch_size, OutputDataSpaceSize))
    for i in range(batch_size):
        Event = event_data[i + idx*BatchSize]
        # Set the layer in which the event happened
        if Event.OriginPositionZ > ZMin and Event.OriginPositionZ < ZMax:
            LayerBin = int ((Event.OriginPositionZ - ZMin) / ((ZMax- ZMin)/ ZBins) )
            tensor[i][LayerBin] = 1
        else:
            #TODO May need to reevaluate this line
            tensor[i][OutputDataSpaceSize-1] = 1
    return tensor

training_generator = tensor_generator(TrainingDataSets, BatchSize)
validation_generator = tensor_generator(ValidationDataSets, BatchSize)
# testing_generator = tensor_generator(TestingDataSets, BatchSize)

print("Training Model...")

history = combined_model.fit_generator(generator=training_generator, verbose=1, epochs=50, validation_data=validation_generator, shuffle=True)

print("Finished Training\nHistory is: \n")
print(history.history)

print("Evaluating Model...")

loss, acc = combined_model.evaluate_generator(generator=validation_generator, verbose=1)

print("Final Model has accuracy: {} with loss: {}".format(acc, loss))

#Non generated way of setting up data
# numBatches = int(len(TrainingDataSets)/BatchSize)
#
# #Elements are each one batch with [In, Out]--for running in batches
# tensors = []
# energy_tensors = []
#
# for i in range(numBatches):
#     if i % 10 == 0 and i > 0:
#         print("Created {} train tensors".format(i))
#
#     InputTensor = np.zeros(shape=(BatchSize, XBins, YBins, ZBins, 1))
#     OutputTensor = np.zeros(shape=(BatchSize, OutputDataSpaceSize))
#     InputEnergyTensor = np.zeros(shape=(BatchSize, 1))
#
#     for j in range(BatchSize):
#         Event = TrainingDataSets[j + i*BatchSize]
#         # Set the layer in which the event happened
#         if Event.OriginPositionZ > ZMin and Event.OriginPositionZ < ZMax:
#             LayerBin = int ((Event.OriginPositionZ - ZMin) / ((ZMax- ZMin)/ ZBins) )
#             OutputTensor[j][LayerBin] = 1
#         else:
#             #May need to reevaluate this line
#             OutputTensor[j][OutputDataSpaceSize-1] = 1
#
#       # Set all the hit locations and energies
#         for k in range(len(Event.X)):
#             XBin = int( (Event.X[k] - XMin) / ((XMax - XMin) / XBins) )
#             YBin = int( (Event.Y[k] - YMin) / ((YMax - YMin) / YBins) )
#             ZBin = int( (Event.Z[k] - ZMin) / ((ZMax - ZMin) / ZBins) )
#             if XBin >= 0 and YBin >= 0 and ZBin >= 0 and XBin < XBins and YBin < YBins and ZBin < ZBins:
#                 InputTensor[j][XBin][YBin][ZBin][0] = Event.E[k]
#
#         InputEnergyTensor[j][0] = Event.GammaEnergy
#
#     tensors.append([InputTensor, OutputTensor])
#     energy_tensors.append([InputEnergyTensor, OutputTensor])
#
#
# test_tensors = []
# test_energy_tensors = []
#
# for i in range(int(len(TestingDataSets)/BatchSize)):
#     if i % 10 == 0 and i > 0:
#         print("Created {} test tensors".format(i))
#
#     InputTensor = np.zeros(shape=(BatchSize, XBins, YBins, ZBins, 1))
#     OutputTensor = np.zeros(shape=(BatchSize, OutputDataSpaceSize))
#     InputEnergyTensor = np.zeros(shape=(BatchSize, 1))
#
#     for j in range(BatchSize):
#         Event = TestingDataSets[j + i*BatchSize]
#         # Set the layer in which the event happened
#         if Event.OriginPositionZ > ZMin and Event.OriginPositionZ < ZMax:
#             LayerBin = int ((Event.OriginPositionZ - ZMin) / ((ZMax- ZMin)/ ZBins) )
#             OutputTensor[j][LayerBin] = 1
#         else:
#             OutputTensor[j][OutputDataSpaceSize-1] = 1
#
#       # Set all the hit locations and energies
#         for k in range(len(Event.X)):
#             XBin = int( (Event.X[k] - XMin) / ((XMax - XMin) / XBins) )
#             YBin = int( (Event.Y[k] - YMin) / ((YMax - YMin) / YBins) )
#             ZBin = int( (Event.Z[k] - ZMin) / ((ZMax - ZMin) / ZBins) )
#             if XBin >= 0 and YBin >= 0 and ZBin >= 0 and XBin < XBins and YBin < YBins and ZBin < ZBins:
#                 InputTensor[j][XBin][YBin][ZBin][0] = Event.E[k]
#
#         InputEnergyTensor[j][0] = Event.GammaEnergy
#
#     test_tensors.append([InputTensor, OutputTensor])
#     test_energy_tensors.append([InputEnergyTensor, OutputTensor])
#
# if len(tensors) != len(energy_tensors):
#     print("ERROR two training inputs not of same size")
#
# if len(test_tensors) != len(test_energy_tensors):
#     print("ERROR two test inputs not of same size")
#
# print("Training Model...")
# history = []
# for i in range(len(tensors)):
#     history.append(combined_model.fit([tensors[i][0], energy_tensors[i][0]], tensors[i][1], epochs=5))
#
#
#
# print("Checking Performance...")
# acc_list = []
# loss_list = []
#
# accTestlist = []
# lossTestlist = []
#
# for i in range(len(tensors)):
#     loss, acc = combined_model.evaluate([tensors[i][0], energy_tensors[i][0]], tensors[i][1])
#     accTestlist.append(acc)
#     lossTestlist.append(loss)
#
# for i in range(len(test_tensors)):
#     loss, acc = combined_model.evaluate([test_tensors[i][0], test_energy_tensors[i][0]], test_tensors[i][1])
#     acc_list.append(acc)
#     loss_list.append(loss)
#
# for i in range(len(accTestlist)):
#     print('On training round {} the accuracy was {} with loss {}'.format(i, accTestlist[i], lossTestlist[i]))
#
# for i in range(len(acc_list)):
#     print('On test round {} the accuracy was {} with loss {}'.format(i, acc_list[i], loss_list[i]))


import os, urllib.request, json, subprocess
from urllib.error import HTTPError

path = 'C:\\Users\\Benjamin\\Desktop\\BensFolder\\DEV\\Candco\\CPP\\Projects\\YouTubeCommentsGraph\\CommentsCounter\\'

os.chdir(path)

def getURL(url):
    res = ""
    try:
        res = urllib.request.urlopen(url).read().decode('utf-8')
    except HTTPError as e:
        res = e.read()
    return res

def exec(cmd):
    return subprocess.check_output(cmd, shell = True)

FranceInter = 'UCJldRgT_D7Am-ErRHQZ90uw'
France24 = 'UCCCPCZNChQdGa9EkATeye4g'
France24Arabic = 'UCdTyuXgmJkG_O8_75eqej-w'
euronewsInFrench = 'UCW2QcKZiU8aUGg4yxCIditg'

youtuberId = FranceInter

url = 'https://www.youtube.com/channel/' + youtuberId + '/videos'
content = getURL(url)
content = content.split('var ytInitialData = ')[1].split(";</script>")[0]
data = json.loads(content)
#tokenPattern = '"token":"'
#contentParts = content.split(tokenPattern)
#token = contentParts[1].split('"')[0]
#print(token)

"""obj = data#['onResponseReceivedActions']
for key in obj.keys():
    print(key)"""

videosIds = []

def retrieveVideosFromContent(content):
    global videosIds
    if '"videoId": "' in content:
        contentParts = content.split('"videoId": "')
    else:
        contentParts = content.split('"videoId":"')
    contentPartsLen = len(contentParts)
    for contentPartsIndex in range(contentPartsLen):
        contentPart = contentParts[contentPartsIndex]
        contentPartParts = contentPart.split('"')
        videoId = contentPartParts[0]
        #print(videoId)
        videoIdLen = len(videoId)
        if not videoId in videosIds and videoIdLen == 11:
            videosIds += [videoId]

retrieveVideosFromContent(content)

videosIdsLen = len(videosIds)
print(videosIdsLen, videosIds)

entries = data['contents']['twoColumnBrowseResultsRenderer']['tabs']
"""entriesLen = len(entries)
for entriesIndex in range(entriesLen):
    entry = entries[entriesIndex]
    #print(entry)
    if 'continuationItemRenderer' in str(entry):
        print('hey', entriesIndex)
"""

entry = entries[1]['tabRenderer']['content']['sectionListRenderer']['contents'][0]['itemSectionRenderer']['contents'][0]['gridRenderer']['items'][-1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']['token'] # nous ce que l'on aime chez YouTube c'est les listes d'un élément
#print(str(entry))
token = entry

cmd = 'curl -s \'https://www.youtube.com/youtubei/v1/browse?key=YOUR_API_KEY\' -H \'Content-Type: application/json\' --data-raw \'{"context":{"client":{"clientName":"WEB","clientVersion":"CENSORED"}},"continuation":"' + token + '"}\''
cmd = cmd.replace('"', '\\"') # otherwise doesn't work on Windows if doesn't do both replacements
cmd = cmd.replace("\'", '"')
#print(cmd)
content = exec(cmd).decode('utf-8')

retrieveVideosFromContent(content)

data = json.loads(content)
entry = data['onResponseReceivedActions'][0]['appendContinuationItemsAction']['continuationItems'][-1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']['token']
token = entry


import datetime as dt
import dateutil as du

customerGeneralShittyness =  {'Resyurdui': 0.1,
								'Loytehroa': 0.01,
								'Pufyoffio': 0.2,
								'Semsortee': 0.07,
								'Gotyadgee': 0.05,
								'Fopliwfoo': 0.1,
								'Tilpetpae': 0.125}

countryLocales = {'Resyurdui': 'hr_HR',
				  'Loytehroa': 'dk_DK',
				  'Pufyoffio': 'bg_BG',
				  'Semsortee': 'et_EE',
				  'Gotyadgee': 'cs_CZ',
				  'Fopliwfoo': 'en_GB',
				  'Tilpetpae': 'el_GR'}

#Customers means yearly customers.
geography = [
	{	'city': 'Henna', 'country': 'Resyurdui', 'population': '2406219', 'customers': 192498},
	{	'city': 'Jilli', 'country': 'Loytehroa', 'population': '1069431', 'customers': 106943},
	{	'city': 'Proled', 'country': 'Pufyoffio', 'population': '1960623', 'customers': 176456},
	{	'city': 'Gumfe', 'country': 'Semsortee', 'population': '1960623', 'customers': 137244},
	{	'city': 'Wuchu', 'country': 'Gotyadgee', 'population': '2584457', 'customers': 129223},
	{	'city': 'Sirke', 'country': 'Gotyadgee', 'population': '1425907', 'customers': 128332},
	{	'city': 'Quisxy', 'country': 'Resyurdui', 'population': '1425907', 'customers': 14259},
	{	'city': 'Qurli', 'country': 'Loytehroa', 'population': '1336788', 'customers': 93575},
	{	'city': 'Wahns', 'country': 'Loytehroa', 'population': '712954', 'customers': 71295},
	{	'city': 'Puwka', 'country': 'Loytehroa', 'population': '2406219', 'customers': 48124},
	{	'city': 'Tamvo', 'country': 'Loytehroa', 'population': '89119', 'customers': 6238},
	{	'city': 'Bubri', 'country': 'Pufyoffio', 'population': '1069431', 'customers': 32083},
	{	'city': 'Nuvso', 'country': 'Semsortee', 'population': '445596', 'customers': 31192},
	{	'city': 'Burtie', 'country': 'Semsortee', 'population': '534715', 'customers': 16041},
	{	'city': 'Capdu', 'country': 'Gotyadgee', 'population': '1425907', 'customers': 99813},
	{	'city': 'Zutne', 'country': 'Fopliwfoo', 'population': '891192', 'customers': 62383},
	{	'city': 'Macru', 'country': 'Gotyadgee', 'population': '623834', 'customers': 62383},
	{	'city': 'Gerfed', 'country': 'Fopliwfoo', 'population': '623834', 'customers': 18715},
	{	'city': 'Texgo', 'country': 'Tilpetpae', 'population': '1871503', 'customers': 74860},
	{	'city': 'Layvo', 'country': 'Tilpetpae', 'population': '267358', 'customers': 26736},
	{	'city': 'Pepxi', 'country': 'Tilpetpae', 'population': '1158550', 'customers': 23171}
]

DateStart = dt.date(2016,1,1)
DateEnd = dt.date(2016,11,30)

presentedIdentificationTypes = ['Passport', 'Photo ID', 'Driving Licence', 'Government Papers', 'Visa', 'Other']
badIDNotes = ["Names didn't match", "ID expired", "Not the customers own ID", "ID in poor condition and couldn't be identified.", "Fake ID"]
badIDSelection = [0,0,1,1,1,1,2,2,3,4]
customerPaymentMethods = ['Debit Card', 'Money Order', 'Credit Order', 'Credit Card', 'Postal Order', 'Cash']


from sqlalchemy import Column, Integer, String, Date, DECIMAL, ForeignKey

from db_util import Base, Session

sess = Session()

class Dept(Base):
    __tablename__ = 'dept'
    deptno = Column(Integer, primary_key=True)
    dname = Column(String(14))
    loc = Column(String(13))

    def __repr__(self):
        return str({
            'deptno': self.deptno,
            'dname': self.dname,
            'loc': self.loc
        })

class Emp(Base):
    __tablename__ = 'emp'
    empno = Column(Integer, primary_key=True)
    ename = Column(String(10))
    job = Column(String(9))
    mgr = Column(Integer)
    hiredate = Column(Date)
    sal = Column(DECIMAL(7, 2))
    comm = Column(DECIMAL(7, 2))
    deptno = Column(Integer, ForeignKey('dept.deptno'))

    def __repr__(self):
        return str({
            'empno': self.empno,
            'ename': self.ename,
            'job': self.job,
            'deptno': self.deptno,
            'sal': self.sal,
            'comm': self.comm
        })