import hashlib
import logging
import os
import shutil
import subprocess  # nosec
import pkg_resources

from tern.utils import constants
from tern.utils import general

# remove root filesystems
remove = ['rm', '-rf']

# tar commands
check_tar = ['tar', '-tf']
extract_tar = ['tar', '-xf']

# mount commands
mount = ['mount', '-o', 'bind']
mount_proc = ['mount', '-t', 'proc', '/proc']
mount_sys = ['mount', '-o', 'bind', '/sys']
mount_dev = ['mount', '-o', 'bind', '/dev']
unmount = ['umount']

mount_dir = None

# enable host DNS settings
host_dns = ['cp', constants.resolv_path]

# unshare PID within rootfs
unshare_pid = ['unshare', '-pf']

# union mount
union_mount = ['mount', '-t', 'overlay', 'overlay', '-o']

# global logger
logger = logging.getLogger(constants.logger_name)

def set_mount_dir(bind=None):
    '''Set mount directory according to --bind-mount CLI option (or lack
    thereof). The mount_dir value is used to set the working directory
    properly in get_working_dir().'''
    global mount_dir
    if bind:
        mount_dir = bind
    else:
        mount_dir = general.get_top_dir()

def root_command(command, *extra):
    '''Invoke a shell command as root or using sudo. The command is a
    list of shell command words'''
    full_cmd = []
    # add sudo before a command if the user is not root
    if not general.check_root():
        full_cmd.append('sudo')
    full_cmd.extend(command)
    for arg in extra:
        full_cmd.append(arg)
    # invoke
    logger.debug("Running command: %s", ' '.join(full_cmd))
    pipes = subprocess.Popen(full_cmd, stdout=subprocess.PIPE,  # nosec
                             stderr=subprocess.PIPE)
    result, error = pipes.communicate()  # nosec
    if error:
        logger.error("Command failed. %s", error.decode())
        raise subprocess.CalledProcessError(  # nosec
            1, cmd=full_cmd, output=error)
    return result

def shell_command(is_sudo, command, *extra):
    '''Invoke a shell command as a regular user unless explicitly stated.
    This is used to check the result and error message of the command'''
    full_cmd = []
    if not isinstance(is_sudo, bool):
        raise TypeError("First argument should be of type bool")
    if not general.check_root() and is_sudo:
        full_cmd.append('sudo')
    full_cmd.extend(command)
    for arg in extra:
        full_cmd.append(arg)
    # invoke
    logger.debug("Running command: %s", ' '.join(full_cmd))
    pipes = subprocess.Popen(full_cmd, stdout=subprocess.PIPE,  # nosec
                             stderr=subprocess.PIPE)
    return pipes.communicate()  # nosec

def check_tar_permissions(tar_file, directory_path):
    '''Invoke a shell command as the current user. If the error contains
    'Operation not permitted' then return False. Else return True'''
    _, error = shell_command(
        False, extract_tar, tar_file, '-C', directory_path)
    if "Operation not permitted" in error.decode():
        return False
    return True

def check_tar_members(tar_file):
    '''Given the path to the tar file, check to see if there is an error with
    the members of the tarfile or if it is empty'''
    result, error = shell_command(False, check_tar, tar_file)
    if error:
        logger.error("Malformed tar: %s", error.decode())
        raise EOFError("Malformed tarball: {}".format(tar_file))
    if not result:
        raise ValueError("Empty tarball: {}".format(tar_file))
    return result

def get_working_dir():
    '''General purpose utility to return the absolute path of the working
    directory'''
    return os.path.join(mount_dir, constants.temp_folder)

def get_untar_dir(layer_tarfile):
    '''get the directory to untar the layer tar file'''
    return os.path.join(get_working_dir(), os.path.dirname(
        layer_tarfile), constants.untar_dir)

def get_layer_tar_path(layer_tarfile):
    '''get the full path of the layer tar file'''
    return os.path.join(get_working_dir(), layer_tarfile)

def set_up():
    '''Create required directories'''
    op_dir = get_working_dir()
    workdir_path = os.path.join(op_dir, constants.workdir)
    mergedir_path = os.path.join(op_dir, constants.mergedir)
    # create higher level directory
    if not os.path.isdir(op_dir):
        os.mkdir(op_dir)
    # create mount points
    if not os.path.isdir(workdir_path):
        os.mkdir(workdir_path)
    if not os.path.isdir(mergedir_path):
        os.mkdir(mergedir_path)

def extract_tarfile(tar_path, directory_path):
    '''Give the full path to the tar file, extract the tar file into the
    given directory'''
    try:
        os.mkdir(directory_path)
    except FileExistsError:
        # attempt to remove using user permissions
        try:
            shutil.rmtree(directory_path)
            os.mkdir(directory_path)
        except PermissionError:
            # attempt to remove using root permissions
            root_command(remove, directory_path)
            os.mkdir(directory_path)
    # check tarball - should raise an error if anything is wrong
    check_tar_members(tar_path)
    # check if user can extract tarball
    success = check_tar_permissions(tar_path, directory_path)
    if not success:
        # attempt to extract using root permissions
        root_command(extract_tar, tar_path, '-C', directory_path)

def prep_rootfs(rootfs_dir):
    '''Mount required filesystems in the rootfs directory'''
    rootfs_path = os.path.abspath(rootfs_dir)
    try:
        root_command(mount_proc, os.path.join(rootfs_path, 'proc'))
        root_command(mount_sys, os.path.join(rootfs_path, 'sys'))
        root_command(mount_dev, os.path.join(rootfs_path, 'dev'))
        root_command(host_dns, os.path.join(
            rootfs_path, constants.resolv_path[1:]))
    except subprocess.CalledProcessError as error:
        logger.error("%s", error.output)
        raise

def mount_base_layer(base_layer_tar):
    '''To mount to base layer:
        1. Untar the base layer tar file
        2. Mount into mergedir'''
    base_rootfs_path = get_untar_dir(base_layer_tar)
    target_dir_path = os.path.join(get_working_dir(), constants.mergedir)
    root_command(mount, base_rootfs_path, target_dir_path)
    return target_dir_path

def mount_diff_layers(diff_layers_tar):
    '''Using overlayfs, mount all the layer tarballs'''
    # make a list of directory paths to give to lowerdir
    lower_dir_paths = []
    for layer_tar in diff_layers_tar:
        lower_dir_paths.append(get_untar_dir(layer_tar))
    upper_dir = lower_dir_paths.pop()
    lower_dir = ':'.join(list(reversed(lower_dir_paths)))
    merge_dir_path = os.path.join(get_working_dir(), constants.mergedir)
    workdir_path = os.path.join(get_working_dir(), constants.workdir)
    args = 'lowerdir=' + lower_dir + ',upperdir=' + upper_dir + \
           ',workdir=' + workdir_path
    root_command(union_mount, args, merge_dir_path)
    return merge_dir_path

def run_chroot_command(command_string, shell):
    '''Run the command string in a chroot jail within the rootfs namespace'''
    target_dir = os.path.join(get_working_dir(), constants.mergedir)
    mount_proc = '--mount-proc=' + os.path.join(
        os.path.abspath(target_dir), 'proc')
    try:
        result = root_command(unshare_pid, mount_proc, 'chroot', target_dir,
                              shell, '-c', command_string)
        return result
    except subprocess.CalledProcessError as error:
        logger.warning("Error executing command in chroot")
        raise subprocess.CalledProcessError(
            1, cmd=command_string, output=error.output.decode('utf-8'))

def undo_mount():
    '''Unmount proc, sys, and dev directories'''
    rootfs_path = os.path.join(get_working_dir(), constants.mergedir)
    root_command(unmount, os.path.join(rootfs_path, 'proc'))
    root_command(unmount, os.path.join(rootfs_path, 'sys'))
    root_command(unmount, os.path.join(rootfs_path, 'dev'))

def unmount_rootfs():
    '''Unmount the filesystem'''
    rootfs_path = os.path.join(get_working_dir(), constants.mergedir)
    root_command(unmount, '-rl', rootfs_path)

def clean_up():
    '''Remove all the setup directories'''
    mergedir_path = os.path.join(get_working_dir(), constants.mergedir)
    workdir_path = os.path.join(get_working_dir(), constants.workdir)
    root_command(remove, mergedir_path)
    root_command(remove, workdir_path)

def recover():
    '''Recover after some external error'''
    # try to unmount proc, sys and dev first
    try:
        undo_mount()
    except subprocess.CalledProcessError:
        pass
    try:
        unmount_rootfs()
    except subprocess.CalledProcessError:
        pass
    # clean up the working directory
    clean_up()

def calc_fs_hash(fs_path):
    '''Given the path to the filesystem, calculate the filesystem hash
    We run a shell script located in the tools directory to get the
    file stats and the file content's sha256sum. We then calculate the
    sha256sum of the contents and write the contents in the layer
    directory.
    Note that this file will be deleted if the -k flag is not given'''
    try:
        fs_hash_path = pkg_resources.resource_filename(
            "tern", "tools/fs_hash.sh")
        hash_contents = root_command(
            [fs_hash_path], os.path.abspath(fs_path))
        file_name = hashlib.sha256(hash_contents).hexdigest()
        # write file to an appropriate location
        hash_file = os.path.join(os.path.dirname(fs_path), file_name) + '.txt'
        with open(hash_file, 'w') as f:
            f.write(hash_contents.decode('utf-8'))
        return file_name
    except subprocess.CalledProcessError:  # pylint: disable=try-except-raise
        raise


"""Валидация полученного json через схему и запись в базу."""

import jsonschema
import psycopg2
import json
from config import config

def read_json(file: str) -> dict:
    """Чтение из файла в словарь."""
    with open(file, "r", encoding="utf-8") as json_file:
        data = json.load(json_file)
        return data

input_json = read_json("input_data.json")
validation_schema = read_json("goods_schema.json")

def validate(file: dict, schema: dict) -> bool:
    """Валидация входного файля через схему."""
    try:
        jsonschema.validate(file, schema)
        return True
    except jsonschema.exceptions.ValidationError:
        return False

def prepare_tuple(input_dict: dict) -> tuple:
    """
    Убирает вложенность и формирует кортежи для последующей записи в БД.

    Для каждой таблицы создаётся отдельный кортеж.
    """
    goods = []
    shops_goods = []
    for key, value in input_dict.items():
        if key == "id":
            goods.append(value)
            shops_goods.append(value)
        if key == "name":
            goods.append(value)
        if isinstance(value, dict):
            sub_element = value
            for sub_key, sub_value in sub_element.items():
                if sub_key in ["width", "height"]:
                    goods.append(sub_value)
        if isinstance(value, list):
            for sub_element in value:
                for sub_key, sub_value in sub_element.items():
                    if sub_key in ["location", "amount"]:
                        shops_goods.append(sub_value)
    return tuple(goods), tuple(shops_goods)

goods_list, shops_list = prepare_tuple(input_json)

create_query = """create table if not exists goods
(id int unique not null primary key, name varchar not null,
package_height float not null, package_width float not null);
create table if not exists shops_goods
(id serial not null, id_good int unique not null references goods (id) unique,
location varchar not null, amount int not null);
COMMENT on column goods.id IS 'уникальный идентификатор товара';
COMMENT on column goods.name IS 'наименование товара';
COMMENT on column goods.package_height IS 'высота упакованного товара';
COMMENT on column goods.package_width IS 'ширина упакованного товара';
COMMENT on column shops_goods.id IS 'идентификатор записи';
COMMENT on column shops_goods.id_good IS 'идентификатор товара';
COMMENT on column shops_goods.location IS 'адрес магазина';
COMMENT on column shops_goods.amount is 'количество этого товара в этом магазине';"""

add_goods_query = f"""Insert into goods
(id, name, package_width, package_height) values {goods_list}
ON CONFLICT(id) do update
set name = excluded.name, package_width = excluded.package_width, package_height = excluded.package_height;"""

add_shops_query = f"""Insert into shops_goods (id_good, location, amount) values {shops_list}
                    ON CONFLICT(id_good) do update
                    set
                    location = excluded.location,
                    amount = excluded.amount;"""

def write_db() -> str:
    """Подключение к БД и выполнение запросов."""
    conn = None
    try:
        params = config()
        print("Подключение к базе данных...")
        conn = psycopg2.connect(**params)
        db_cursor = conn.cursor()
        print("Выполнение запроса...")
        db_cursor.execute(create_query)
        db_cursor.execute(add_goods_query)
        db_cursor.execute(add_shops_query)
        print("Запрос выполнен успешно.")
        conn.commit()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()
            print("Соединение закрыто.")
    return "Success"

def main() -> str:
    """Проверка валидации, запись в БД."""
    if validate(input_json, validation_schema):
        write_db()
        return "Success"
    else:
        print("Входной json-файл не валиден")
        return "non valid json"

main()


#!/usr/bin/env python
#from distutils.core import setup
from setuptools import setup, find_packages
import glob
import os

home = os.path.expanduser("~")
setup(
name='cloudmesh_auth',
version = '0.1',
description='A secure rest framework for Cloudmesh',
packages=['cloudmesh_auth']
)


import fire

import benchmarker as B
import random_tester as R
import schema_generator_pipeline as S

def benchmark(test_suite_filename, description, algo, iterations=1):
    """
    Perform a benchmark on test suite from a file.
    :param test_suite_filename: file to take tests from
    :param description: a string to be stored in benchmark
    results for easier understanding of the change made
    :param algo: one of `brute`, `fences`
    :param iterations: how many times to run the benchmark
    to reduce runtime drift (default 1)
    """
    B.benchmark(test_suite_filename, description, algo, iterations)
    pass

def run_fixed(filename, max_size, algo):
    """
    Run and validate one test from a file.
    :param filename: name of the file with one defined test
    :param max_size: maximum size of generated schema, in
    :param algo: one of `brute`, `fences`
    additional nodes
    """
    S.run_fixed(filename, max_size, algo)

def run_random(n, m, amount, max_size, algo):
    """
    Run and validate `amount` tests where size of each
    test does not exceed `n` in variables and `m` in
    functions.
    :param n: max number of parameters for boolean function
    :param m: max number of boolean functions
    :param amount: number of tests to generate
    :param max_size: maximum size of generated schema, in
    :param algo: one of `brute`, `fences`
    additional nodes
    """
    R.run_random(n, m, amount, max_size, algo)

def compare_on_random(n, m, amount, max_size):
    """
    """
    R.compare_on_random(n, m, amount, max_size)

if __name__ == "__main__":
    # import cProfile, pstats
    # profiler = cProfile.Profile()
    # profiler.enable()
    fire.core.Display = lambda lines, out: print(*lines, file=out)
    fire.Fire({
        "benchmark": benchmark,
        "run_fixed": run_fixed,
        "run_random": run_random,
        "compare_on_random": compare_on_random
    })
    # profiler.disable()
    # stats = pstats.Stats(profiler).sort_stats('cumtime')
    # stats.print_stats()
    # filename = 'profile.prof'
    # profiler.dump_stats(filename)


#!/usr/bin/env python
# coding: utf-8

import serial
import numpy as np
import matplotlib.pyplot as plt
import linecache
import datetime
import csv
import time
import smtplib
import sys
import getpass

import adafruit_max31865
import Adafruit_DHT as dht
import board
import busio
import digitalio
import os.path
from string import Template
from email.message import EmailMessage
from email.mime.image import MIMEImage

class DataAQ():

    def __init__(self, filename='abc.csv',salinity = 30):
        self.filename = filename
        self.name=self.filename.split('.')[0]
        self.My_Email = input('Please provide the sending email address:\n')
        self.Pass = getpass.getpass(prompt='Please provide the sending email password:\n')
        self.Check_Login()

    def Check_Login(self):
        try:
            server=smtplib.SMTP('smtp.gmail.com', 587, timeout=300)
            server.starttls()
            server.login(self.My_Email, self.Pass)
            server.quit()
            print('Login to email server successful!')
        except:
            print('Login to email server failed! Please enter the correct credientials.')
            sys.exit()

    def Sartorius_init(self,COM):
        #Initialize Sartorius Scale
        ser = serial.Serial(port = '/dev/serial/by-id/usb-Sartorius_Sartorius_Composite_device_0000000000-if00', baudrate = 9600, parity = serial.PARITY_ODD, stopbits = serial.STOPBITS_ONE, bytesize = serial.EIGHTBITS, timeout = None)
        self.ser = ser

    def RTD_init(self):
        #Initalize RTP100 sensor to GPIO5
        spi = busio.SPI(board.SCK, MOSI=board.MOSI, MISO=board.MISO)
        cs = digitalio.DigitalInOut(board.D5)
        rtdsensor = adafruit_max31865.MAX31865(spi,cs, wires = 4)
        return rtdsensor

    def Scale_Value(self):
        self.ser.write(b"\x1bP\r\n")
        weight = self.ser.readline()
        weight = str(weight)
        weight=weight[11:18]
        weight = float(weight)
        return weight

    def get_contacts(self,file):
        names=[]
        emails=[]
        with open(file, mode='r') as contacts:
            for a_contact in contacts:
                names.append(a_contact.split()[0])
                emails.append(a_contact.split()[1])
        return names, emails

    def read_template(self, file):
        with open(file, 'r') as template_file:
            template_file_content = template_file.read()
        return Template(template_file_content)

    def send_email(self,wchange):

        names,emails = self.get_contacts('lab_contacts.txt')
        message_template = self.read_template('Regular_Message.txt')

        server=smtplib.SMTP('smtp.gmail.com', 587, timeout=300)
        server.starttls()
        server.login(self.My_Email, self.Pass)

        for name, email in zip(names, emails):
            self.simple_plot()
            msg = EmailMessage()
            message = message_template.substitute(PERSON_NAME=name.title(),WEIGHT_CHANGE=wchange)
            msg['From'] = self.My_Email
            msg['To']= email
            msg['Subject'] = self.name
            msg.set_content(message)
            msg.add_attachment(open(self.filename, 'r').read(), filename = self.filename)

            pdfopen=open(f'{self.name}.pdf','rb').read()
            image=MIMEImage(pdfopen, _subtype='pdf', name=os.path.basename(f'{self.name}.pdf'))
            msg.attach(image)
            server.send_message(msg)
            del msg
        print(f'Email(s) sent! Data Aquisition in progress @ {datetime.datetime.now()}!')

    def send_final_email(self):
        names,emails = self.get_contacts('lab_contacts.txt')
        message_template = self.read_template('Final_Message.txt')

        server=smtplib.SMTP('smtp.gmail.com', 587, timeout=300)
        server.starttls()
        server.login(self.My_Email, self.Pass)

        for name, email in zip(names, emails):
            self.simple_plot()
            msg = EmailMessage()
            message = message_template.substitute(PERSON_NAME=name.title())
            msg['From'] = self.My_Email
            msg['To']= email
            msg['Subject'] = self.name
            msg.set_content(message)
            msg.add_attachment(open(self.filename, 'r').read(), filename = self.filename)

            pdfopen=open(f'{self.name}.pdf','rb').read()
            image=MIMEImage(pdfopen, _subtype='pdf', name=os.path.basename(f'{self.name}.pdf'))
            msg.attach(image)
            server.send_message(msg)
            del msg
        print(f'Final Email(s) Sent!')
        server.quit()

    def simple_plot(self):
        plottime, real_weight, plottemp, adj_weight = np.loadtxt(self.filename,skiprows=3,delimiter=',', usecols=(1,2,3,6)).T
        fig, ax = plt.subplots(figsize=(8,6), dpi=80)
        ax.plot(plottime,real_weight,'b-',label='Recorded_Weight')
        ax.plot(plottime,adj_weight,'g--',label='Temp Corrected Weight')
        ax.set_ylabel('Weight(g)',fontsize=10)
        ax.set_xlabel('Time (hours)',fontsize = 10)

        ax2 = ax.twinx()
        ax2.set_ylabel('Temperature (C)')
        ax2.plot(plottime,plottemp,'r')

        ax.legend()
        plt.title(self.name,fontsize = 18)
        if os.path.isfile(f'{self.name}.pdf'):
            os.remove(f'{self.name}.pdf')
        plt.savefig(f'{self.name}.pdf')
        plt.close()

    def Density_Calc(self,temp):
        if self.salinity == 'C12':
            a = -.731*10**-3
            b = -.32*10**-6
            TrueDens = .75 + a*(temp-20) + b/2*(temp-20)**2
        else:
            CleanDens = 1000 * (1 - (temp + 288.9414) / (508929.2 * (temp + 68.12963)) * (temp - 3.9863) ** 2)
            A = 0.824493 - 0.0040899 * temp + 0.000076438 * temp ** 2 - 0.00000082467 * temp ** 3 + 0.0000000053675 * temp ** 4
            B = -0.005724 + 0.00010227 * temp - 0.0000016546 * temp ** 2
            C = 0.00048314
            TrueDens = (CleanDens + A * self.salinity + B * self.salinity ** (3 / 2) + C * self.salinity ** 2) / 1000
        return TrueDens

    def Temp_Correction(self,temp,RefD,raw_weight):
        TrueDens = self.Density_Calc(temp)
        TrueWeight = raw_weight + (TrueDens-RefD)*self.Vs
        return TrueWeight

    def Error(self,temp, RefD, raw_weight):
        #Error Calculation for RTD Sensor
        Temp_Error = (.15 + temp*.002)
        High_Temp = temp + Temp_Error
        Low_Temp = temp - Temp_Error
        High_Weight_Error = self.Temp_Correction(High_Temp,RefD,raw_weight)
        Low_Weight_Error = self.Temp_Correction(Low_Temp,RefD,raw_weight)
        return High_Weight_Error, Low_Weight_Error

    def Aquire(self):
        count = 0
        ecount = 0
        plt.show(block=True)
        rtdsensor=self.RTD_init()
        self.Sartorius_init('/dev/ttyUSB0')
        if os.path.isfile(self.filename):
            answer = input('File already exists! Would you like to append to the exsisting file? (Y/N) \n')
            if answer == 'Y':
                wfile=open(f'{self.filename}', mode='a+')
                writer = csv.writer(wfile, lineterminator = '\n')
                variable_line=linecache.getline(self.filename, 2).split(',')
                self.salinity=float(variable_line[1])
                self.Vs=float(variable_line[3])
                data_line=linecache.getline(self.filename,4).split(',')
                #line_time=data_line[0].split('.')
                starttime=datetime.datetime.strptime(data_line[0], '%Y-%m-%d %H:%M:%S')
                refD=self.Density_Calc(float(data_line[3]))
            else:
                sys.exit('Answer was not Y! Cancel DataAQ()')
        else:
            starttime=datetime.datetime.now().replace(microsecond=0)
            self.salinity = float(input('Please provide salinity of the brine (mg/L):\n'))
            self.Vs = float(input('Please provide an estimate for the total solid volume submerged (cm^3):\n'))
            refD=self.Density_Calc(rtdsensor.temperature)
            wfile=open(f'{self.filename}', mode='a+')
            writer = csv.writer(wfile, lineterminator = '\n')
            writer.writerow([f'Experiment: {self.name}'])
            writer.writerow(['Brine_Salinity/C12',self.salinity,'Vs',self.Vs])
            writer.writerow(['Time','Relative Time (Hours)','Weight (g)','Liq Temperature (C)','Air Temperature (C)','Humidity','Temp Adjusted Weight (g)', '(+)Error from RTD sensor (g)','(-)Error from RTD sensor (g)','Comments'])
        initialweight=self.Scale_Value()
        print('Data Aquisition in progress!')
        while True:
            timestamp = datetime.datetime.now().replace(microsecond=0)
            relativetime = (timestamp-starttime)/datetime.timedelta(hours=1)
            weight = self.Scale_Value()
            Liq_Temperature = rtdsensor.temperature
            Humidity, Air_Temperature = dht.read_retry(dht.DHT22, 4)
            AdjWeight = self.Temp_Correction(Liq_Temperature,refD,weight)
            PlusError, MinusError = self.Error(Liq_Temperature, refD, weight)
            writer.writerow([timestamp,relativetime,weight,Liq_Temperature,Air_Temperature, Humidity, AdjWeight, PlusError, MinusError])
            #needs to be 15,30,45,60 sec interval)
            time.sleep(60)
            count+=1
            ecount+=1
            #log every 30 min count==120,60,40,30
            if count==60:
                wfile.close()
                #email every 4 hrs ecount==960,480,320,240
                print(f'DATA LOGGED @ {datetime.datetime.now().replace(microsecond=0)}!')
                if ecount >= 240:
                    try:
                        self.send_email(weight-initialweight)
                        initialweight=self.Scale_Value()
                        ecount = 0
                    except:
                        print('Email(s) not sent! Still recording data!')
                wfile=open(f'{self.filename}', mode='a+')
                writer = csv.writer(wfile, lineterminator = '\n')
                count=0

if __name__ == '__main__':
    try:
        print('Program for Spontaneous Imbibtion Experiments\nThis program is for use with a Sartorius Scale and RTD temperature sensor.\nEmail updates will be sent every 4 hours from an email the users specifies\n')
        filename=input('Please type destination filename with .csv \n')
        if filename[-4:]=='.csv':
            Experiment=DataAQ(filename)
            Experiment.Aquire()
        else:
            print('You did not add ".csv" to the end of the file!')
    except KeyboardInterrupt:
        print('Data Aquisition Complete!')
        plt.show(block=False)
        wfile=open(filename, mode='a+')
        wfile.close()
        final=input('Is this experiment complete? (Y/N)')
        if final == 'Y':
            Experiment.send_final_email()
            print('Experiment Complete!')
        else:
            print('Data Aquisition has been paused!')


"""Helper class to create repository and it's respective dependency nodes in graph DB."""

import requests

from utils import GREMLIN_SERVER_URL_REST, fix_gremlin_output

class RepoDependencyCreator:
    """Finds out direct and indirect dependencies from a given github repository."""

    # TODO: refactor this static method so it will be possible to test it properly
    @staticmethod
    def create_repo_node_and_get_cve(github_repo, deps_list):
        """Create a repository node in the graphdb and create its edges to all deps.

        :param github_repo: git repository for scanning
        :param deps_list: dependency list for scanning
        """
        gremlin_str = ("repo=g.V().has('repo_url', '{repo_url}').tryNext().orElseGet{{"
                       "graph.addVertex('vertex_label', 'Repo', 'repo_url', '{repo_url}')}};"
                       "g.V(repo).outE('has_dependency').drop().iterate();"
                       "g.V(repo).outE('has_transitive_dependency').drop().iterate();".format(
                        repo_url=github_repo))

        # Create an edge between repo -> direct dependencies
        for pkg in deps_list.get('direct'):
            if len(pkg.split(':')) == 4:
                ecosystem, group_id, artifact_id, version = pkg.split(':')
                name = group_id + ':' + artifact_id if group_id and artifact_id \
                    else ''
            else:
                ecosystem, name, version = pkg.split(':')

            gremlin_str += ("ver=g.V().has('pecosystem', '{ecosystem}').has('pname', '{name}')."
                            "has('version', '{version}');ver.hasNext() && "
                            "g.V(repo).next().addEdge('has_dependency', ver.next());".format(
                                ecosystem=ecosystem, name=name, version=version))

        # Create an edge between repo -> transitive dependencies
        for pkg in deps_list.get('transitive'):
            if len(pkg.split(':')) == 4:
                ecosystem, group_id, artifact_id, version = pkg.split(':')
                name = group_id + ':' + artifact_id if group_id and artifact_id \
                    else ''
            else:
                ecosystem, name, version = pkg.split(':')

            gremlin_str += ("ver=g.V().has('pecosystem', '{ecosystem}').has('pname', '{name}')."
                            "has('version', '{version}');ver.hasNext() && "
                            "g.V(repo).next().addEdge('has_transitive_dependency', ver.next());"
                            .format(ecosystem=ecosystem, name=name, version=version))

        # Traverse the Repo to Direct/Transitive dependencies that have CVE's and report them
        gremlin_str += (
            "g.V(repo).as('rp').outE('has_dependency','has_transitive_dependency')"
            ".as('ed').inV().as('epv').out('has_cve').as('cve')"
            ".select('rp','ed','epv','cve').by(valueMap(true));"
        )
        payload = {"gremlin": gremlin_str}
        try:
            rawresp = requests.post(url=GREMLIN_SERVER_URL_REST, json=payload)
            resp = rawresp.json()
            if rawresp.status_code != 200:
                raise Exception("Error creating repository node for {repo_url} - "
                                "{resp}".format(repo_url=github_repo, resp=resp))

        except Exception:
            raise Exception(
                "Error creating repository node for {repo_url}".format(repo_url=github_repo))

        return fix_gremlin_output(resp)

    @staticmethod
    def generate_report(repo_cves, deps_list):
        """
        Generate a json structure to include cve details for dependencies.

        :param repo_cves: CVEs found in the repository
        :param deps_list: Dependency list
        :return: list
        """
        repo_list = []
        for repo_cve in repo_cves.get('result').get('data', []):
            epv = repo_cve.get('epv')
            repo_url = repo_cve.get('rp').get('repo_url')[0]
            name = epv.get('pname')[0]
            version = epv.get('version')[0]
            ecosystem = epv.get('pecosystem')[0]
            str_epv = ecosystem + ":" + name + ":" + version
            cves = repo_cve.get('cves', [])
            vulnerable_deps = []
            first = True
            if cves and (str_epv in i for x, i in deps_list.items()):
                cve_list = []
                for cve in cves:
                    cve_list.append(
                        {'CVE': cve.get('cve_id')[0], 'CVSS': cve.get('cvss_v2', [10.0])[0]}
                    )
                vulnerable_deps.append({
                    'ecosystem': epv.get('pecosystem')[0],
                    'name': epv.get('pname')[0],
                    'version': epv.get('version')[0],
                    'cve_count': len(cves), 'cves': cve_list,
                    'is_transitive': repo_cve.get('ed').get('label') == 'has_transitive_dependency'
                })

            for repo in repo_list:
                if repo_url == repo.get('repo_url'):
                    repo_vul_deps = repo.get('vulnerable_deps')
                    repo['vulnerable_deps'] = vulnerable_deps + repo_vul_deps
                    first = False
            if first:
                repo_list.append({'repo_url': repo_url, 'vulnerable_deps': vulnerable_deps})

        return repo_list


from inspect import stack
import streamlit as st
import pandas as pd
from main import query_db
import matplotlib.pyplot as plt
import seaborn as sns
from connection.sql_utils import flg_country_agg

df_report=query_db()

st.title("Challenge - Python Data Engineer")
st.write("Hey, welcome to the Python Data Engineer code challenge. In this challenge, we are interested in seeing your knowledge about data management and visualizations. We will give you some data, and your final objective is to show us specific metrics in some chart visualizations.")
st.write("The visualizations that we are expecting are:")

# Plotting data on pie chart
# ----------------------------------------------------------------------------------
st.write("Hired by technology")
fig = plt.figure(figsize= (12,9))
dfg= df_report.groupby("technology")["status_candidate"].count()
plt.pie(list(dfg.values), labels =list(dfg.index), autopct='%.1f%%')
st.pyplot(fig)
# ----------------------------------------------------------------------------------

total = len(df_report)
# Plotting data on horizontal bar chart
# ----------------------------------------------------------------------------------
st.write("Hired by year")
fig = plt.figure(figsize= (10,4))
dfg= pd.DataFrame({'count':df_report.groupby(["year_application"]).size()}).reset_index()
dfg['percentage']=(dfg['count']/total)*100
ax=sns.barplot(x =dfg['count'],y=dfg['year_application'],data=dfg,orient='h')
for i, v in enumerate(dfg['percentage']):
    ax.text(v + 0, i + 0.15, str("{0:.2f}%".format(v)), color='black', fontsize=8)
st.pyplot(fig)
# ----------------------------------------------------------------------------------
#https://stackoverflow.com/questions/54687788/adding-percentage-values-onto-horizontal-bar-charts-in-matplotlib

# Plotting data on bar chart
# ----------------------------------------------------------------------------------
st.write("Hired by seniority")
fig = plt.figure(figsize= (10,5))
ax=sns.countplot(data=df_report,x='seniority',order=df_report['seniority'].value_counts().index)
for p in ax.patches:
    percentage = f'{100 * p.get_height() / total:.1f}%\n'
    x = p.get_x() + p.get_width() / 2
    y = p.get_height()
    ax.annotate(percentage, (x, y), ha='center', va='center')
plt.tight_layout()
st.pyplot(fig)
# ----------------------------------------------------------------------------------

# Plotting data on Hires by country over years
# ----------------------------------------------------------------------------------
filter_countries=['United States of America','Brazil','Colombia','Ecuador']

st.write("Hires by country over years")
fig = plt.figure(figsize= (10,5))
df_report['flg_country'] = df_report['country'].apply(lambda x: flg_country_agg(x))
df_report=df_report.loc[df_report['country'].isin(filter_countries)]
table=pd.crosstab(df_report['year_application'],df_report['flg_country'])
table=table.div(table.sum(1).astype(float), axis=0)
ax=table.plot.bar(stacked=True,legend=None).figure
ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05),ncol=4)
st.pyplot(ax)
# ----------------------------------------------------------------------------------


from aiogram import types, Dispatcher
from config import bot, dp

@dp.message_handler()
async def echo(message: types.Message):
    try:
        await message.text.isdigit ** 2
    except:
        await message.answer(message.text)

async def dice_game():
    message.answer(dice_game())

def register_handlers_extra(dp: Dispatcher):
    dp.register_message_handler(echo)


import numpy as np
import math
from pendulumCart import pendulumCart
from Controllers.pidController import pidController
import matplotlib.pyplot as plt

DT = 1/1000

class manualControl(pendulumCart,pidController):
    """
    This is a trial class to see if the system is controllable
    This has a simple PID controller that attempts to bring the pendulum to the
    top and make sure it stays there.
    """
    def __init__(self):
        pendulumCart.__init__(self,cartMass=0.5, pendulumMass=0.2,pendulumLength=0.3,
                               damping = 0.1, springStiffness=0,timeStep=DT,
                               initialStateVector=[0,0,5*math.pi/180,0])
        pidController.__init__(self,kp = 10,ki = 0.10, kd = 1, dt = DT)
        self.currentTime = 0
        self.targetPosition = 0

        # Initializing graphs and setting the axes titles
        self.fig,self.axs = plt.subplots(3,1,sharex=True)
        self.axs[0].set_title("Pendulum position vs time")
        self.axs[1].set_title("Cart position vs time")
        self.axs[2].set_title("Force vs time")
        self.axs[2].set_xlabel("Time")
        self.axs[0].set_ylabel("Angular position (deg)")
        self.axs[1].set_ylabel("Cart position (m)")
        self.axs[2].set_ylabel("Force(N)")

    def systemUpdate(self):
        actuatorSignal = self.computeOutput(self.targetPosition,self.pendulumPosition)
        self.systemSolver(self.currentTime,actuatorSignal)
        return actuatorSignal

    def plotGraph(self,time,pendulumAngle,cartPos,force):
        self.axs[0].plot(time,pendulumAngle)
        self.axs[1].plot(time,cartPos)
        self.axs[2].plot(time,force)
        plt.show()

    def runSystem(self):
        time,pendulumAngle,cartPos,force = [0],[180],[0],[0]
        while self.currentTime < 3:
            f = self.systemUpdate()
            temp = math.degrees(self.pendulumPosition)
            if temp < 0:
                temp+=360
            if 359 < temp < 361:
                temp = 0
            time.append(self.currentTime)
            pendulumAngle.append(temp)
            cartPos.append(self.cartPosition)
            force.append(f)
            # self.plotGraph()
            # self.fig.canvas.draw()
            self.currentTime+=DT
        self.plotGraph(time,pendulumAngle,cartPos,force)

mc = manualControl()
mc.runSystem()
print("Done")


import random
import string

import pytest

from mlops_utilities import helpers
from mlops_utilities.actions import run_pipeline, upsert_pipeline

class TestPackageActions:
    @pytest.mark.skip("FIXME")
    def test_upsert_pipeline(self):
        upsert_pipeline(
            pipeline_module="pipelines",
            pipeline_package="training_pipeline",
            pipeline_name="test_pipeline",
            config_type="training_pipeline.defaults",
            role="arn:aws:iam::123456789000:role/AmazonSageMaker-ExecutionRole",
            dryrun=True,
        )

    def test_run_pipeline(self):
        run_pipeline(
            pipeline_name="test_pipeline",
            execution_name_prefix="test_pipeline",
            pipeline_params={
                "RegisterModel": False,
                "InputDataS3Uri": f"s3://mlops-sagemaker-project/abalonedata/data.csv",
                "InputLabelS3Uri": f"s3://mlops-sagemaker-project/abalonelabels/data.csv",
            },
            dryrun=True,
        )

class TestHelpers:
    def test_convert_param_dict_to_key_value_list_wrong_input(self):
        with pytest.raises(AttributeError):
            helpers.convert_param_dict_to_key_value_list(None)

    def test_convert_param_dict_to_key_value_list_empty_input(self):
        assert helpers.convert_param_dict_to_key_value_list({}) == []

    def test_convert_param_dict_to_key_value_list_correct_input(self):
        assert helpers.convert_param_dict_to_key_value_list(
            {
                "key_1": "v",
                "key_2": "vv",
                "key_3": "1",
            }
        ) == [
                   {"Key": "key_1", "Value": "v"},
                   {"Key": "key_2", "Value": "vv"},
                   {"Key": "key_3", "Value": "1"},
               ]

    def test_convert_param_dict_to_name_value_list_correct_input(self):
        assert helpers.convert_param_dict_to_name_value_list(
            {
                "key_1": "v",
                "key_2": "vv",
                "key_3": "1",
            }
        ) == [
                   {"Name": "key_1", "Value": "v"},
                   {"Name": "key_2", "Value": "vv"},
                   {"Name": "key_3", "Value": "1"},
               ]

    def test_get_value_from_dict(self):
        test_dict = {"regression_metrics": {"mse": {"value": 4.9}}}
        path = ["regression_metrics", "mse", "value"]
        assert helpers.get_value_from_dict(test_dict, path) == 4.9

    def test_get_model_name(self):
        test_model_arn = "arn:aws:sagemaker:us-east-1:123456789000:model-package-group/framework-version"
        assert helpers.get_model_name(test_model_arn) == "framework-version"

    def test_get_job_name(self):
        test_job_arn = "arn:aws:sagemaker:us-east-1:123456789000:processing-job/pipelines-joqqfahmal9e-splitdataset-ikz14ulv8j"
        assert (
                helpers.get_job_name(test_job_arn)
                == "pipelines-joqqfahmal9e-splitdataset-ikz14ulv8j"
        )

    def test_ensure_min_length(self):
        test_token = "".join(random.choices(string.ascii_letters, k=15))
        assert helpers.ensure_min_length(test_token, 17) == test_token + "_ 00"

    def test_normalize_pipeline_name(self):
        pipeline_test_name = "".join(random.choices(string.ascii_letters, k=83))
        assert len(helpers.normalize_pipeline_name(pipeline_test_name)) == 82

    def test_generate_data_capture_config(self):
        assert helpers._generate_data_capture_config("test_prefix") == {
            "EnableCapture": True,
            "InitialSamplingPercentage": 100,
            "DestinationS3Uri": "test_prefix",
            "CaptureOptions": [
                {"CaptureMode": "Input"},
                {"CaptureMode": "Output"},
            ],  # both by default
            "CaptureContentTypeHeader": {"CsvContentTypes": ["text/csv"]},
        }


from skimage.metrics import structural_similarity
import cv2
import numpy as np

import sys
import os

def compare(path1, path2):
    first = cv2.imread(path1)
    second = cv2.imread(path2)

    if first.shape != second.shape:
        # change to size of smaller one
        if first.shape[0]*first.shape[1] < second.shape[0] * second.shape[1]:
            first, second = second, first
        second = cv2.resize(second, (first.shape[1], first.shape[0]), interpolation = cv2.INTER_AREA)

    # Compute SSIM between two images
    score, diff = structural_similarity(first, second, full=True, channel_axis=2)
    return score

source_image = sys.argv[2]
path_of_images = sys.argv[1]

mx = 0
curr_best = ""

options = []

for filename in os.listdir(path_of_images):
    if filename.endswith('.jpg'):
        curr = compare(path_of_images + filename, source_image)

        options.append((curr,filename))
        # if curr > mx:
        #     mx = curr
        #     curr_best = filename

options.sort()
options.reverse()
print()
print()
print(path_of_images + options[0][1])
print("Similarity Score: {:.3f}%".format(options[0][0] * 100))

print(options[0:5])


import json
import backend
import requests

def scan_book(rfid):

    parameters = {
        'rfid': rfid,
    }

    print(parameters)

    response = backend.request('scan_book', data=parameters)
    print(response)
    jsonobject = json.loads(response.text)

    if jsonobject["error"]:
        raise ConnectionError('Feil i databasen: ' + jsonobject["error"])

    #Check for error by each book, for instance that the book was not found in the database
    for i in jsonobject["status"]:
        if i["error"]:
            raise ConnectionError('Feil i databasen: ' + i["error"])

    return jsonobject

def get_book_info(bookrfid):
    parameters = {
        'rfid': bookrfid,
    }

    response = backend.request('get_book_info', data=parameters)
    jsonobject = json.loads(response.text)

    if jsonobject["error"]:
        raise ConnectionError('Feil i databasen: ' + jsonobject["error"])

    return jsonobject

def give_feedback(rfid, userrfid, ratingtype, value):
    #ratingtype "star"
    parameters = {
        'rfid': rfid,
        'userrfid': userrfid,
        'type': ratingtype,
        'value': value,
    }

    response = backend.request('give_feedback', data=parameters)
    jsonobject = json.loads(response.text)

    if jsonobject["error"]:
        raise ConnectionError('Feil i databasen: ' + jsonobject["error"])

    return jsonobject


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun May 20 16:50:56 2018

@author: user
"""

import sys, os
import pandas as pd
from importlib import reload
from bs4 import BeautifulSoup
import urllib.request
from tqdm import tqdm
import PyPDF2
import numpy as np

import grimsel.auxiliary.timemap as timemap
import grimsel.auxiliary.aux_sql_func as aql

import PROFILE_READER.profile_reader as profile_reader

reload(profile_reader)

# %% Complement days

class ProfileYearAdder(profile_reader.ProfileReader):
    '''
    Adds missing years to profile tables and complements gaps in the existing
    years' data.
    Makes use of some ProfileReader methods, but doesn't call its __init__.
    '''

    all_yr = list(range(2005, 2018))

    data_dir = ''

    dict_sql_default = dict(sc='profiles_raw',
                            tb='load_complete',
                            db='storage2')

    tb_cols = [('"DateTime"', 'TIMESTAMP'),
                    ('nd_id', 'VARCHAR'),
                    ('value', 'DOUBLE PRECISION'),
                    ('hy', 'SMALLINT'),
                    ('year', 'SMALLINT')]
    tb_pk = ['nd_id', 'year', 'hy']

    def __init__(self, kw_dict, tb='entsoe_load', filt=[]):
        ''' Sets up the object. Doesn't run the methods. '''

        super().__init__(**kw_dict)

        self.tb = tb
        self.filt = filt

    def read_data(self):

        self.df = aql.read_sql(self.dict_sql['db'], 'profiles_raw',
                               self.tb, filt=self.filt)

        # delete incomplete days
        self.df['doy'] = self.df.DateTime.dt.dayofyear
        slct_incomp = self.df.pivot_table(values='value',
                                          index=['year', 'doy'],
                                          aggfunc=len)
        slct_incomp = slct_incomp.loc[pd.DataFrame(slct_incomp)['value'] < 24].index.values
        self.df = self.df.loc[-self.df[['year', 'doy']]
                                   .apply(tuple, axis=1)
                                   .isin(slct_incomp)]

        # delete
        self.df = self.filter_leap_day(self.df)

        # there must be only one nd_id
        self.nd_id = self.df.nd_id.unique()[0]

    def add_time_cols(self):

        self.df_tot['year'] = self.df_tot.DateTime.dt.year
        self.df_tot['dow'] = self.df_tot.DateTime.dt.dayofweek
        self.df_tot['wk'] = self.df_tot.DateTime.dt.week
        self.df_tot['hour'] = self.df_tot.DateTime.dt.hour
        self.df_tot['mt'] = self.df_tot.DateTime.dt.month
        self.df_tot['day'] = self.df_tot.DateTime.dt.day

    def normalize_value_column(self):
        ''' Normalizes the value column to one for each year separately. '''

        self.df_tot['value'] = (self.df_tot.groupby('year')['value']
                                           .apply((lambda x: x/x.sum())))

    def init_df_tot(self):
        '''
        Merge the timemap for all years with the input data.
        This yields the total table with the gaps to be filled subsequently.
        '''

        self.df_tot = (self.tm.df_time_map[['DateTime', 'dow_name']]
                              .join(self.df.set_index('DateTime'),
                                    on='DateTime'))
        self.df_tot = self.df_tot.reset_index(drop=True)

        self.normalize_value_column()

        self.df_tot['nd_id'] = self.nd_id

        self.add_time_cols()

        # if the year's last partial week is equal to 1,
        # set its value to max + 1;
        # likewise, if the year's first partial week is in [52, 53],
        # set its value to 0
        def fix_last_week_index(df):
            if df.iloc[0]['mt'] == 12 and 1 in df.wk.values:
                df.loc[df.wk == 1, 'wk'] = df.wk.max() + 1
            if df.iloc[0]['mt'] == 1 and (df.wk > 25).sum() > 0:
                df.loc[df.wk > 25, 'wk'] = 0
            return df
        self.df_tot = (self.df_tot.groupby(['year', 'mt'])
                                  .apply(fix_last_week_index))

    def init_df_pv(self):

        # pivot tables drop all-nan columns. therefore we set inexistant
        # values to 0. then the nan values in the pivot table are
        # invalid dates which can be disregarded.
        self.df_tot['value'] = self.df_tot.value.fillna(0)

        pv_kwargs = dict(values='value', index=['wk', 'dow', 'hour'],
                         columns='year')

        if self.df_tot.pivot_table(**pv_kwargs, aggfunc=len).max().max() > 1:
            raise ValueError('pivot_table aggregates multiple values.')
        self.df_pv = self.df_tot.pivot_table(**pv_kwargs, aggfunc=np.sum)

        pv_day_kwargs = dict(values='value', index=['wk', 'dow'], columns='year')
        self.df_pv_day = self.df_tot.pivot_table(**pv_day_kwargs, aggfunc=np.sum)

    def get_missing_years(self):
        df_max = self.df_pv.max()
        self.list_old = (df_max.loc[df_max > 0]
                               .index.get_level_values('year').tolist())
        self.list_new = (df_max.loc[df_max == 0]
                               .index.get_level_values('year').tolist())

    def fill_incomplete_by_day(self):
        '''
        For each day, select the existing year X which contains thee median
        day by total energy. Then copy the corresponding day profile of year X
        to all years where this day is missing.
        '''

        def get_median_day(x):
            # Note: not using median function to avoid averaging
            x_sort = x.loc[-x.isnull()].sort_values()
            med_yr = (x_sort.reset_index()
                            .iloc[int(len(x_sort) // 2)]
                            .loc['year']
                            .values[0])
            return med_yr

        self.df_pv_day['median_year'] = (self.df_pv_day[self.list_old]
                                             .apply(get_median_day, axis=1))

        dict_day_year = self.df_pv_day['median_year'].to_dict()

        # add per-day median year selection to df_pv
        self.df_pv = (self.df_pv.reset_index()
                                .join(self.df_pv_day['median_year'],
                                      on=self.df_pv_day.index.names)
                                .set_index(self.df_pv.index.names))

        # switching from nan to inf for invalid datetimes so we can use fillna
        # on the missing data
        self.df_pv[self.df_pv.isnull()] = np.inf
        self.df_pv[self.df_pv == 0] = np.nan

        fill_slct_day = lambda x: x.T.fillna(x[dict_day_year[x.name]].T).T
        self.df_pv = (self.df_pv.groupby(level=['wk', 'dow'])
                                .apply(fill_slct_day))

    def finalize_table(self):

        # stack final table by years
        self.dfnew = self.df_pv[self.list_old + self.list_new].stack().reset_index()
        self.dfnew.columns = self.df_pv.index.names + ['year', 'value']

        # remove invalid datetimes
        self.dfnew = self.dfnew.loc[-(self.dfnew.value == np.inf)]

        # add dfnew to df_tot
        self.df_tot = self.df_tot.rename(columns={'value': 'value_orig'}).join(self.dfnew.set_index(self.df_pv.index.names + ['year']), on=self.df_pv.index.names+ ['year'])

        self.df_tot['DateTime'] = self.df_tot.DateTime.dt.tz_localize('UTC')

        self.df_tot = self.get_hour_of_the_year(self.df_tot[['DateTime', 'nd_id', 'value', 'value_orig', 'year']])

        # re-add the time index columns which were lost due to the hy generation
        self.add_time_cols()

        # re-normalize all years to one
        self.normalize_value_column()

        print('Writing final table to database...', end='')
        self.append_to_sql(self.df_tot[[c for c in self.df_tot.columns
                                        if c in [col[0].replace('"', '')
                                        for col in self.tb_cols]]].copy())
        print('done.')

# %%

if __name__ == '__main__':
    pya = ProfileYearAdder('', [])#, filt=[('nd_id', ['DE0'])])
    self = pya

    for tb, filt in [('swissgrid_load', []),
                     ('terna_load', []),
                     ('rte_load', []),
                     ('econtrol_load', []),
                     ('entsoe_load', [('nd_id', ['DE0'])])]:

        self.tb = tb
        self.filt = filt

        self.read_data()
        self.init_tm(self.all_yr)
        self.init_df_tot()
        self.init_df_pv()
        self.get_missing_years()
        self.fill_incomplete_by_day()
        self.finalize_table()

# %%
if __name__ == '__main__':

    df = aql.read_sql('storage2', 'profiles_raw', 'entsoe_load', filt=[('nd_id', ['DE0'])])
    df.pivot_table(index='hy', values='value', columns='year', aggfunc=sum).plot()
    df.year.drop_duplicates()

    df = aql.read_sql('storage2', 'profiles_raw', 'terna_load')
    df.pivot_table(index='hy', values='value', columns='year', aggfunc=sum).plot()
    df.year.drop_duplicates()

    df = aql.read_sql('storage2', 'profiles_raw', 'econtrol_load')
    df.pivot_table(index='hy', values='value', columns='year', aggfunc=sum).plot()
    df.year.drop_duplicates()

    df = aql.read_sql('storage2', 'profiles_raw', 'rte_load')
    df.pivot_table(index='hy', values='value', columns='year', aggfunc=sum).plot()
    df.year.drop_duplicates()

    df = aql.read_sql('storage2', 'profiles_raw', 'swissgrid_load')
    df.pivot_table(index='hy', values='value', columns='year', aggfunc=sum).plot()
    df.year.drop_duplicates()

# %%
reload(profile_reader)

class EntsoeLoadReader(profile_reader.ProfileReader):
    '''
    '''

    dict_sql_default = dict(sc='profiles_raw', tb='entsoe_load')
    data_dir = os.path.normpath('DEMAND/ENTSOE/')

    tb_cols = [('"DateTime"', 'TIMESTAMP'),
               ('nd_id', 'VARCHAR'),
               ('value', 'DOUBLE PRECISION'),
               ('hy', 'SMALLINT'),
               ('year', 'SMALLINT')]
    tb_pk = ['nd_id', 'year', 'hy']

    exclude_substrings=[]

    def __init__(self, kw_dict):
        super().__init__(**kw_dict)

        self.get_fn_list()

    def read(self, fn):

        df_add = pd.read_csv(fn).fillna(0)

        df_add['DateTime'] = pd.to_datetime(df_add['Time (UTC)'].apply(lambda x: x.split(' - ')[0]))
        df_add['DateTime'] = df_add['DateTime'].dt.tz_localize('UTC')

        df_add = df_add.iloc[:, [2, -1]]
        df_add.columns = ['value', 'DateTime']
        df_add['nd_id'] = fn.split(os.path.sep)[-1].split('_')[0] + '0'

        df_add = self.time_resample(df_add, res='H')

        return df_add

    def post_processing(self, df):

        return self.filter_years_by_data_length(df)

if __name__ == '__main__':

    dict_sql = dict(db='storage2')

    kw_dict = dict(dict_sql=dict_sql,
                   exclude_substrings=[],
                   col_filt=[],
                   ext=['csv'])

    op = EntsoeLoadReader(kw_dict)

    self = op

    fn = self.fn_list[10]

    self.read_all()

    self.df_tot.loc[self.df_tot.nd_id == 'DE0'].pivot_table(values='value', index='hy', columns=['nd_id', 'year']).plot()

reload(profile_reader)

# %%

reload(profile_reader)

class TernaProfileReader(profile_reader.ProfileReader):
    '''
    '''

    dict_sql_default = dict(sc='profiles_raw', tb='terna_load')
    data_dir = os.path.normpath('DEMAND/ITALY/')

    tb_cols = [('"DateTime"', 'TIMESTAMP'),
               ('nd_id', 'VARCHAR'),
               ('value', 'DOUBLE PRECISION'),
               ('hy', 'SMALLINT'),
               ('year', 'SMALLINT')]
    tb_pk = ['nd_id', 'year', 'hy']

    exclude_substrings=[]

    def __init__(self, kw_dict):
        super().__init__(**kw_dict)

        self.get_fn_list()
        self.date_list = []
        self.date_fn_list = []

        yr_list = list(set([int(yy.split('.')[0].split(os.path.sep)[-1].split('_')[0].replace('Confronto', ''))
               for yy in self.fn_list_0]))

        self.init_tm(yr_list) # needs TimeMap for dst hours fixing

    def get_fn_list(self):
        '''
        Get htm files, parse the urls to the PDFs, download the PDFs and
        save the file names in the fn_list attribute.

        https://www.terna.it/it-it/sistemaelettrico/dispacciamento/datiesercizio/datigiornalieri/confronto.aspx
        '''

        super().get_fn_list()

        # switching from htm file list to pdf file list
        self.fn_list_0 = self.fn_list.copy()

        # get complete list of urls
        url_list = [url for fn0 in self.fn_list_0 for url in self.get_urls(fn0)]

        self.fn_list = self.download_files(url_list)

    def get_urls(self, fn):

        files = []

        soup = BeautifulSoup(open(fn))
        links = soup.find_all('a')

        for tag in links:
            link = tag.get('href',None)
            if link is not None:
                if 'download.terna' in link:
                    files.append(link)

        return files

    def download_files(self, file_list):
        fn_w_list = []
        for download_url in tqdm(file_list):

            fn_w = os.path.join(self._dir, '_'.join(download_url.split('/')[-2:]))

            if not os.path.exists(fn_w):

                response = urllib.request.urlopen(download_url)
                file = open(fn_w, 'wb')
                file.write(response.read())
                file.close()

            fn_w_list.append(os.path.join(self._dir, fn_w))

        return fn_w_list

    def parse_pdf(self, fn):
        '''
        Seems to require different styles depending on the year.

        date and time formats change over the years.
        '''

        pdfFileObj = open(fn, 'rb')
        pdfReader = PyPDF2.PdfFileReader(pdfFileObj)
        pageObj = pdfReader.getPage(0)
        str_text = pageObj.extractText()

        time = np.array([])
        cons = np.array([])
        prev = np.array([])

        cc = 0

        dict_ital_months = {'dic': '12', 'nov': '11', 'ott': '10', 'set': '09',
                            'ago': '08', 'lug': '07', 'giu': '06', 'mag': '05',
                            'apr': '04', 'mar': '03', 'feb': '02', 'gen': '01'}

        # replace italian months with month number
        repl_months = lambda x: [x.replace(kk, vv)
                                 for kk, vv in dict_ital_months.items()
                                 if kk in x][0]

        file_code = str_text[:6]
        print('File code: ', file_code)

        # skip invalid files
        if file_code in ['CDP Re']:
            return None

        if file_code in ['*OraPr', 'OraPre']:
            date = str_text[str_text.find(', ') + 2:
                            str_text.find('Confronto')]

            date = repl_months(date)
            day, month, year = [int(dd) for dd in date.split(' ')]

            data = str_text[str_text.find('eConsuntivo') + 11:
                            str_text.find('Scostamento Assoluto')]

            for ih in range(1,25):

                nhour = len(str(ih)) # number of digits of the hour

                # note: hours start at 1, hence shifted by one hour
                time = np.append(time, int(data[cc:cc + nhour]) - 1)
                cc += nhour
                cons = np.append(cons, data[cc:cc + 6].replace(',','').replace('.',''))
                cc += 6
                prev = np.append(prev, data[cc:cc + 6].replace(',','').replace('.',''))
                cc += 6

        if file_code == 'Report':
            date = str_text[str_text.find(' : ') + 3:
                            str_text.find(' : ') + 13]
            day, month, year = [int(dd) for dd in date.split('/')]

            ind_italia = str_text.find('ITALIA')

            if ind_italia > -1:
                data = str_text[str_text.find('ITALIA') + 6:]
            else:
                data = str_text[str_text.find('100%PREVISIONEORACONSUNTIVOGiorno di previsione :') +50: ]
            data = data.replace(date, '')
            data = data.replace(',', '')

            len_num = data.find(':') - 2

            for ih in range(1,25):
                cons = np.append(cons, data[cc:cc + len_num].replace(',','').replace('.',''))
                cc += len_num
                # note: hours start at 1, hence shifted by one hour
                time = np.append(time, int(data[cc:cc + 5].split(':')[0]) - 1)
                cc += 5
                prev = np.append(prev, data[cc:cc + len_num].replace(',','').replace('.',''))
                cc += len_num

        print(date)

        df_day = pd.DataFrame(np.stack([time, cons]).T, columns=['hour', 'value'])
        for nn, tt in [('year', year), ('month', month), ('day', day)]:
            df_day[nn] = tt

        return df_day, date

    def postprocessing_tot(self):
        '''
        Various operations once the table df_tot has been assembled.
        '''
        self.fix_double_dates()
        self.fix_missing_dst()
        self.df_tot = self.filter_years_by_data_length(self.df_tot)
        self.tz_localize_convert(tz='Europe/Rome')
        self.df_tot = self.get_hour_of_the_year(self.df_tot[['DateTime', 'value', 'nd_id']])
        self.append_to_sql(self.df_tot.copy())

    def read(self, fn):

        df_add, date = self.parse_pdf(fn)

        self.date_fn_list.append((date, fn))
        self.date_list.append(date)

        if df_add is None:
            return

        df_add['DateTime'] = pd.to_datetime(df_add[['hour', 'month', 'year', 'day']])
        df_add['value'] = df_add['value'].astype(float)

        df_add = df_add[['DateTime', 'year', 'value']]
        df_add['nd_id'] = 'IT0'

        return df_add

if __name__ == '__main__':

    dict_sql = dict(db='storage2')

    kw_dict = dict(dict_sql=dict_sql,
                   exclude_substrings=[],#[str(c) for c in range(2000, 2020) if not c == 2017],
                   col_filt=[],
                   ext=['htm', 'html'])

    op = TernaProfileReader(kw_dict)

    self = op
    fn = self.fn_list[1]

    self.read_all(skip_sql=True)
    self.postprocessing_tot()

# %%
#
#self.df_tot.loc[(self.df_tot.year == 2013)
#              & (self.df_tot.DateTime.dt.month == 12)].pivot_table(index=self.df_tot.DateTime.dt.day, values='value', aggfunc=len)

# %%

class RTELoadReader(profile_reader.ProfileReader):
    '''
    Read load data from RTE Historique_consommation_INST_*.xls files,
    obtained from http://clients.rte-france.com/lang/fr/visiteurs/vie/vie_stats_conso_inst.jsp
    '''

    dict_sql_default = dict(sc='profiles_raw', tb='rte_load')
    data_dir = os.path.normpath('DEMAND/FRANCE')

    tb_cols = [('"DateTime"', 'TIMESTAMP'),
               ('nd_id', 'VARCHAR'),
               ('value', 'DOUBLE PRECISION'),
               ('hy', 'SMALLINT'),
               ('year', 'SMALLINT')]
    tb_pk = ['nd_id', 'year', 'hy']

    exclude_substrings=[]

    def __init__(self, kw_dict):
        print(kw_dict)
        super().__init__(**kw_dict)

        self.get_fn_list()

        yr_list = [int(yy.split('.')[0].split(os.path.sep)[-1].split('_')[-1])
                   for yy in self.fn_list]
        self.init_tm(yr_list) # needs TimeMap for dst hours fixing

    def read(self, fn):
        '''
        Read RTE data from xls files.

        Note: DST switch in March shows doubled values for hours 2-3.
        '''

        df_add = pd.read_excel(fn, skiprows=18).dropna()

        df_add = df_add.drop('Type de données', axis=1).set_index('Date')
        df_add = (df_add.stack().reset_index()
                        .rename(columns={'level_1': 'Time', 0: 'value'}))

        df_add.loc[df_add['Time'] == '24:00', 'Time'] = '00:00'

        df_add['DateTime'] = df_add[['Date', 'Time']].apply(lambda x: ' '.join(x), axis=1)
        df_add['DateTime'] = pd.to_datetime(df_add['DateTime'], format='%d/%m/%Y %H:%M')

        df_add.loc[df_add['Time'] == '00:00', 'DateTime'] += pd.Timedelta(1, 'd')

        df_add = df_add.drop(['Date', 'Time'], axis=1)

        df_add['nd_id'] = 'FR0'

        return df_add[['DateTime', 'value', 'nd_id']]

    def postprocessing_tot(self):
        '''
        Various operations once the table df_tot has been assembled.
        '''

        self.fix_missing_dst()
        self.df_tot = self.filter_years_by_data_length(self.df_tot)
        self.tz_localize_convert(tz='Europe/Paris')
        self.df_tot = self.time_resample(self.df_tot[['DateTime', 'value', 'nd_id']])
        self.df_tot = self.get_hour_of_the_year(self.df_tot)
        self.append_to_sql(self.df_tot.copy())

if __name__ == '__main__':

    dict_sql = dict(db='storage2')
    kw_dict = dict(dict_sql=dict_sql, col_filt=[], ext='xls')
    op = RTELoadReader(kw_dict)

    self = op
    fn = self.fn_list[-1]

    # note: due to conversion to UTC not there is some overlap between the years
    # therefore we cannot write directly
    op.read_all(skip_sql=True)
    self.postprocessing_tot()

# %%

class EControlLoadReader(profile_reader.ProfileReader):
    '''
    Read load data from e-control Belastungsablauf_Oeff_*.xlsx files,
    obtained from https://www.e-control.at/statistik/strom/betriebsstatistik/betriebsstatistik2014
    etc, as well as from google filename.
    '''

    dict_sql_default = dict(sc='profiles_raw', tb='econtrol_load')
    data_dir = os.path.normpath('DEMAND/AUSTRIA')

    tb_cols = [('"DateTime"', 'TIMESTAMP'),
               ('nd_id', 'VARCHAR'), ('value', 'DOUBLE PRECISION'),
               ('hy', 'SMALLINT'), ('year', 'SMALLINT')]
    tb_pk = ['nd_id', 'year', 'hy']

    exclude_substrings=[]

    def __init__(self, kw_dict):
        super().__init__(**kw_dict)

        self.get_fn_list()

    def read(self, fn):
        '''
        Reads input files and generates a dataframe.

        Time zone is Europe/Vienna. Conversion to UTC and comparison to
        UTC entso-e data yield satisfactory results.
        '''

        df_add = pd.DataFrame()
        for imt in range(1,13):
            print('Reading month {}'.format(imt))

            df_addp = pd.read_excel(fn, sheet_name=format(imt, '02'), usecols='K:M', header=23).dropna()

            df_add = pd.concat([df_add, df_addp])

        df_add.columns = ['DateTime', 'energy', 'value']
        df_add = df_add.drop('energy', axis=1)

        df_add['DateTime'] = pd.to_datetime(df_add['DateTime'].apply(lambda x: x.split('-')[0]), format='%d.%m.%y   %H:%M')

        df_add['DateTime'] = df_add['DateTime'].dt.tz_localize('Europe/Vienna', ambiguous='infer')
        df_add['DateTime'] = df_add['DateTime'].dt.tz_convert('UTC')

        df_add['nd_id'] = 'AT0'

        df_add = self.time_resample(df_add)

        return df_add

    def postprocessing_tot(self):

        self.df_tot['year'] = self.df_tot.DateTime.dt.year
        self.df_tot = self.filter_years_by_data_length(self.df_tot)
        self.df_tot = self.get_hour_of_the_year(self.df_tot)
        self.append_to_sql(self.df_tot)

if __name__ == '__main__':

    dict_sql = dict(db='storage2')

    kw_dict = dict(dict_sql=dict_sql,
                   col_filt=[], ext='xlsx')

    op = EControlLoadReader(kw_dict)
    self = op
    fn = self.fn_list[1]
    op.read_all(skip_sql=True)
    op.postprocessing_tot()

# %%

class SwissGridLoadReader(profile_reader.ProfileReader):
    '''
    Read load data from Swissgrid EnergieUebersicht*.xls files,
    obtained from https://www.swissgrid.ch/de/home/operation/grid-data.html
    '''

    dict_sql_default = dict(sc='profiles_raw', tb='swissgrid_load')
    data_dir = os.path.normpath('DEMAND/SWITZERLAND')

    tb_cols = [('"DateTime"', 'TIMESTAMP'),
               ('nd_id', 'VARCHAR'), ('value', 'DOUBLE PRECISION'),
               ('hy', 'SMALLINT'), ('year', 'SMALLINT')]
    tb_pk = ['nd_id', 'year', 'hy']

    exclude_substrings=['final']

    def __init__(self, kw_dict):
        super().__init__(**kw_dict)

        self.get_fn_list()

    def read(self, fn):
        ''' Reads input files and generates a dataframe. '''

        df_add = pd.read_excel(fn, sheet_name='Zeitreihen1h00', usecols='A:B', header=1).dropna()
        df_add.columns = ['DateTime', 'value']

        df_add['DateTime'] = pd.to_datetime(df_add['DateTime'])

        # Swissgrid repeats the hour 3 instead of the hour 2 for the
        # switch dst -> normal time; this needs to be fixed in order for
        # tz_localize to work
        dst_index = df_add.loc[df_add.DateTime.dt.month.isin([10]) # october
                             & df_add.DateTime.dt.dayofweek.isin([6]) # sunday
                             & df_add.DateTime.dt.hour.isin([3]) # faulty switch hour
                             ].iloc[-2].name
        df_add.loc[dst_index, 'DateTime'] -= pd.Timedelta(1, 'h')

        df_add['DateTime'] = df_add['DateTime'].dt.tz_localize('Europe/Zurich', ambiguous='infer')
        df_add['DateTime'] = df_add['DateTime'].dt.tz_convert('UTC')

        df_add['nd_id'] = 'CH0'

        return df_add

    def postprocessing_tot(self):

        self.df_tot['year'] = self.df_tot.DateTime.dt.year
        self.df_tot = self.filter_years_by_data_length(self.df_tot)
        self.df_tot = self.get_hour_of_the_year(self.df_tot)
        self.append_to_sql(self.df_tot)

if __name__ == '__main__':

    dict_sql = dict(db='storage2')

    kw_dict = dict(dict_sql=dict_sql,
                   col_filt=[], ext='xls')

    op = SwissGridLoadReader(kw_dict)
    self = op
    op.read_all(skip_sql=True)
    op.postprocessing_tot()


import networkx as nx
import matplotlib.pyplot as plt
import numpy as np

from fastapi import APIRouter
from src.core.utils import Connector

router = APIRouter(
    prefix="/service",
    tags=["Service"],
    responses={404: {"description": "Not found"}},
)

def dijktra(response):
    nodes = []
    left = []
    right = []
    gas_price = []

    for service in response:
        nodes.append(service['kilometer'])

    for service in response:
        left.append(service['left'])

    for service in response:
        right.append(service['right'])

    for service in response:
        gas_price.append(service['gas_price'])

    for service in response:
        gas_price.append(service['right'])

    G=nx.Graph()

    for i in range(0,np.size(nodes)):
        G.add_node(nodes[i])

    for i in range(0,np.size(left)):
        G.add_weighted_edges_from([(left[i],right[i],gas_price[i])])

    pos=nx.shell_layout(G)

    nx.draw(G,pos,with_labels=True, node_color='white', edge_color='b', node_size=800, alpha=0.5)

    edge_labels = nx.get_edge_attributes(G,'weight')
    nx.draw_networkx_edge_labels(G, pos, edge_labels)
    plt.draw()
    plt.pause(1)
    plt.close()

    path=nx.dijkstra_path(G, source=start, target=end)
    return {"start": start, "end": end, "path":path}

@router.get("/service-area/{start}/{end}")
def read_root(start: str, end: str):
    header = {
            #If i need bearer token to connect
            #'Authorization': f'Bearer {APIKEY}'
            'Content-Type': 'application/json',
            'cache-control': 'no-cache',
        }
    connector = Connector(header, verify_ssl=False)
    response = connector.get('http://localhost:8000/api/adventure/servicearea/')
    return dijktra(response)


from collections import deque

class Graph:
    def __init__(self):
        self.graph = {}
        self.dfs_visited = []
        self.bfs_visited = []

    def insert(self, x1 : int, x2 : int) -> None:
        if x1 not in self.graph.keys():
            self.graph[x1] = [x2]
        else:
            self.graph[x1].append(x2)

        if x2 not in self.graph.keys():
            self.graph[x2] = [x1]
        else:
            self.graph[x2].append(x1)

    def set_sort(self):
        for i in self.graph.keys():
            self.graph[i].sort()

    def dfs(self, root : int):
        print(root, end=' ')

        self.dfs_visited.append(root)

        for child in self.graph[root]:
            if child not in self.dfs_visited:
                self.dfs(child)

    def bfs(self, root : int):
        self.bfs_visited.append(root)

        queue = deque()
        queue.append(root)

        while queue:
            parent = queue.popleft()
            print(parent, end= ' ')

            for child in self.graph[parent]:
                if child not in self.bfs_visited:
                    queue.append(child)
                    self.bfs_visited.append(child)

if __name__ == "__main__":
    n, m, v = map(int, input().split())
    g = Graph()

    for _ in range(m):
        x1, x2 = map(int, input().split())
        g.insert(x1, x2)

    g.set_sort()
    g.dfs(v)
    print()
    g.bfs(v)


import gradio as gr
import openai
import os
from functools import partial

import chat_agent
from langchain.schema import (
    HumanMessage
)

def set_api_key(api_key):
    openai.api_key = api_key
    os.environ["OPENAI_API_KEY"] = api_key
    return "API Key set successfully."

def get_response(chatbot, api_key, selected_model, user_input, conversation_history=""):
    set_api_key(api_key)

    # Preserve the memory of the current chatbot
    preserved_memory = chatbot.memory

    # Create a new chat agent based on the selected model and seed the memory
    chatbot = chat_agent.create_chatbot(model_name=selected_model, seed_memory=preserved_memory)

    chatbot.agent.run(user_input).strip()

    # Replace "\n" with "
" and "\t" with " " in a string
    def format_html(s):
        return s.replace("\n", "
").replace("\t", " ")

    # Iterate through messages in ChatMessageHistory and format the output
    updated_conversation = '
Chat History
'
    for i, message in enumerate(chatbot.memory.chat_memory.messages):
        if isinstance(message, HumanMessage):
            prefix = "User: "
            background_color = "hsl(0, 0%, 40%)"  # Dark grey background
            text_color = "hsl(0, 0%, 100%)"  # White text
            float_direction = "left"
        else:
            prefix = "Chatbot: "
            background_color = "hsl(0, 0%, 95%)"  # White background
            text_color = "hsl(0, 0%, 0%)"  # Black text
            float_direction = "right"

        updated_conversation += f'''

            {prefix}

{format_html(message.content)}

        '''
    # Add a clear div to clear the float property
    updated_conversation += '

'
    return updated_conversation

def main():
    api_key = os.environ.get("OPENAI_API_KEY")

    api_key_input = gr.components.Textbox(
        lines=1,
        label="Enter OpenAI API Key",
        value=api_key,
        type="password",
    )

    model_selection = gr.components.Dropdown(
        choices=["gpt-3.5-turbo-16k-0613", "gpt-4"],
        label="Select a GPT Model",
        value="gpt-4"# "gpt-3.5-turbo-16k-0613",
    )

    user_input = gr.components.Textbox(
        lines=3,
        label="Enter your message",
    )

    output_history = gr.outputs.HTML(
        label="Updated Conversation",
    )

    chatbot = chat_agent.create_chatbot(model_name=model_selection.value)

    inputs = [
        api_key_input,
        model_selection,
        user_input,
    ]

    iface = gr.Interface(
        fn=partial(get_response, chatbot),
        inputs=inputs,
        outputs=[output_history],
        title="LiveQuery GPT-4",
        description="A simple chatbot using GPT-3,4 and Gradio with conversation history",
        allow_flagging="never",
    )

    iface.launch(share=True)

if __name__ == "__main__":
    main()


#!/usr/bin/env python

try:
    from setuptools import setup, find_packages
except ImportError:
    from ez_setup import use_setuptools
    use_setuptools()
    from setuptools import setup, find_packages

tests_require = [
    'Django>=1.3.1',
]

setup(
    name='Voice',
    version='0.1.0',
    author='Jarod Luebbert',
    author_email='jarodluebbert@gmail.com',
    url='http://github.com/jarodl/voice',
    description='A voting app for letting users request features.',
    packages=find_packages(exclude=("example_project",)),
    zip_safe=False,
    install_requires=[
        'Django>=1.3.1',
        'South',
        ],
    tests_require=tests_require,
    extras_require={'test':tests_require},
    test_suite = 'nose.collector',
    include_package_data=True,
    classifiers=[
      'Framework :: Django',
      'Intended Audience :: Developers',
      'Intended Audience :: End Users/Desktop',
      'Operating System :: OS Independent',
    ],
)


from marshmallow import fields, post_load

from app.ma import ma
from app.models.game import GameModel
from app.schemas.round import RoundSchema
from app.schemas.user_game import UserGameSchema

class GameSchema(ma.Schema):
    id = fields.Integer(dump_only=True)
    date_time = fields.DateTime()  # '%H:%M %Y-%m-%d')
    rounds = fields.Nested(RoundSchema(many=True))
    players = fields.Nested(UserGameSchema(many=True), attribute='user_games')

    @post_load
    def make_game(self, data, **kwargs):
        return GameModel(**data)

    class Meta:
        model = GameModel
        load_instance = True
        load_only = ()
        dump_only = ("id", "date_time")


import pandas as pd
from numpy.random import randint
from pathlib import Path
import os
os.chdir(Path(__file__).parent)

def create_base_3d_matrix(i, j, k):
    matrix = {}
    for i in range(1, i + 1):
        matrix[i] = {}
        for j in range(1, j + 1):
            matrix[i][j] = {}
            for k in range(1, k + 1):
                matrix[i][j][k] = 0
    return matrix

def create_energy_cost_matrix(
        n: int,
        S_i: dict,
        p_k: dict,
):
    """
    Creating energy cost matrix for each (work, operation, machine) pair.
    :arg
        n: Number of works,
        S_i: Dictionary of operations per work,
        p_k: Dictionary of positions for each machine
    :return
        P_i_j_k: Dictionary of energy costs.
    """
    # TODO change the counter variable to a uniform dist
    P_i_j_k = {}
    for i in range(1, n + 1):
        P_i_j_k[i] = {}
        for j in range(1, S_i[i] + 1):
            P_i_j_k[i][j] = {}
            for k in range(1, p_k[j] + 1):
                P_i_j_k[i][j][k] = randint(3, 6)
    return P_i_j_k

def create_energy_time_matrix(
        n: int,
        S_i: dict,
        p_k: dict,
):
    """
    Fetching energy for each (work, operation, machine) pair
    and creating related dictionary
    :arg
        n: Number of works,
        S_i: Dictionary of operations per work,
        p_k: Dictionary of positions for each machine
    :return
        Pt_i_j_k: Dictionary of durations.
    """
    # Creating base dictionary
    Pt_i_j_k = create_energy_cost_matrix(n, S_i, p_k)
    Pt_df = pd.read_excel('glob_tables/Pt.xlsx')
    # Number of works
    n_works = len(Pt_df.columns) - 1
    for row in Pt_df.iterrows():
        # Removing index
        row = row[1]

        # Fetching k and j
        k, j = map(int, str(row['k.j/i']).split('.'))

        # Updating values in dictionary
        for i in range(1, n_works + 1):
            Pt_i_j_k[i][j][k] = row[i]

    return Pt_i_j_k

def create_trans_matrix(
        n: int,
        m: int
):
    """
    Creates transportation matrix
    :param n: number of works
    :param m: number of machines
    :return:
        Transportation dictionary.
    """
    trans_mat = create_base_3d_matrix(n, m, m)
    trans_df = pd.read_excel('glob_tables/Trans.xlsx')
    for row in trans_df.iterrows():
        # Removing index
        row = row[1]

        # Fetching k and k prime
        k, k_prime = map(int, str(row['k.k_prime/i']).split('.'))

        # Updating values in dictionary
        for i in range(1, n + 1):
            trans_mat[i][k][k_prime] = row[i]

    return trans_mat

def create_setup_matrix(
        n: int,
        S_i: dict,
        p_k: dict
):
    """
    Creating setup matrix
    :arg
        n: Number of works,
        S_i: Dictionary of operations per work,
        p_k: Dictionary of positions for each machine
    :return:
        Setup matrix
    """
    # Creating base dictionary
    st_mat = create_energy_cost_matrix(n, S_i, p_k)

    St_df = pd.read_excel('glob_tables/Setup.xlsx')
    # Number of works
    for row in St_df.iterrows():
        # Removing index
        row = row[1]

        # Fetching k and j
        j, k = map(int, str(row['j.k/i']).split('.'))

        # Updating values in dictionary
        for i in range(1, n + 1):
            st_mat[i][j][k] = row[i]

    return st_mat

def generate_r(n):
    """
    Clearance time
    :param n: Number of works
    :return: List of clearance time
    """
    r_dict = {
        1: 5,
        2: 3,
        3: 6
    }
    return r_dict

def generate_pidle(m):
    """
    Energy consumed during standby
    :param m: Number of machines
    :return:Dict of random energy unit.
    """
    pidle_dict = {}
    random_energy = randint(1, 4, size=m + 1)
    for k in range(1, m + 1):
        pidle_dict[k] = random_energy[k]
    return pidle_dict

def generate_O(m):
    """
    Time needed for turning on machine k
    :param m: Number of machines
    :return: Random O values
    """
    O_dict = {}
    random_O = randint(8, 17, size=m + 1)
    for k in range(1, m + 1):
        O_dict[k] = random_O[k]
    return O_dict

def generate_energy(m):
    """
    Energy needed for turning on/off machine k
    :param m: Number of machines
    :return: Random Energy values
    """
    energy_dict = {}
    random_energy = randint(10, 21, size=m + 1)
    for k in range(1, m + 1):
        energy_dict[k] = random_energy[k]
    return energy_dict

def generate_TB(
        O,
        Energy,
        Pidle,
        m
):
    """

    :param O:
    :param Energy:
    :param Pidle:
    :param m:
    :return:
    """
    TB_dict = {}
    for k in range(1, m + 1):
        TB_dict[k] = max(O[k], Energy[k] / Pidle[k])

    return TB_dict


from datetime import datetime
from tScrape import scraper, parser

# List of Twitter handles to scrape. It works with pages, not persons.
handles = ['SeleniumHQ', 'ThePSF', 'OSFramework']

# Date for the scraper to go back in time. Must be unix time stamp.
# Format: year, month, day, hour, minute, second
stopTime = int(datetime(2015, 6, 1, 0, 0, 0).strftime('%s'))

# Should the scraper and parser be verbose?
verbose = True

# Path for data to be stored
dataPath = './data/raw/'
parsePath = './data/'

scraper(handles, dataPath, stopTime, verbose)
parser(handles, dataPath, parsePath, stopTime, verbose)


import math
import numpy as np
from numpy.random import random_integers as rnd
import matplotlib.pyplot as plt

class MazeMDP(object):
    #True are obstacles
    def make_maze(self, width=21, height=21, complexity=.05, density =.1):
        # Only odd shapes
        shape = ((height//2)*2+1, (width//2)*2+1)
        # Adjust complexity and density relative to maze size
        complexity = int(complexity*(5*(shape[0]+shape[1])))
        density    = int(density*(shape[0]//2*shape[1]//2))
        # Build actual maze
        Z = np.zeros(shape, dtype=bool)
        # Fill borders
        Z[0,:] = Z[-1,:] = 1
        Z[:,0] = Z[:,-1] = 1
        # Make isles
        for i in range(density):
            x, y = rnd(0,shape[1]//2)*2, rnd(0,shape[0]//2)*2
            Z[y,x] = 1
            for j in range(complexity):
                neighbours = []
                if x > 1:           neighbours.append( (y,x-2) )
                if x < shape[1]-2:  neighbours.append( (y,x+2) )
                if y > 1:           neighbours.append( (y-2,x) )
                if y < shape[0]-2:  neighbours.append( (y+2,x) )
                if len(neighbours):
                    y_,x_ = neighbours[rnd(0,len(neighbours)-1)]
                    if Z[y_,x_] == 0:
                        Z[y_,x_] = 1
                        Z[y_+(y-y_)//2, x_+(x-x_)//2] = 1
                        x, y = x_, y_
        return Z, width, height

    def __init__(self):
        self.maze, self.width, self.height = self.make_maze()
        self.start_loc = np.array([0,0])
        x,y = np.where(self.maze == True)
        non_term_vars = np.zeros_like(x)
        self.obstacles = np.squeeze(np.array([[x],[y],[non_term_vars]])).T
        x,y = np.where(self.maze == False)
        non_term_vars = np.zeros_like(x)
        self.valid_points = np.squeeze(np.array([[x],[y],[non_term_vars]])).T

    def show_path(self, points, color = 'r', name = 'curr', stayAwake = False):
        plt.imshow(self.maze, cmap=plt.cm.binary,interpolation='nearest')
        plt.scatter(points[:,1], points[:,0], s=7, c=color)
        plt.savefig(name + '_maze_path.png')

        if not stayAwake:
            plt.close()

    def get_next_states(self, state):
        x,y,term = state
        states = np.array([(nx, ny, 0) for nx, ny in[(x, y - 1), (x, y + 1), (x - 1, y), (x + 1, y)]if 0 <= nx < self.height and 0 <= ny < self.width and self.maze[nx][ny] == False] + [[x,y,1]])
        return states

    def get_valid_loc_near(self, state):
        #This will search manhattan for a free location i.e., with a value of False
        x,y = state
        ret = state
        #check index ordering
        if self.maze[x][y] == True:
            states = np.array([(nx, ny) for nx, ny in[(x, y - 1), (x, y + 1), (x - 1, y), (x + 1, y)]if 0 <= nx < self.height and 0 <= ny < self.width and self.maze[nx][ny] == False])
            ret = states[0,:]
        return ret

    def reset(self):
        self.start_loc = np.array([0,0])
        return self.start_loc

    def step(self, action):
        #print(action)
        if action == 0:
            self.start_loc[0] -= 1
        if action == 1:
            self.start_loc[0] += 1
        if action == 2:
            self.start_loc[1] -= 1
        if action == 3:
            self.start_loc[1] += 1
        return self.start_loc

    def goal_Reached(self, state):
        return state[2] == 1

    def action_space(self):
        return [0,1,2,3]

    # Function to plot the maze and path
    def render(self, path):
        plt.imshow(self.maze, cmap=plt.cm.binary,interpolation='nearest')
        plt.scatter(path[:,1], path[:,0], s=7, c='g')
        plt.show()

#


import mediapipe as mp
import numpy as np

mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands
mp_pose = mp.solutions.pose

def annotate_hands(cam_feed: np.ndarray):
    with mp_hands.Hands(
        static_image_mode=True,
        max_num_hands=2,
        min_detection_confidence=0.5,
        # min_tracking_confidence=0.5,
    ) as hands:
        cam_feed.flags.writeable = True
        results = hands.process(cam_feed)

        if results.multi_hand_landmarks:
            print("found landmarks")
            for hand_landmarks in results.multi_hand_landmarks:
                mp_drawing.draw_landmarks(
                    cam_feed,
                    hand_landmarks,
                    mp_hands.HAND_CONNECTIONS,
                    mp_drawing_styles.get_default_hand_landmarks_style(),
                    mp_drawing_styles.get_default_hand_connections_style(),
                )

def annotate_body(cam_feed: np.ndarray):
    with mp_pose.Pose(
        static_image_mode=True,
        model_complexity=2,
        enable_segmentation=True,
        min_detection_confidence=0.5,
        min_tracking_confidence=0.5,
    ) as pose:
        cam_feed.flags.writeable = True
        results = pose.process(cam_feed)
        if results.pose_landmarks:
            mp_drawing.draw_landmarks(
                cam_feed,
                results.pose_landmarks,
                landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),
            )


from datetime import datetime
from typing import TYPE_CHECKING, Any, Generator
from unittest import mock

import pytest
from freezegun import freeze_time

from sentry.processing import realtime_metrics
from sentry.processing.realtime_metrics.base import (
    BucketedCounts,
    BucketedDurationsHistograms,
    DurationsHistogram,
    RealtimeMetricsStore,
)
from sentry.processing.realtime_metrics.redis import RedisRealtimeMetricsStore
from sentry.tasks import low_priority_symbolication
from sentry.tasks.low_priority_symbolication import (
    _scan_for_suspect_projects,
    _update_lpq_eligibility,
    excessive_event_duration,
    excessive_event_rate,
)
from sentry.testutils.helpers.task_runner import TaskRunner
from sentry.utils.services import LazyServiceWrapper

if TYPE_CHECKING:
    from typing import Callable

    def _fixture(func: Callable[..., Any]) -> Callable[..., None]:
        ...

    pytest.fixture = _fixture

@pytest.fixture
def store() -> Generator[RealtimeMetricsStore, None, None]:
    store = LazyServiceWrapper(
        RealtimeMetricsStore,
        "sentry.processing.realtime_metrics.redis.RedisRealtimeMetricsStore",
        {
            "cluster": "default",
            "counter_bucket_size": 10,
            "counter_time_window": 120,
            "duration_bucket_size": 10,
            "duration_time_window": 120,
            "backoff_timer": 0,
        },
    )

    old_properties = realtime_metrics.__dict__.copy()
    store.expose(realtime_metrics.__dict__)
    yield store

    # cleanup
    realtime_metrics.__dict__.update(old_properties)

class TestScanForSuspectProjects:
    @pytest.fixture  # type: ignore
    def mock_update_lpq_eligibility(
        self, monkeypatch: "pytest.MonkeyPatch"
    ) -> Generator[mock.Mock, None, None]:
        mock_fn = mock.Mock()
        monkeypatch.setattr(low_priority_symbolication, "update_lpq_eligibility", mock_fn)
        yield mock_fn

    def test_no_metrics_not_in_lpq(
        self, store: RealtimeMetricsStore, mock_update_lpq_eligibility: mock.Mock
    ) -> None:
        assert store.get_lpq_projects() == set()

        with TaskRunner():
            _scan_for_suspect_projects()

        assert store.get_lpq_projects() == set()
        assert not mock_update_lpq_eligibility.delay.called

    def test_no_metrics_in_lpq(
        self, store: RealtimeMetricsStore, mock_update_lpq_eligibility: mock.Mock
    ) -> None:
        store.add_project_to_lpq(17)
        assert store.get_lpq_projects() == {17}

        with TaskRunner():
            _scan_for_suspect_projects()

        assert store.get_lpq_projects() == set()
        assert not mock_update_lpq_eligibility.delay.called

    @freeze_time(datetime.fromtimestamp(0))
    def test_has_metric(
        self, store: RealtimeMetricsStore, mock_update_lpq_eligibility: mock.Mock
    ) -> None:
        store.increment_project_event_counter(project_id=17, timestamp=0)

        with TaskRunner():
            _scan_for_suspect_projects()

        assert mock_update_lpq_eligibility.delay.called

class TestUpdateLpqEligibility:
    def test_no_counts_no_durations_in_lpq(self, store: RealtimeMetricsStore) -> None:
        store.add_project_to_lpq(17)
        assert store.get_lpq_projects() == {17}

        _update_lpq_eligibility(project_id=17, cutoff=10)
        assert store.get_lpq_projects() == set()

    def test_no_counts_no_durations_not_lpq(self, store: RealtimeMetricsStore) -> None:
        _update_lpq_eligibility(project_id=17, cutoff=10)
        assert store.get_lpq_projects() == set()

    @freeze_time(datetime.fromtimestamp(0))
    def test_is_eligible_not_lpq(
        self, store: RealtimeMetricsStore, monkeypatch: "pytest.MonkeyPatch"
    ) -> None:
        store.increment_project_event_counter(project_id=17, timestamp=0)
        assert store.get_lpq_projects() == set()

        monkeypatch.setattr(
            low_priority_symbolication, "excessive_event_rate", lambda proj, counts: True
        )

        _update_lpq_eligibility(project_id=17, cutoff=10)
        assert store.get_lpq_projects() == {17}

    @freeze_time(datetime.fromtimestamp(0))
    def test_is_eligible_in_lpq(
        self, store: RealtimeMetricsStore, monkeypatch: "pytest.MonkeyPatch"
    ) -> None:
        store.add_project_to_lpq(17)
        monkeypatch.setattr(
            low_priority_symbolication, "excessive_event_rate", lambda proj, counts: True
        )

        _update_lpq_eligibility(project_id=17, cutoff=10)
        assert store.get_lpq_projects() == {17}

    def test_not_eligible_in_lpq(self, store: RealtimeMetricsStore) -> None:
        store.add_project_to_lpq(17)

        _update_lpq_eligibility(project_id=17, cutoff=10)
        assert store.get_lpq_projects() == set()

    def test_not_eligible_not_lpq(self, store: RealtimeMetricsStore) -> None:
        _update_lpq_eligibility(project_id=17, cutoff=10)
        assert store.get_lpq_projects() == set()

    @freeze_time(datetime.fromtimestamp(0))
    def test_is_eligible_recently_moved(
        self, store: RedisRealtimeMetricsStore, monkeypatch: "pytest.MonkeyPatch"
    ) -> None:
        store._backoff_timer = 10
        # Abusing the fact that removing always updates the backoff timer even if it's a noop
        store.remove_projects_from_lpq({17})

        monkeypatch.setattr(
            low_priority_symbolication, "excessive_event_rate", lambda proj, counts: True
        )

        _update_lpq_eligibility(17, 10)
        assert store.get_lpq_projects() == set()

    def test_not_eligible_recently_moved(self, store: RedisRealtimeMetricsStore) -> None:
        store._backoff_timer = 10
        store.add_project_to_lpq(17)

        _update_lpq_eligibility(17, 10)
        assert store.get_lpq_projects() == {17}

class TestExcessiveEventRate:
    def test_high_rate_no_spike(self) -> None:
        # 600 events/10s for 2 minutes
        event_counts = BucketedCounts(timestamp=0, width=10, counts=[600] * 12)
        assert not excessive_event_rate(project_id=1, event_counts=event_counts)

    def test_low_rate_no_spike(self) -> None:
        # 1 event/s for 2 minutes
        event_counts = BucketedCounts(timestamp=0, width=10, counts=[10] * 12)
        assert not excessive_event_rate(project_id=1, event_counts=event_counts)

    def test_low_rate_spike(self) -> None:
        # 0 events for 5m, then 5 events/10s for 1m
        # total event rate = 30/360 = 1/12,
        # recent event rate = 30/60 = 1/2 > 5 * total event rate
        event_counts = BucketedCounts(timestamp=0, width=10, counts=[0] * 30 + [5] * 6)
        assert not excessive_event_rate(project_id=1, event_counts=event_counts)

    def test_high_rate_spike(self) -> None:
        # 0 events for 5m, then 500 events/10s for 1m
        # total event rate = 3600/360 = 10,
        # recent event rate = 3600/60 = 60 > 5 * total event rate
        event_counts = BucketedCounts(timestamp=0, width=10, counts=[0] * 30 + [600] * 6)
        assert excessive_event_rate(project_id=1, event_counts=event_counts)

    def test_flatline(self) -> None:
        event_counts = BucketedCounts(timestamp=0, width=10, counts=[0] * 12)
        assert not excessive_event_rate(project_id=1, event_counts=event_counts)

class TestExcessiveEventDuration:
    # Configured time window of these metrics is 3 minutes.

    def test_no_durations(self) -> None:
        durations = BucketedDurationsHistograms(
            timestamp=0, width=10, histograms=[DurationsHistogram()] * 5 * 6
        )
        assert not excessive_event_duration(project_id=1, durations=durations)

    def test_normal_rate_normal_duration(self) -> None:
        # 1 event/s for 3m, 1-5m durations
        histograms = []
        for _ in range(10 * 6 * 3):
            hist = DurationsHistogram(bucket_size=10)
            hist.incr(duration=60, count=5)
            hist.incr(duration=180, count=5)
            histograms.append(hist)
        durations = BucketedDurationsHistograms(timestamp=0, width=10, histograms=histograms)

        assert not excessive_event_duration(project_id=1, durations=durations)

    def test_normal_rate_long_duration(self) -> None:
        # 1 event/s for 3m, 9m durations
        histograms = []
        for _ in range(6 * 3):
            hist = DurationsHistogram(bucket_size=10)
            hist.incr(duration=9 * 60, count=10)
            histograms.append(hist)
        durations = BucketedDurationsHistograms(timestamp=0, width=10, histograms=histograms)

        assert excessive_event_duration(project_id=1, durations=durations)

    def test_low_rate_long_duration(self) -> None:
        # 1 event/m for 3m, 9m durations
        histograms = []
        durations = None
        for i in range(6 * 3):
            hist = DurationsHistogram(bucket_size=10)
            if i % 6 == 0:
                hist.incr(duration=9 * 60, count=1)
            histograms.append(hist)
            durations = BucketedDurationsHistograms(timestamp=0, width=10, histograms=histograms)

        assert durations is not None
        assert not excessive_event_duration(project_id=1, durations=durations)


from django.shortcuts import render
from django.http import HttpResponse
from django.db.models import Count
from rest_framework import viewsets, status
from rest_framework.response import Response
from .models import *
from .serializers import *
from .utils import *
from .filters import *
from rest_framework import filters

class RatingViewSet(viewsets.ModelViewSet):
    queryset = Rating.objects.all()
    serializer_class = RatingSerializer

class CommentViewSet(viewsets.ModelViewSet):

    '''
    List all comments or comments by movie_id
    parameters:
    movie - the movie id
    '''

    queryset = Comment.objects.all()
    serializer_class = CommentSerializer

    def get_queryset(self):
        queryset = Comment.objects.all()
        movie_id = self.request.query_params.get('movie', None)
        if movie_id is not None:
            queryset = queryset.filter(movie__id=movie_id)
        return queryset

class MovieViewSet(viewsets.ModelViewSet):
    '''
    List all movies
    '''
    queryset = Movie.objects.all()
    serializer_class = MovieSerializer
    filter_backends = [YearRangeFilterBackend, MovieFieldFilterBackend, filters.OrderingFilter]

    def create(self, request):
        movie_serializer = MovieSerializer(data=request.data)
        movie_serializer.is_valid(raise_exception=True)
        movie_obj = movie_serializer.save()
        return Response(movie_serializer.data, status=status.HTTP_201_CREATED)

class TopViewSet(viewsets.ReadOnlyModelViewSet):
    '''
    List movies in order by number of comments
    parameters:
    start - the beginning year in the range of movies to list
    end - the end year in the range of movies to list
    '''
    queryset = Movie.objects.annotate(total_comments=Count('comments__id')).order_by('total_comments').reverse()
    serializer_class = TopSerializer
    filter_backends = [YearRangeFilterBackend]