from tkinter import *
from chat import get_responce, bot_name

import pyttsx3 as pp
import speech_recognition as s
import threading

engine=pp.init()
voices=engine.getProperty('voices')
print(voices)
engine.setProperty('voice',voices[1].id)

#To make chatbot speak
def speak(word):
    engine.say(word)
    engine.runAndWait()

# take query: it takes audio as input from user and convert it into string
def takeQuery():
    sr = s.Recognizer()
    sr.pause_threshold = 1
    print("Your Bot is listening")
    with s.Microphone() as m:
        try:
            audio = sr.listen(m)
            query = sr.recognize_google(audio, language='eng-in')
            print(query)
            app._insert_message(query,"You")

        except Exception as e:
            print(e)
            err_response="I didn’t get that. Can you repeat again ? "
            speak(err_response)
            print(err_response)
            msg_init = f"{bot_name}: {err_response}\n\n"
            app.text_widget.configure(state=NORMAL)
            app.text_widget.insert(END, msg_init)
            app.text_widget.configure(staCte=DISABLED)

            app.text_widget.see(END)

def repeat():
        while True:
            takeQuery()

#Set Font and Background properties
BG_GRAY="#ABB2B9"
BG_COLOR="#002244"
TEXT_COLOR="#EAECEE"

FONT="Helvetica 11"
FONT_BOLD="Helvetica 11 bold"

class ChatApplication:

    def __init__(self):
        self.window=Tk()
        self._setup_main_window()

    def run(self):
        t = threading.Thread(target=repeat)
        t.start()
        self.window.mainloop()

    def _setup_main_window(self):
        self.window.title("Chatbot")
        self.window.resizable(width=False, height=False)
        self.window.configure(width=550,height=600,bg=BG_COLOR)

        #head label
        head_label=Label(self.window,bg=BG_COLOR,fg=TEXT_COLOR,text="Welcome to ABCD Store",font=FONT_BOLD,pady=10)
        head_label.place(relwidth=1)

        #tiny divider
        line=Label(self.window,width=480,bg=BG_GRAY)
        line.place(relwidth=1,rely=0.07,relheight=0.012)

        #text widget
        self.text_widget=Text(self.window,width=20,height=2,bg=BG_COLOR,fg=TEXT_COLOR,font=FONT,padx=5,pady=5)
        self.text_widget.place(relheight=0.745,relwidth=1,rely=0.08)
        self.text_widget.configure(cursor="arrow",state=DISABLED)

        #scroll bar
        scrollbar=Scrollbar(self.text_widget)
        scrollbar.place(relheight=1,relx=0.974)
        scrollbar.configure(command=self.text_widget.yview)

        #bottom label
        bottom_label=Label(self.window,bg=BG_GRAY,height=80)
        bottom_label.place(relwidth=1,rely=0.825)

        #message entry box
        self.msg_entry=Entry(bottom_label,bg="#B9D9EB",fg="#002244",font=FONT)
        self.msg_entry.place(relwidth=0.74,relheight=0.06,rely=0.008,relx=0.011)
        self.msg_entry.focus()
        self.msg_entry.bind("<Return>",self._on_enter_pressed)

        #send button
        send_button=Button(bottom_label,text="Send",font=FONT_BOLD,width=20,bg=BG_GRAY,  command=lambda:self._on_enter_pressed(None))
        send_button.place(relx=0.77,rely=0.008, relheight=0.06, relwidth=0.22)

        init_response="Hello!! I am Bot who can assist your grocery shopping."
        msg_init = f"{bot_name}: {init_response}\n\n"
        # speak(init_response)
        self.text_widget.configure(state=NORMAL)
        self.text_widget.insert(END, msg_init)
        self.text_widget.configure(state=DISABLED)

        init_response_2 = "Please enter your list of items one by one to get the shelf number"
        msg_init_2 = f"{bot_name}: {init_response_2}\n\n"
        # speak(init_response_2)
        self.text_widget.configure(state=NORMAL)
        self.text_widget.insert(END, msg_init_2)
        self.text_widget.configure(state=DISABLED)

        self.text_widget.see(END)

    def _on_enter_pressed(self,event): #To send message when pressed on enter key
        msg=self.msg_entry.get()
        self._insert_message(msg,"You")

    def _insert_message(self,msg,sender):
        if not msg:
            return

        self.msg_entry.delete(0,END)
        msg1=f"{sender}: {msg}\n\n"
        self.text_widget.configure(state=NORMAL)
        self.text_widget.insert(END,msg1)
        self.text_widget.configure(state=DISABLED)

        response=get_responce(msg)
        msg2 = f"{bot_name}: {response}\n\n"
        speak(response)
        self.text_widget.configure(state=NORMAL)
        self.text_widget.insert(END, msg2)
        self.text_widget.configure(state=DISABLED)

        self.text_widget.see(END)

if __name__=="__main__":
    app=ChatApplication()
    app.run()


from fileinput import filename
import cv2
import numpy as np

path = '/home/nicole/propwave/ParkingLot/parking_lot/images/parking_lot_2.png'

img = cv2.imread(filename = path)
hsv_img = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)

boundaries = [([0,0,100],[255,50,225]), #white
              ([20,100,100], [30,255,255])  #yellow
              ]
for (lower, upper) in boundaries:
    lower = np.array(lower, dtype= 'uint8')
    upper = np.array(upper, dtype= 'uint8')
    mask = cv2.inRange(hsv_img, lower, upper)
    cv2.imshow('parking_lot',mask)
    cv2.waitKey()
    cv2.destroyAllWindows()

# line_image = np.copy(img) * 0
# gray = cv2.cvtColor(src=img, code = cv2.COLOR_BGR2GRAY)
# blur_gray = cv2.GaussianBlur(src=gray, ksize=(5, 5), sigmaX=0)
# edges = cv2.Canny(image=blur_gray, threshold1=50, threshold2=150, apertureSize=3)
# lines = cv2.HoughLinesP(image=edges, rho=1, theta=np.pi/180, threshold=100, minLineLength=30, maxLineGap=5)
# for i in range(len(lines)):
#     for x1,y1,x2,y2 in lines[i]:
#         cv2.line(img,(x1,y1),(x2,y2),(0,255,0),2)

# lines_edges = cv2.addWeighted(img, 0.8, line_image, 1, 0)


from django.core.files.storage import default_storage
from django.db.models import FileField, ImageField
from core.utils import convert_to_webp
from pathlib import Path
import os

def format_img(sender, **kwargs):

    instance = kwargs['instance']

    if not instance:
        return

    for fieldname in [f.name for f in sender._meta.get_fields()]:
        try:
            field = sender._meta.get_field(fieldname)
        except:
            field = None

        if field and isinstance(field, ImageField):
            f = getattr(instance, fieldname)
            m = instance.__class__._default_manager

            if (
                hasattr(f, "path")
                and os.path.exists(f.path)
                and not m.filter(
                    **{"%s__exact" % fieldname: getattr(instance, fieldname)}
                ).exclude(pk=instance._get_pk_val())
            ):
                try:
                    webp_path = convert_to_webp(Path(f.path))
                except Exception:
                    return

def file_cleanup(sender, **kwargs):
    """
    File cleanup callback used to emulate the old delete
    behavior using signals. Initially django deleted linked
    files when an object containing a File/ImageField was deleted.
    Usage:
    >>> from django.db.models.signals import post_delete
    >>> post_delete.connect(file_cleanup, sender=MyModel, dispatch_uid="mymodel.file_cleanup")
    """
    for fieldname in [f.name for f in sender._meta.get_fields()]:
        try:
            field = sender._meta.get_field(fieldname)
        except:
            field = None

        if field and (isinstance(field, FileField) or isinstance(field, ImageField)):
            inst = kwargs["instance"]
            f = getattr(inst, fieldname)
            m = inst.__class__._default_manager
            if (
                hasattr(f, "path")
                and os.path.exists(f.path)
                and not m.filter(
                    **{"%s__exact" % fieldname: getattr(inst, fieldname)}
                ).exclude(pk=inst._get_pk_val())
            ):
                try:
                    default_storage.delete(f.path)
                except:
                    pass


from rnn_jhlim import *

import matplotlib
# Force matplotlib to not use any Xwindows backend.
matplotlib.use('Agg')
import matplotlib.pyplot as plt
plt.ion()

if __name__ == '__main__':
    encoder = 'lstm'
    optimizer=adadelta
    initializer=ortho_weight

    # plot predpred err
    plt.close('all')
    fig_pred = plt.figure(1)
    plt.grid()
    plt.title('prediction error per step (%s_%s_%s)' % (encoder, optimizer.__name__, initializer.__name__))

    fig_pred_per_frame = plt.figure(2)
    plt.grid()
    plt.title('prediction error per frame (%s_%s_%s)' % (encoder, optimizer.__name__, initializer.__name__))

    for noise_std in [0., 0.001]:
        for dim_proj in [10, 100]:
            for input_dim in [1, 5]:

                #noise_std = 0.#0.001
                #dim_proj = 10
                #input_dim=5
                output_dim=input_dim # 1
                stride=input_dim # 1
                pred_steps = ((160-(input_dim+output_dim))/stride+1) - ((50-input_dim)/stride+1)
                #encoder = 'lstm'
                mode = 'test'
                #max_epochs=800
                #optimizer=adadelta
                #initializer=ortho_weight
                filename="models/0model_%s_%s_%s_input_dim_%d_hidden_%d_noise_std_%.4f.npz" % (encoder, optimizer.__name__, initializer.__name__, input_dim, dim_proj, noise_std)

                is_plot_save=False

                # test
                reload_model=filename
                saveto=""
                max_epochs=0

                train_err, valid_err, test_err, pred_err, pred_err_per_frame = train_lstm(
                        dim_proj=dim_proj,
                        input_dim=input_dim,
                        output_dim=output_dim,
                        stride=stride,
                        pred_steps=pred_steps,
                        reload_model=reload_model,
                        saveto=saveto,
                        noise_std=noise_std,
                        max_epochs=max_epochs,
                        test_size=500,
                        use_dropout=False,
                        encoder=encoder,
                        initializer=initializer,
                        optimizer=optimizer,
                        is_plot_save=is_plot_save
                )
                plt.figure(1) # fig_pred
                plot_pred_err, = plt.plot(pred_err, '--o', label='i=%d h=%d n_std=%.4f' % (input_dim, dim_proj, noise_std))

                plt.figure(2) #fig_pred_per_frame
                plot_pred_err_per_frame, = plt.plot(pred_err_per_frame, '--o', label='i=%d h=%d n_std=%.4f' % (input_dim, dim_proj, noise_std))

    # Save as a file
    plt.figure(1)
    lgd = plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    plt.yscale('log')
    plt.xlabel('step')
    plt.ylabel('mse')
    plt.savefig('0pred_err_%s_%s_%s.png' % (encoder, optimizer.__name__, initializer.__name__), bbox_extra_artists=(lgd,), bbox_inches='tight')

    plt.figure(1)
    lgd = plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    plt.yscale('log')
    plt.xlim((0, 20))
    plt.xlabel('step')
    plt.ylabel('mse')
    plt.savefig('0pred_err_%s_%s_%s_zoom.png' % (encoder, optimizer.__name__, initializer.__name__), bbox_extra_artists=(lgd,), bbox_inches='tight')

    plt.figure(2)
    lgd = plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    plt.yscale('log')
    plt.xlabel('k')
    plt.ylabel('mse')
    plt.savefig('0pred_err_per_frame_%s_%s_%s.png' % (encoder, optimizer.__name__, initializer.__name__), bbox_extra_artists=(lgd,), bbox_inches='tight')

    plt.figure(2)
    lgd = plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    plt.yscale('log')
    plt.xlim((0, 20))
    plt.xlabel('k')
    plt.ylabel('mse')
    plt.savefig('0pred_err_per_frame_%s_%s_%s_zoom.png' % (encoder, optimizer.__name__, initializer.__name__), bbox_extra_artists=(lgd,), bbox_inches='tight')


#metrics_service.py
from flask import Flask, request, redirect, jsonify
from flask import Blueprint
import pymysql
import databasecon
import auth_service
import requests
import user_service
import urllib.parse
import datetime
import random
import circuitbreaker

metrics_api = Blueprint('metrics_api', __name__)
breaker = circuitbreaker.CircuitBreaker()

#Entry point of the microservice
@metrics_api.route('/update_metrics', methods=['POST'])
def update_metrics():
        print("good")
        gitusername=request.args.get("gitusername")
        apikey=request.args.get("apikey")
        key_status=auth_service.validate_key(gitusername,apikey)
        print(key_status)

        #call github api for metrics if key is valied
        if key_status==1:

            #Get total number of commits in a project as one metrics

            url='https://api.github.com/repos/manuja/LASMS/commits?author='

            #invoke the circiut breaker
            user_commits = breaker.execute(gitusername,url)

            #return user_commits

            if (user_commits) == "500" :
                return "Github api handshake could not be established"
            elif (user_commits) == "424" :
                return "Circuit Breaker on !!!"
            else:
                no_of_user_commits=len(user_commits)

            # #Get no of issues in a project as one metrics

            url_issues = 'https://api.github.com/repos/manuja/LASMS/issues?assignee='
            user_issues = breaker.execute(gitusername,url_issues)

            if (user_issues) == "500" :
                return "Github api handshake could not be established"
            elif (user_issues) == "424" :
                return "Circuit Breaker on !!!"
            else:
                no_of_user_issues=len(user_issues)

            #Get no of imoji icons

            url_imoji = 'https://api.github.com/repos/manuja/LASMS/issues?user='
            user_imoji = breaker.execute(gitusername,url_imoji)

            if (user_imoji) == "500" :
                return "Github api handshake could not be established"
            elif (user_imoji) == "424" :
                return "Circuit Breaker on !!!"
            else:
                no_of_user_comments=len(user_imoji)

                no_of_user_comments=user_imoji
                total_imoji=0
                for x in range(2):
                    total_imoji=total_imoji+no_of_user_comments[x]['reactions']['+1']
                #print(no_of_user_comments[0]['reactions']['hooray'])
                print(total_imoji)
                #print(no_of_user_comments[0]['reactions']['hooray'])

                #check exsistance of user in metrics table
                #exsist_metrics=user_service.check_exsistance(gitusername)
                total_imoji_value=total_imoji

            now = random. random()

            #update the metrics table
            cursor = databasecon.connection.cursor()
            update_table_query = '''UPDATE tblusermetrics SET commits=(%s),issues=(%s),other=(%s),record_log_time=(%s)  WHERE user_id=(SELECT id FROM tbluser WHERE gitusername=(%s)) ;'''
            cursor.execute(update_table_query,(no_of_user_commits,no_of_user_issues,total_imoji_value,now,gitusername))
            # Check the affected rows count
            affected_rows = cursor.rowcount
            databasecon.connection.commit()

            print("gyyyy")
            print(affected_rows)

            if affected_rows == 0:
                print("I am here no worries")

                print(no_of_user_commits)
                print(no_of_user_issues)
                print(total_imoji_value)
                gituserid=user_service.getuserid(gitusername)
                print(gituserid)
                #insert the metrics table
                cursorinsert = databasecon.connection.cursor()
                create_table_query = '''INSERT INTO tblusermetrics (user_id,commits, issues, other) VALUES (%s,%s,%s,%s);'''
                cursorinsert.execute(create_table_query,(gituserid,no_of_user_commits,no_of_user_issues,total_imoji_value))
                databasecon.connection.commit()

                return f"Searched User: {gitusername} has {no_of_user_commits} of commits, {no_of_user_issues} of issues and {total_imoji_value} of Thubsup Emoji. Record updated successfully!!!"
            else:

                return f"Searched User: {gitusername} has {no_of_user_commits} of commits, {no_of_user_issues} of issues and {total_imoji_value} of Thumbsup Emoji.Record updated successfully!!! "

        else:

            return redirect("/")


import time
import json
import random
import requests
import urllib3
from bs4 import BeautifulSoup
from django.db import models

class Address(models.Model):
    address_1 = models.CharField(max_length=100)
    address_2 = models.CharField(blank=True, max_length=100)
    city = models.CharField(max_length=50)
    region = models.CharField(max_length=50)
    country = models.CharField(max_length=50)
    region_code = models.CharField(max_length=10)

    @property
    def full_address(self):
        address = "{}, {}".format(
            self.address_1, self.address_2
        ) if self.address_2 else self.address_1
        return "{}, {}, {}, {}  {}".format(
            address, self.city, self.region, self.country, self.region_code)

    class Meta:
        verbose_name_plural = 'addresses'

    def __str__(self):
        return self.full_address

class Buyer(models.Model):
    first_name = models.CharField(max_length=30)
    last_name = models.CharField(max_length=30)
    email = models.EmailField()
    phone_number = models.CharField(max_length=15)
    shipping_address = models.ForeignKey(Address, on_delete=models.PROTECT)

    @property
    def full_name(self):
        return "{} {}".format(self.first_name, self.last_name)

    def __str__(self):
        return self.full_name

class CreditCard(models.Model):
    description = models.CharField(max_length=50)
    card_holder = models.CharField(max_length=60)
    card_number = models.CharField(max_length=16)
    cvv = models.CharField(max_length=5, verbose_name='CVV')
    expiry_month = models.CharField(max_length=2)
    expiry_year = models.CharField(max_length=4)
    billing_address = models.ForeignKey(Address, on_delete=models.PROTECT)

    @property
    def display(self):
        return "************{}".format(self.card_number[12:])

    def __str__(self):
        return "{} ({})".format(self.description, self.display)

class Product(models.Model):
    description = models.CharField(max_length=50)
    site_url = models.URLField()
    keywords = models.TextField(
        help_text="In-quotation and comma-separated list"
    )
    variant = models.CharField(
        blank=True,
        help_text="Specific colour, size, etc",
        max_length=30
    )
    random = models.BooleanField(help_text="Ignore variant if not found")

    def __str__(self):
        return self.description

class Bot(models.Model):
    description = models.CharField(max_length=50)
    buyer = models.ForeignKey(Buyer, on_delete=models.PROTECT)
    credit_card = models.ForeignKey(CreditCard, on_delete=models.PROTECT)
    product = models.ForeignKey(Product, on_delete=models.PROTECT)
    delay = models.PositiveIntegerField()

    def get_keyword_list(self):
        return list(self.product.keywords.split(', '))

    def get_products(self, session):
        link = self.product.site_url  # TO DO plus?
        request = session.get(link, verify=False)
        return json.loads(request.text)['products']

    def filter_products_by_keyword(self, products):
        for product in products:
            keys = 0
            for keyword in self.get_keyword_list():
                if keyword.upper() in product['title'].upper():
                    keys += 1
                if keys == len(self.get_keyword_list()):
                    return product

    def get_product_variant(self, product):
        if self.product.variant in product['title']:
            return str(product['id'])

        if self.product.random:
            variants = []

            for variant in product['variants']:
                variants.append(variant['id'])

            variant = str(random.choice(variants))
            return variant

    def get_product(self, session):
        return self.get_product_variant(
            self.filter_products_by_keyword(
                self.get_products(session)))

    def generate_cart_link(self, product):
        return self.product.site_url + "/cart/" + product + ":1"

    def get_payment_token(self):
        link = "https://elb.deposit.shopifycs.com/sessions"

        payload = {
            "credit_card": {
                "number": self.credit_card.card_number,
                "name": self.credit_card.card_holder,
                "month": self.credit_card.expiry_month,
                "year": self.credit_card.expiry_year,
                "verification_value": self.credit_card.cvv
            }
        }

        request = requests.post(link, json=payload, verify=False)
        return json.loads(request.text)['id']

    def get_shipping(self, session, cookie_jar):
        link = (
            self.product.site_url
            + "//cart/shipping_rates.json?shipping_address[zip]="
            + self.buyer.shipping_address.region_code
            + "&shipping_address[COUNTRY]="
            + self.buyer.shipping_address.country
            + "&shipping_address[PROVINCE]="
            + self.buyer.shipping_address.province
        )
        request = session.get(link, cookies=cookie_jar, verify=False)
        options = json.loads(request.text)
        option = options['shipping_rates'][0]['name'].replace(' ', '%20')
        price = options["shipping_rates"][0]["price"]
        return "shopify-" + option + "-" + price

    def add_to_cart(self, session, product):
        link = self.product.site_url + '/cart/add.js?quantity=1&id=' + product
        return session.get(link, verify=False)

    def submit_customer_info(self, session, cookie_jar):
        buyer = self.buyer
        address = self.buyer.shipping_address
        payload = {
            "utf8": u"\u2713",
            "_method": "patch",
            "authenticity_token": "",
            "previous_step": "contact_information",
            "step": "shipping_method",
            "checkout[EMAIL]": buyer.email,
            "checkout[buyer_accepts_marketing]": "0",
            "checkout[shipping_address][FIRST_NAME]": buyer.first_name,
            "checkout[shipping_address][LAST_NAME]": buyer.last_name,
            "checkout[shipping_address][company]": "",
            "checkout[shipping_address][ADDRESS_1]": address.address_1,
            "checkout[shipping_address][ADDRESS_2]": address.address_2,
            "checkout[shipping_address][CITY]": address.city,
            "checkout[shipping_address][COUNTRY]": address.country,
            "checkout[shipping_address][PROVINCE]": address.region,
            "checkout[shipping_address][zip]": address.region_code,
            "checkout[shipping_address][PHONE_NUMBER]": buyer.phone_number,
            "checkout[remember_me]": "0",
            "checkout[client_details][browser_width]": "1710",
            "checkout[client_details][browser_height]": "1289",
            "checkout[client_details][javascript_enabled]": "1",
            "button": ""
        }

        link = self.product.site_url + "//checkout.json"
        response = session.get(link, cookies=cookie_jar, verify=False)

        link = response.url
        checkout_link = link

        response = session.post(
            link, cookies=cookie_jar, data=payload, verify=False)

        return response, checkout_link

    def perform_task(self):
        session = requests.session()
        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
        product = None

        while product is None:
            product = self.get_product(session)
            if product is None:
                time.sleep(self.delay)

        cart_link = self.generate_cart_link(product)
        request = self.add_to_cart(session, product)
        cookies = request.cookies
        payment_token = self.get_payment_token()
        request, checkout_link = self.submit_customer_info(session, cookies)
        ship = self.get_shipping(cookies)
        link = checkout_link + "?step=payment_method"
        request = session.get(link, cookies=cookies, verify=False)
        bs = BeautifulSoup(request.text, "html.parser")
        div = bs.find("div", {"class": "radio__input"})
        print(div)

        gateway = ""
        values = str(div.input).split('"')
        for value in values:
            if value.isnumeric():
                gateway = value
                break

        link = checkout_link
        buyer = self.buyer
        address = self.credit_card.billing_address
        payload = {
            "utf8": u"\u2713",
            "_method": "patch",
            "authenticity_token": "",
            "previous_step": "payment_method",
            "step": "",
            "s": payment_token,
            "checkout[payment_gateway]": gateway,
            "checkout[credit_card][vault]": "false",
            "checkout[different_billing_address]": "true",
            "checkout[billing_address][FIRST_NAME]": buyer.first_name,
            "checkout[billing_address][LAST_NAME]": buyer.last_name,
            "checkout[billing_address][ADDRESS_1]": address.address_1,
            "checkout[billing_address][ADDRESS_2]": address.address_2,
            "checkout[billing_address][CITY]": address.city,
            "checkout[billing_address][COUNTRY]": address.country,
            "checkout[billing_address][PROVINCE]": address.region,
            "checkout[billing_address][zip]": address.region_code,
            "checkout[billing_address][PHONE_NUMBER]": buyer.phone_number,
            "checkout[shipping_rate][id]": ship,
            "complete": "1",
            "checkout[client_details][browser_width]": str(
                random.randint(1000, 2000)),
            "checkout[client_details][browser_height]": str(
                random.randint(1000, 2000)),
            "checkout[client_details][javascript_enabled]": "1",
            "g-recaptcha-repsonse": "",
            "button": ""
        }

        session.post(link, cookies=cookies, data=payload, verify=False)

    def __str__(self):
        return self.description


# -*- coding: utf-8 -*-

"""
A partir de este código se obtiene la tabla .fits con toda la información
relativa al proyecto GAMA que se va a utilizar en el matching; el fichero de
salida será uno de los ficheros de entrada de main_matching_gh.py

  Python 3.6.0    scipy 0.18    numpy 1.11.3    astropy 1.3

@author: Javier Gutiérrez Solórzano
"""
#==============================================================================
#    Nombres de los ficheros de entrada/salida y su localización.
#==============================================================================
#--------------------------
#   Directorio en el que se encuentran los modulos.
#--------------------------
import sys
sys.path.append("../pakages/")
#--------------------------
#   Directorios de los ficheros.
#--------------------------
inroute = './inroute/'
outroute = './outroute_gama/'
#--------------------------
#   Nombre de los ficheros.
#--------------------------
infile_catalogo = "GamaCoreDR1_v1.fits"
outfile = "Gama.fits"

import get
import rojo

#==============================================================================
#  Desplazamiento al rojo disponible de los objetos de GAMA.
#==============================================================================
z_HELIO = get.columna_fits(inroute + infile_catalogo,6)
z_QUALITY = get.columna_fits(inroute + infile_catalogo,7)

[z_gama, E_z_gama, flag_gama] = rojo.z_gama(z_HELIO,z_QUALITY)

#==============================================================================
# Creación del fichero fits de 'salida'.
#==============================================================================

outfile_array_GAMA_IAU_ID = get.columna_fits(inroute + infile_catalogo,0)
outfile_array_RA_g = get.columna_fits(inroute + infile_catalogo,3)
outfile_array_DEC_g = get.columna_fits(inroute + infile_catalogo,4)

from astropy.io import fits

#--------------------------
#   Creamos las columnas del fichero de salida.
#--------------------------

col0 = fits.Column(name='GAMA_IAU_ID', format='23A'
                   , array=outfile_array_GAMA_IAU_ID)
col1 = fits.Column(name='RA_G', format='D',  array=outfile_array_RA_g
                   , unit='Degrees')
col2 = fits.Column(name='DEC_G', format='D', array=outfile_array_DEC_g
                   , unit='Degrees')

    # Información proporcinada por HATLAS.
col3 = fits.Column(name='z_HELIO', format='D', array=z_HELIO)
col4 = fits.Column(name='z_QUALITY', format='D', array=z_QUALITY)

    # Desplazamientos al rojo considerados válidos.
col5 = fits.Column(name='Z_G', format='D', array=z_gama)
col6 = fits.Column(name='Error_Z_G', format='D', array=E_z_gama)
col7 = fits.Column(name='Flag_gama', format='D', array=flag_gama)

cols = fits.ColDefs([col0, col1, col2, col3, col4, col5, col6, col7])

tbhdu = fits.BinTableHDU.from_columns(cols)

tbhdu.writeto(outroute + outfile,overwrite=True)


""" Script to train
Command Line:
    python3 -m bball_strategies.scripts.train
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import datetime
import os
os.environ["CUDA_VISIBLE_DEVICES"] = '0'
import numpy as np
import gym
import tensorflow as tf
from tensorflow.python import debug as tf_debug

from agents import tools
from agents.scripts import utility
from bball_strategies.gym_bball.envs.bball_pretrain_env import BBallPretrainEnv
from bball_strategies.gym_bball.tools import BBallWrapper
from bball_strategies.scripts.pretrain import configs
from bball_strategies.algorithms import pretrain_model

def norm_obs(env, data):
    min_ = env.observation_space.low
    max_ = env.observation_space.high
    data = 2 * (data - min_) / (max_ - min_) - 1
    return data

def pack_action(actions, team):
    if team == 'offense':
        decision_logits, action = actions
        decision = np.argmax(decision_logits.reshape([3, ]))
        ret_action = tuple((
            np.array(decision),
            action[:2],
            action[2:].reshape([5, 2]),
            np.array([[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]])
        ))
    elif team == 'defense':
        ret_action = tuple((
            np.array(0),
            np.array([0, 0]),
            np.array([[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]),
            actions.reshape([5, 2])
        ))
    return ret_action

def vis_data(off_data, off_label, def_data, def_label, outdir, start_idx=0):
    """ vis the dataset of transitions

    Args
    ----
    """

    idx_ = start_idx

    init_pos = [
        np.array(off_data[idx_, 0, -1, 0, :]),
        np.array(off_data[idx_, 0, -1, 1:6, :], dtype=np.float),
        np.array(off_data[idx_, 0, -1, 6:11, :], dtype=np.float)
    ]
    env = gym.make('bball-pretrain-v0')
    env = BBallWrapper(env, if_clip=False, if_norm_obs=False, if_norm_act=False, init_mode=2, if_vis_visual_aid=True,
                       if_vis_trajectory=False, init_positions=init_pos)
    env = gym.wrappers.Monitor(
        env, outdir, lambda unused_episode_number: True, force=False, resume=True)
    obs = env.reset()

    while True:
        # prevent from modification
        temp_off_label = np.array(off_label[idx_, 0])
        temp_def_label = np.array(def_label[idx_, 0])
        if idx_ == start_idx:
            # the env's velocity is zero, so we add the last velocity after env reset.
            last_vel = off_data[idx_, 0, -1, 1:6, :] - \
                off_data[idx_, 0, -2, 1:6, :]
            temp_off_label[5:] += last_vel.reshape([10, ])
            last_vel = def_data[idx_, 0, -1, 6:11, :] - \
                def_data[idx_, 0, -2, 6:11, :]
            temp_def_label += last_vel.reshape([10, ])
        # offense
        action = pack_action(
            [temp_off_label[:3], temp_off_label[3:]], team='offense')
        obs, _, done, _ = env.step(action)
        if done:
            env.close()
            break
        # deffense
        action = pack_action(temp_def_label, team='defense')
        obs, _, done, _ = env.step(action)
        if done:
            env.close()
            break
        idx_ += 1

def vis_result(sess, model, off_data, off_label, def_data, def_label, outdir, num_video):
    """ vis the results by using the pretrain output interacting with env

    Args
    ----
    """

    data_len = np.load('bball_strategies/data/FixedFPS5Length.npy')
    accumulator = 0
    for i, v in enumerate(data_len):
        data_len[i] += accumulator
        accumulator += v
    for i in range(num_video):
        start_idx = data_len[i]
        idx_ = start_idx
        init_pos = [
            np.array(off_data[idx_, 0, -1, 0, :]),
            np.array(off_data[idx_, 0, -1, 1:6, :], dtype=np.float),
            np.array(off_data[idx_, 0, -1, 6:11, :], dtype=np.float)
        ]
        env = gym.make('bball-pretrain-v0')
        env = BBallWrapper(env, if_clip=False, if_norm_obs=False, if_norm_act=False, init_mode=2, if_vis_visual_aid=True,
                           if_vis_trajectory=False, init_positions=init_pos)
        env = gym.wrappers.Monitor(
            env, outdir, lambda unused_episode_number: True, force=False, resume=True)
        obs = env.reset()

        while True:
            # prevent from modification
            temp_off_label = np.array(off_label[idx_, 0])
            temp_def_label = np.array(def_label[idx_, 0])
            if idx_ == start_idx:
                # the env's velocity is zero, so we add the last velocity after env reset.
                last_vel = off_data[idx_, 0, -1, 1:6, :] - \
                    off_data[idx_, 0, -2, 1:6, :]
                temp_off_label[5:] += last_vel.reshape([10, ])
                last_vel = def_data[idx_, 0, -1, 6:11, :] - \
                    def_data[idx_, 0, -2, 6:11, :]
                temp_def_label += last_vel.reshape([10, ])
            if FLAGS.config == 'offense':
                # offense turn
                obs = norm_obs(env, obs)
                logits, actions = model.perform(sess, obs[None, None])
                actions = pack_action(
                    [logits[0, 0], actions[0, 0]], FLAGS.config)
                obs, _, done, _ = env.step(actions)
                if done:
                    env.close()
                    break
                # defense turn
                actions = pack_action(temp_def_label, team='defense')
                obs, _, done, _ = env.step(actions)
                if done:
                    env.close()
                    break
            elif FLAGS.config == 'defense':
                # offense turn
                actions = pack_action(
                    [temp_off_label[:3], temp_off_label[3:]], team='offense')
                obs, _, done, _ = env.step(actions)
                if done:
                    env.close()
                    break
                # defense turn
                obs = norm_obs(env, obs)
                actions = model.perform(sess, obs[None, None])
                actions = pack_action(actions, FLAGS.config)
                obs, _, done, _ = env.step(actions)
                if done:
                    env.close()
                    break
            idx_ += 1

def testing(config, off_data, off_label, def_data, def_label, outdir):
    """

    Args
    ----
    config : Object providing configurations via attributes.

    Yields
    ------
    score : Evaluation scores.
    """
    # split into train and eval
    off_train_data, off_eval_data = np.split(
        off_data, [off_data.shape[0]*9//10])
    off_train_label, off_eval_label = np.split(
        off_label, [off_data.shape[0]*9//10])
    def_train_data, def_eval_data = np.split(
        def_data, [def_data.shape[0]*9//10])
    def_train_label, def_eval_label = np.split(
        def_label, [def_data.shape[0]*9//10])
    print(off_train_data.shape)
    print(off_eval_data.shape)
    print(off_train_label.shape)
    print(off_eval_label.shape)
    print(def_train_data.shape)
    print(def_eval_data.shape)
    print(def_train_label.shape)
    print(def_eval_label.shape)

    # graph
    tf.reset_default_graph()
    if FLAGS.config == 'offense':
        model = pretrain_model.PretrainOffense(config)
    elif FLAGS.config == 'defense':
        model = pretrain_model.PretrainDefense(config)
    else:
        raise ValueError('{} is not an available config'.format(FLAGS.config))

    message = 'Graph contains {} trainable variables.'
    tf.logging.info(message.format(tools.count_weights()))
    saver = utility.define_saver()
    sess_config = tf.ConfigProto(
        allow_soft_placement=True, log_device_placement=config.log_device_placement)
    sess_config.gpu_options.allow_growth = True

    with tf.Session(config=sess_config) as sess:
        if FLAGS.debug:
            sess = tf_debug.LocalCLIDebugWrapperSession(
                sess, ui_type=FLAGS.ui_type)
        utility.initialize_variables(
            sess, saver, config.logdir, resume=True)
        vis_result(sess, model, off_train_data, off_train_label,
                   def_train_data, def_train_label, os.path.join(outdir, 'train'), 3)
        vis_result(sess, model, off_eval_data, off_eval_label,
                   def_eval_data, def_eval_label, os.path.join(outdir, 'eval'), 3)

def main(_):
    """ Create or load configuration and launch the trainer.
    """
    off_data = np.load('bball_strategies/pretrain/data/off_obs.npy')
    off_label = np.load('bball_strategies/pretrain/data/off_actions.npy')
    def_data = np.load('bball_strategies/pretrain/data/def_obs.npy')
    def_label = np.load('bball_strategies/pretrain/data/def_actions.npy')

    utility.set_up_logging()

    logdir = FLAGS.logdir
    try:
        config = utility.load_config(logdir)
    except IOError:
        if not FLAGS.config:
            raise KeyError('You must specify a configuration.')
        config = tools.AttrDict(getattr(configs, FLAGS.config)())
        config = utility.save_config(config, logdir)
    outdir = os.path.expanduser(os.path.join(FLAGS.logdir, 'vis'))

    vis_data(off_data, off_label, def_data, def_label, outdir, start_idx=0)
    testing(config, off_data, off_label, def_data, def_label, outdir)

if __name__ == '__main__':
    FLAGS = tf.app.flags.FLAGS
    tf.app.flags.DEFINE_string(
        'logdir', 'logdir/pretrain',
        'Base directory to store logs.')
    tf.app.flags.DEFINE_string(
        'timestamp', datetime.datetime.now().strftime('%Y%m%dT%H%M%S'),
        'Sub directory to store logs.')
    tf.app.flags.DEFINE_string(
        'config', None,
        'Configuration to execute.')
    tf.app.flags.DEFINE_boolean(
        'debug', False,
        'whether to enable debug mode')
    tf.app.flags.DEFINE_string(
        'ui_type', 'curses',
        "Command-line user interface type (curses | readline)")
    tf.app.run()


from Person import Person
import datetime

me = Person('Michael Guttag')
him = Person('Barack Hussein Obama')
her = Person('Madonna')
print(him.getLastName())
print(her.getLastName())
him.setBirthday(datetime.date(1961, 8, 4))
her.setBirthday(datetime.date(1958, 8, 16))
print(him.getName(), 'is', him.getAge(), 'days old')
print(her < me)

pList = [me, him, her]
for p in pList:
    print(p)
pList.sort()
for p in pList:
    print(p)


from selenium import webdriver
import unittest

class BaseTest(unittest.TestCase):
    def setUp(self):
        # server = "http://127.0.0.1:8000/login/"
        self.browser = webdriver.Chrome('/home/wade/Desktop/work/selenium/chromedriver')
        self.browser.get("http://127.0.0.1:8000/")
        print("browser set up and running")

    def tearDown(self):
        print("browser closed")
        self.browser.close()


"""
Custom train and val loop
"""

import torch
from save_model import save_model
from tqdm.auto import tqdm
from sklearn.metrics import f1_score, accuracy_score, balanced_accuracy_score, roc_auc_score, precision_score, recall_score
import matplotlib.pyplot as plt
import pickle
from pathlib import Path

device = "cuda" if torch.cuda.is_available() else "cpu"

def train_model(model: torch.nn.Module,
                train_dataloader: torch.utils.data.DataLoader,
                val_dataloader: torch.utils.data.DataLoader,
                optimizer: torch.optim.Optimizer,
                loss_fn: torch.nn.Module,
                target_dir: str,
                figure_dir: str,
                model_name: str,
                epochs: int = 50,
                device: torch.device = "cuda" if torch.cuda.is_available() else "cpu"):

    """Full training/validate loop for a pytorch model and save model with the least loss

    Note:
    - This function's logits is passed directly to the loss_function( for use with loss function such as crossentropy or BCEwithlogits).
    - If you want to use loss function which accept other input, you need to modify the training code
    - Does not utilize early stopping, only early save with val loss as criteria.
    - Automatically save model as .pth. It saves state_dict, epoch, best_val_loss and optimizer_state_dict.
    - Autoamtically save model's training result in pickle file in the same folder as the model.
    - The metrics used is f1_score, accuracy_score, balanced_accuracy_score, roc_auc_score, precision_score, recall_score from scikitlearn.metrics.
    - Use Macro averaging for applicable metrics due to considering all class as equal.
    - Depends on other custom function, save_model, found in utils_save_model.py
    - Automatically plot loss and metrics using matplotlib and save figure as SVG in figure_dir folder.
    - Device is prefebly setup in the beginning of the code in the device agnostic code cell

   Args:
    model: A PyTorch model to be trained and tested.
    train_dataloader: A DataLoader instance for the model to be trained on.
    val_dataloader: A DataLoader instance for the model to be validated on.
    - The code unpack 2 variables from the data loader: X,y. X is output of bert tokenizer which has input_ids, attention_mask and token_type_ids
    - Since BERT expect input_ids and attention_mask, the input to the model needs to be X.input_ids, X.attention_mask
    epochs: An integer indicating how many epochs to train for.
    optimizer: A PyTorch optimizer to help minimize the loss function.
    loss_fn: A PyTorch loss function to calculate loss on both datasets.
    device: A target device to compute on (e.g. "cuda" or "cpu").
    target_dir: target directory name to save the best model and train results with the least loss
    figure_dir: target directory name to save figure of plot
    model_name: Name of the model. Do not include .pth in the name.

  Returns:
    A dictionary of training and testing values for later plotting
    In the form: {train_loss_values: [...],
                  val_loss_values: [...],
                  etc...}
    """

    print(f'Training on {device}.')
    # Create empty loss lists to track values

    train_loss_values = []
    val_loss_values = []
    epoch_count = []

    train_accuracy_values = []
    train_balanced_accuracy_values = []
    train_f1_values = []
    train_roc_auc_values = []
    train_precision_values = []
    train_recall_values = []

    val_accuracy_values = []
    val_balanced_accuracy_values = []
    val_f1_values = []
    val_roc_auc_values = []
    val_precision_values = []
    val_recall_values = []

    #initialize best lost
    val_loss_best = float('inf') #initialize best loss to inifite

    # Create training and testing loop
    for epoch in tqdm(range(epochs)): #use tqdm for progressbar
        print(f"Epoch: {epoch}\n-------") #print epoch
        ### Training
        train_loss = 0 #default train loss for that epoch.
        y_true_train, y_pred_train, y_prob_train = [], [], []  # Initialize empty lists for true and predicted labels
        y_true_val, y_pred_val, y_prob_val = [], [], []  # Initialize empty lists for true and predicted labels

        #This is just for tracking purpose

        # Add a loop to loop through training mini batches
        for batch, (X, y) in enumerate(train_dataloader):
            #loop throuugh all minibatch in an epoch
            #We actually do not need to enumerate the dataloader and define batch
            #It can be just for X, y, path in train_dataloader:
            #But batch can be used to keep track of progress of batch in each epoch so I just leave it in case you want to track the number of batch done

            X, y = X.to(device), y.to(device)

            model.train()  #set mode

            # 1. Forward pass
            y_logits = model(X.input_ids.squeeze(), X.attention_mask.squeeze()) #squeeze to get rid of extra dimension before inputting into BERT
            y_prob = torch.softmax(y_logits, dim = 1)
            y_pred = torch.argmax(y_prob, dim=1)

            #To get y_pred, we convert logits to probability using softmax in case of multiclass).
            #Then use argmax to get the label
            #This is not needed for training since we use BCE with LogitLoss which can directly use logits. However, we do it to calculate metrics.

            y_true_train.extend(y.tolist()) #accumulate true label
            y_prob_train.extend(y_prob.tolist())
            y_pred_train.extend(y_pred.tolist()) #accumulate pred label

            # 2. Calculate loss (per mini batch)
            #minibatch gradient descent calculate the loss and optimize the weight per mini batches
            #as opposed to calculating and optimizing after all data has bee seen in an epoch
            loss = loss_fn(y_logits, y.type(torch.int64)) #Directly pass logits for Cross Entropy loss

            train_loss += loss # accumulatively add up the loss from ech minibatches fo tracking purpose

            # 3. Optimizer zero grad
            optimizer.zero_grad()

            # 4. Loss backward
            loss.backward()

            # 5. Optimizer step
            optimizer.step() #optimize network per mini batches batch

            # Print out how many samples or batch that have been seen every x iteration
            #sample
            # if batch % 1 == 0:
            #     print(f'Looked at {batch * len(X)}/{len(train_dataloader)} samples', end='\r')
            #batch
            print(f'Number of minibatch: {batch}/{len(train_dataloader)} batches', end='\r')

        #Find average train loss per minibatch
        train_loss /= len(train_dataloader)

        #Calculate metrics for that epoch using the accumulated label
        train_accuracy = accuracy_score(y_true_train, y_pred_train)
        train_balanced_accuracy = balanced_accuracy_score(y_true_train, y_pred_train, adjusted = True)
        train_f1_score = f1_score(y_true_train, y_pred_train, average = 'macro', zero_division = 'warn')
        train_roc_auc_score = roc_auc_score(y_true_train, y_prob_train, average = 'macro', multi_class = 'ovo') #Documentation on Sklearn stated that ovo with macro si insensitive to class imbalance
        train_precision = precision_score(y_true_train, y_pred_train, average = 'macro', zero_division = 'warn')
        train_recall = recall_score(y_true_train, y_pred_train, average = 'macro', zero_division = 'warn')

        ### val
        # Setup variables for accumulatively adding up loss and accuracy for that epoch
        val_loss = 0
        model.eval()

        with torch.inference_mode(): # context management
            for X, y in val_dataloader: # Loop through all mini batch in the val set. We don't use enumerate for this because
                # We don't care to track what test minibatch the progress is at.
                X, y = X.to(device), y.to(device)

                val_logits = model(X.input_ids.squeeze(), X.attention_mask.squeeze())
                val_prob = torch.softmax(val_logits, dim = 1)
                val_pred = torch.argmax(val_prob, dim=1)

                # 2. Calculate loss (accumatively)
                val_loss += loss_fn(val_logits, y.type(torch.int64)) # accumulatively add up the loss per mini batch for that epoch

                # 3. accumulate label for metric calculation
                y_true_val.extend(y.tolist()) #accumulate true label
                y_prob_val.extend(val_prob.tolist()) #accumualte predicted prob
                y_pred_val.extend(val_pred.tolist()) #accumulate pred label

            # Calculations on val metrics need to happen inside torch.inference_mode()
            # Divide total val loss by length of val dataloader (per batch)
            val_loss /= len(val_dataloader)

            #Now you get the average test loss per minibatch for that epoch

            # For metric
            # Calculate metric based on accumulate true and pred label
            val_accuracy = accuracy_score(y_true_val, y_pred_val)
            val_balanced_accuracy = balanced_accuracy_score(y_true_val, y_pred_val, adjusted = True)
            val_f1_score = f1_score(y_true_val, y_pred_val, average = 'macro', zero_division = 'warn')
            val_roc_auc_score = roc_auc_score(y_true_val, y_prob_val, average = 'macro', multi_class = 'ovo') #Documentation on Sklearn stated that ovo with macro si insensitive to class imbalance
            val_precision = precision_score(y_true_val, y_pred_val, average = 'macro', zero_division = 'warn')
            val_recall = recall_score(y_true_val, y_pred_val, average = 'macro', zero_division = 'warn')

            #for save model
            #check if the loss is less than the best val loss (using loss as threshold). You can change this to any metric.
            if val_loss < val_loss_best:
                val_loss_best = val_loss
                best_epoch = epoch
                best_checkpoint = {
                    'best_epoch': best_epoch,
                    'best_val_loss': val_loss_best,
                    'model_state_dict': model.state_dict(),
                    'optimizer_state_dict': optimizer.state_dict()
                }

        #Printing of statistic every x epoch. Default to every epoch.
        if epoch % 1 == 0:

            epoch_count.append(epoch) #append the number of epoch to be used as X axis to plot loss

            train_loss_values.append(train_loss.item()) #append data for train loss to plot

            val_loss_values.append(val_loss.item()) #append data for val loss to plot

            #append data for metric to plot
            train_accuracy_values.append(train_accuracy)
            train_balanced_accuracy_values.append(train_balanced_accuracy)
            train_f1_values.append(train_f1_score)
            train_roc_auc_values.append(train_roc_auc_score)
            train_precision_values.append(train_precision)
            train_recall_values.append(train_recall)

            val_accuracy_values.append(val_accuracy)
            val_balanced_accuracy_values.append(val_balanced_accuracy)
            val_f1_values.append(val_f1_score)
            val_roc_auc_values.append(val_roc_auc_score)
            val_precision_values.append(val_precision)
            val_recall_values.append(val_recall)

        ## Print out what's happening
        print(f"Train loss: {train_loss:.5f} | Val loss: {val_loss:.5f}")
        print(f"Train accuracy: {train_accuracy:.5f} | Train balanced accuracy: {train_balanced_accuracy:.5f} | Train F1 score: {train_f1_score:.5f} | Train ROC AUC score: {train_roc_auc_score:.5f}")
        print(f"Validation accuracy: {val_accuracy:.5f} | Validation balanced accuracy: {val_balanced_accuracy:.5f} | Validation F1 score: {val_f1_score:.5f} | Val ROC AUC score: {val_roc_auc_score:.5f}\n")

    #Gather results into dictionary
    results = {'epochs': epoch_count,
               'best_epoch': best_epoch,
               'train_loss_values': train_loss_values,
               'val_loss_values': val_loss_values,
               'train_accuracy_values': train_accuracy_values,
               'val_accuracy_values': val_accuracy_values,
               'train_balanced_accuracy_values': train_balanced_accuracy_values,
               'val_balanced_accuracy_values': val_balanced_accuracy_values,
               'train_f1_values': train_f1_values,
               'val_f1_values': val_f1_values,
               'train_roc_auc_values': train_roc_auc_values,
               'val_roc_auc_values': val_roc_auc_values,
               'train_precision_values': train_precision_values,
               'val_precision_values': val_precision_values,
               'train_recall_values': train_recall_values,
               'val_recall_values': val_recall_values
    }

    #Saving model with the best epoch.
    print(f'The best model is at epoch {best_epoch}')
    save_model(best_checkpoint, target_dir=target_dir, model_name=f'{model_name}_best_epoch_{best_epoch}.pth')

    #Saving the result dictionary in the same path which the model checkpoint is saved.
    results_file_path = f'{target_dir}\{model_name}_best_epoch_{best_epoch}_train_val_results.pkl'
    print(f'Saving results dictionary as pickle file at ({results_file_path})')
    with open(results_file_path, 'wb') as file:  # Open the file in write mode
        # Write the dictionary to the pickle file
        pickle.dump(results, file)

    #If want to open, do it like this:
    # Open the file in read mode and load the data
    # with open('..\model\BERT_LSTM_base_best_epoch_123_test_results.pkl', 'rb') as f:
    #     loaded_results = pickle.load(f)

    #plot and save curves
    # Create target directory
    figure_dir_path = Path(figure_dir)
    figure_dir_path.mkdir(parents=True,
                        exist_ok=True)
    figure_save_path = figure_dir_path

    print(f'Plotting and saving figure at: {figure_save_path}')
    # Plot the loss plot
    plt.figure(figsize=(8, 4))
    plt.plot(results['epochs'], results['train_loss_values'], label="Train loss")
    plt.plot(results['epochs'], results['val_loss_values'], label="Val loss")
    plt.axvline(results['best_epoch'], linestyle='--', color='r',label=f'Best Epoch - {best_epoch} (lowest val loss)')
    plt.title(f'{model_name} - Training/Val loss curves')
    plt.ylabel("Loss")
    plt.xlabel("Epochs")
    plt.legend()
    plt.savefig(f'{figure_save_path}\{model_name}_best_epoch_{best_epoch}_train_val_loss_curves.svg', format='svg')
    plt.show()

    # Plot the metric
    plt.figure(figsize=(8, 4))
    plt.plot(results['epochs'], results['train_accuracy_values'], label="Train Accuracy")
    plt.plot(results['epochs'], results['train_balanced_accuracy_values'], label="Train Balanced Accuracy")
    plt.plot(results['epochs'], results['train_f1_values'], label="Train F1 (Macro)")
    plt.plot(results['epochs'], results['val_accuracy_values'], label="Val Accuracy")
    plt.plot(results['epochs'], results['val_balanced_accuracy_values'], label="Val Balanced Accuracy")
    plt.plot(results['epochs'], results['val_f1_values'], label="Val F1 (Macro)")
    plt.axvline(results['best_epoch'], linestyle='--', color='r',label=f'Best Epoch - {best_epoch} (lowest val loss)')
    plt.title(f'{model_name} - Train/Val metric 1')
    plt.ylabel("Score")
    plt.xlabel("Epochs")
    plt.legend()
    plt.savefig(f'{figure_save_path}\{model_name}_best_epoch_{best_epoch}_train_val_metric_1.svg', format='svg')
    plt.show()

    # Plot the additional metric
    plt.figure(figsize=(8, 4))
    plt.plot(results['epochs'], results['train_roc_auc_values'], label="Train ROC AUC (Macro - OVO)")
    plt.plot(results['epochs'], results['train_precision_values'], label="Train Precision (Macro)")
    plt.plot(results['epochs'], results['train_recall_values'], label="Train Recall (Macro)")
    plt.plot(results['epochs'], results['val_roc_auc_values'], label="Val ROC AUC (Macro - OVO)")
    plt.plot(results['epochs'], results['val_precision_values'], label="Val Precision (Macro)")
    plt.plot(results['epochs'], results['val_recall_values'], label="Val Recall (Macro)")
    plt.axvline(results['best_epoch'], linestyle='--', color='r',label=f'Best Epoch: {best_epoch} (lowest val loss)')
    plt.title(f'{model_name} - Train/Val metric 2')
    plt.ylabel("Score")
    plt.xlabel("Epochs")
    plt.legend()
    plt.savefig(f'{figure_save_path}\{model_name}_best_epoch_{best_epoch}_train_val_metric_2.svg', format='svg')
    plt.show()

    return results


from django.urls import path
from . import views

urlpatterns = [
    # home

    path('', views.index, name="home"),

    #     authentication part

    path('login/', views.login, name="login"),
    path('logout/', views.logout, name="logout"),

    #     admin part

    path('fadmin', views.admin_dashboard, name="admin_dashboard"),
    path('admin-member-list', views.admin_member_list,
         name="admin_member_list"),
    path('admin-librarian-list', views.admin_librarian_list,
         name="admin_librarian_list"),
    path('admin-student-list', views.admin_student_list,
         name="admin_student_list"),
    path('member-add', views.admin_member_add,
         name="admin_member_add"),
    path('librarian-delete/<int:id>/', views.admin_member_delete,
         name="admin_member_delete"),
    path('<int:id>/', views.admin_member_update,
         name="admin_member_update"),

    # librarian part

    path('ldashboard', views.LibrarianView, name="librarian_dashboard"),
    path('librarian-book-list', views.librarian_book_list,
         name="librarian_book_list"),
    path('book-add', views.librarian_book_add,
         name="librarian_book_add"),
    path('book-delete/<int:id>/', views.librarian_book_delete,
         name="librarian_book_delete"),
    path('book/<int:id>/', views.librarian_book_update,
         name="librarian_book_update"),

    # student part
    path('sdashboard', views.StudentView, name="student_dashboard"),
    path('student-book-list', views.student_book_list,
         name="student_book_list"),
    path('reading-list/<int:id>/', views.add_to_reading_list,
         name="add_to_reading_list"),
    path("reading-book-list", views.reading_book_list, name="reading_book_list"),
    path('read-delete/<int:id>/', views.reading_list_book_delete,
         name="reading_list_book_delete"),

]


import sqlite3
import os
from tkinter import INSERT
from unicodedata import name
import spotipy
from spotipy.oauth2 import SpotifyClientCredentials
import re

connection = sqlite3.connect("music-data.db")

cur = connection.cursor()

with open('client_id.txt') as f:
    client_id = f.read()
    f.close()

with open('client_secret.txt') as f:
    client_secret = f.read()
    f.close()

sp = spotipy.Spotify(auth_manager=SpotifyClientCredentials(client_id, client_secret))

def get_input_url():
    inputlink = ""
    print("Enter sharing link for a playlist or song")
    inputlink = input()

    type = re.findall("playlist|track", inputlink)[0]
    relevantpart = re.findall("/[0-9a-zA-Z]*\?", inputlink)
    id = re.findall("[0-9a-zA-Z]+", relevantpart[0])[0]

    output = [type, id]
    return output

def get_playlist_items(playlist_id):
    result = sp.playlist_items(playlist_id)
    items = result["items"]
    track_list = []
    track_count = 0
    for item in items:
        trackid = item["track"]["id"]
        track_list.append(trackid)
        track_count += 1

    output = [track_list, track_count]
    return output

def add_track_db(id):

    trackname, trackartists, trackalbum, trackduration, trackid = get_track_data(id)

    #write into track list
    count = 0
    readout = cur.execute('SELECT spotify_id FROM tracklist WHERE spotify_id=?', (trackid,))
    for i in readout:
        count += 1
    if count == 0:
        cur.execute('INSERT INTO tracklist (title, spotify_id, duration_ms) VALUES (?,?,?)', (trackname, trackid, trackduration))

        # #write into interpret list and track-interpret-relations
        cur.execute('SELECT idtrack FROM tracklist WHERE spotify_id=?', (trackid,))
        idtrack = int(cur.fetchone()[0])

        for artistid in trackartists:
            count = 0
            readout = cur.execute('SELECT spotify_id FROM interpreten WHERE spotify_id=?', (artistid,))
            for i in readout:
                count += 1
            if count == 0:
                name = get_artist_data(artistid)
                cur.execute('INSERT INTO interpreten (name, spotify_id) VALUES (?,?)', (name, artistid))

            cur.execute('SELECT idinterpret FROM interpreten WHERE spotify_id=?', (artistid,))
            idinterpret = int(cur.fetchone()[0])

        cur.execute('SELECT idinterpret FROM interpreten WHERE spotify_id=?', (trackartists[0],))
        idinterpret1 = int(cur.fetchone()[0])
        cur.execute('INSERT INTO track_interpret_relations (idtrack, idinterpret) VALUES (?,?)', (idtrack, idinterpret))
        # #write into album list and create relations
        count = 0
        readout = cur.execute('SELECT spotify_id FROM albums WHERE spotify_id=?', (trackalbum,))

        for i in readout:
            count += 1
        if count == 0:
            name = get_album_data(trackalbum)
            cur.execute('INSERT INTO albums (album_name, spotify_id) VALUES (?,?)', (name, trackalbum))
            cur.execute('SELECT idalbum FROM albums WHERE spotify_id=?', (trackalbum,))
            idalbum = int(cur.fetchone()[0])
            cur.execute('INSERT INTO album_interpret_relations (idalbum, idinterpret) VALUES (?,?)', (idalbum, idinterpret1))
        else:
            cur.execute('SELECT idalbum FROM albums WHERE spotify_id=?', (trackalbum,))
            idalbum = int(cur.fetchone()[0])

        cur.execute('INSERT INTO album_track_relations (idalbum, idtrack) VALUES (?,?)', (idalbum, idtrack))

        #write into relations

        print("------- ", id, "imported")
    else:
        print("------- ", id, "already in database")

def get_track_data(id):
    trackartists = []

    raw_data = sp.track(id)

    trackname = raw_data['name']

    for key in raw_data['artists']:
        trackartists.append(key['id'])

    trackalbum = raw_data['album']['id']
    trackduration = raw_data['duration_ms']
    trackid = raw_data['id']

    return(trackname, trackartists, trackalbum, trackduration, trackid)

def get_album_data(id):
    raw_data = sp.album(id)
    name = raw_data['name']
    return(name)

def get_artist_data(id):
    raw_data = sp.artist(id)
    name = raw_data['name']
    return(name)

#https://open.spotify.com/track/3mXOssdHHkN6AvLw6ZgKiL?si=e1139c79407844bc = same energy - kid

#example to get info of the song
# result = sp.track("5L0JMH1LRJrsjbjFhWXB3k")
# for key in result:
#     relevant_data = [
#         'name',
#         'artists',
#         'album',
#         'duration_ms',
#         'id'
#     ]
# for data_type in relevant_data:
#     print(result[data_type])

#programm starts here:

items_to_import = []
type, id = get_input_url()
print(type, id)
if type == "playlist":
    items_to_import, item_count = get_playlist_items(id)
else:
    item_count = 1
    items_to_import.append(id)

if item_count == 1:
    print("importing 1 track into database")
    add_track_db(id)
else:
    print("importing", item_count, "tracks into database")
    for i in items_to_import:
        add_track_db(i)

connection.commit()
connection.close()


"""
This is..
Welcome to your very own [drumroll]

_Behavior Modification Stimulus Toolkit_

i.e. a package designed to produce stimuli
designed to produce behavior modification.

TBDetermined

_Neural Feedback Training (NFT)_

For starters, here, will wrap
* espeak
* xmessage
to issue you commands according
to a timer and/or randomness.
You may also want to learn
to add text to pics or videos
to carry the commands.

Most of what will interest you here
is the data structure(s) containing
the commands and how they are issued.

Try to divide Stimuli into
* work
* hobby
* break
 ***
"""

import time
import multiprocessing as mp
import datetime as dt

#
# example of trimming a video
#
# ffmpeg -i ${in}.mp4 -ss 00:00:45 -to 00:02:50 -c:v copy -c:a copy ${out}.mp4
#

#
# example of extracting audio
#
# ffmpeg -i input.mp4 output.wav
#

###
# all this is super-buggy.
# * message timer a one-call that's-all
# * add espeak to message time
# * multimedia kickoffs
# * desktop-switching...
#   ... set a timer, run a task
#    (where tasks are composable)

def _desktop_message(text):
    p = subps.Popen(['xmessage', text])

# todo: consider stdlib threading.Timer
class Timer:
    """
    This is a countdown.
    Call a countup something else,
      a Watch, maybe.
    """

    def __init__(self,
                 dur_min,
                 text='hi there',
                 subps_args=[]):
        self._dur_min = dur_min
        self._text = text
        self._subps_args = subps_args
        self._start = dt.datetime.now()
        self._has_acted = False

    def min_left(self):
        time_delta = (dt.datetime.now() -
                      self._start)
        return (
            time_delta.total_seconds() /
            60.0)

    def poll(self):
        if self._has_acted:
            return 'already'
        if (self.min_left() <
            self._dur_min):
            return 'running'
        if self._text != '':
            _desktop_message(self._text)
        if len(self._subps_args) > 0:
            self._sub_proc = subps.Popen(
                self._subps_args,
            )
        self._has_acted = True
        return 'now'

__timers = []
__timer_heartbeat_proc = None

def set_message_timer(dur_min, text):
    __timers.append(Timer(
        dur_min,
        text=text,
    ))

def _timer_heartbeat_main():
    for timer in __timers:
        action = timer.poll()
    time.sleep(60)

def start_timer_heartbeat():
    if __timer_heartbeat_proc is not None:
        if __timer_heartbeat_proc.is_alive():
            __timer_heartbeat_proc.terminate()
        __timer_heartbeat_proc.join()
    __timer_heartbeat_proc = mp.Process(
        _timer_heartbeat_main,
    )
    __timer_heartbeat_proc.start()


# https://dodona.ugent.be/nl/courses/264/series/3260/activities/659392988/#
import functools

class Maan:
    def __init__(self, val):
        if not (isinstance(val, (int, float)) and val >= 0):
            raise AssertionError("ongeldige waarde")
        self.val = int(val)

    def __int__(self):
        return self.val

    def __str__(self):
        return str(self.val)

    def __repr__(self):
        return f"Maan({self.val})"

    def __add__(self, other):
        temp = ""
        if isinstance(other, Maan):
            smallest: int = self.val if len(str(self.val)) < len(
                str(other.val)) else other.val
            largest: int = self.val if len(str(self.val)) >= len(
                str(other.val)) else other.val
        else:
            smallest: int = self.val if len(
                str(self.val)) < len(str(other)) else other
            largest: int = self.val if len(
                str(self.val)) >= len(str(other)) else other
        for _ in range(0, len(str(largest))):
            temp += str(smallest %
                        10) if smallest % 10 > largest % 10 else str(largest % 10)
            smallest //= 10
            largest //= 10

        return Maan(int(temp[::-1]))

    def __radd__(self, other):
        return self + other

    def __mul__(self, other):
        mult = []

        if isinstance(other, Maan):
            smallest: int = self.val if len(str(self.val)) < len(
                str(other.val)) else other.val
            largest: int = self.val if len(str(self.val)) >= len(
                str(other.val)) else other.val
        else:
            smallest: int = self.val if len(
                str(self.val)) < len(str(other)) else other
            largest: int = self.val if len(
                str(self.val)) >= len(str(other)) else other
        for el in str(smallest)[::-1]:
            temp = "0" * len(mult)
            for k in str(largest)[::-1]:
                temp += el if el < k else k

            mult.append(Maan(int(temp[::-1])))
        return functools.reduce(lambda a, b: a + b, mult)

    def __rmul__(self, other):
        return self * other

    def __pow__(self, power):
        if isinstance(power, Maan):
            power = power.val
        temp = self
        for _ in range(0, power - 1):
            temp *= self
        return temp

    def __rpow__(self, other):
        return Maan(other) ** self


import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
from transformers import BertTokenizer

class GBVDataset(Dataset):
    """Custom Dataset for the VBG dataset"""

    def __init__(self, texts, labels,  max_len):
        self.texts = texts
        self.labels = labels
        self.tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
        self.max_len = max_len

    def __len__(self):
        return len(self.texts)

    def __getitem__(self, item):
        text = str(self.texts[item])
        label = self.labels[item]
        tokenized_text = self.tokenizer(
            text,
            max_length=self.max_len,
            padding='max_length',
            add_special_tokens=True,
            return_tensors='pt'
        )

        return {
            'ids': tokenized_text["input_ids"].flatten(),
            'mask': tokenized_text["attention_mask"].flatten(),
            'token_type_ids': tokenized_text["token_type_ids"].flatten(),
            'labels': torch.tensor(label, dtype=torch.long)

        }

def VBGDataloader(df ,batch_size, max_len):

    train_dataset = GBVDataset(
                            texts=df.tweet.values,
                            labels=df.label.values,
                            max_len=max_len
                        )

    train_loader = DataLoader(
                                train_dataset,
                                batch_size=batch_size,
                                shuffle=False

                            )

    return train_loader


import discord
import os
import asyncio
import time

client = discord.Client()

@client.event
async def on_ready():
    print('Logged in as')
    print(client.user.name)

guildids = ["519451035836874763"]
channelids = ["519490720583778307", "520494704442212382", "519491305324281857"]

@client.event
async def on_message(message):
    if message.author.bot:
        return
    if message.channel.id not in channelids and message.server.id in guildids:
        def check(message):
            return message.content
        spam = await client.wait_for_message(author=message.author, channel= message.channel,check=check, timeout=1)
        spam1 = await client.wait_for_message(author=message.author, check=check, timeout=1, channel= message.channel)
        spam2 = await client.wait_for_message(author=message.author, check=check, timeout=1, channel= message.channel)
        await client.delete_message(spam)
        await client.delete_message(spam1)
        await client.delete_message(spam2)
    else:
      return

client.run(os.environ['Token'])


import logging
import numpy as np
import os
import pandas as pd

from typing import Any, Dict, Tuple

# Create logger
log = logging.getLogger(__name__)

def load_latest_date_file(oag_data_folder_path: str) -> pd.DataFrame:
    # Load latest file
    dfs = []
    for filename in os.listdir(oag_data_folder_path):
        if not filename.endswith('.csv'):
            continue

        file_path = oag_data_folder_path + '/' + filename
        dfs.append(pd.read_csv(file_path))

    data = pd.concat(dfs)
    return data

def previous_year_capacity(data: pd.DataFrame) -> pd.DataFrame:
    # Process dates
    data['DepLocalDate'] = pd.to_datetime(data['DepLocalDate'])
    data['futureDate'] = data['DepLocalDate'] + pd.DateOffset(years=1)

    # Get previous year figures
    data = data.merge(
        data.copy(),
        how='outer',
        left_on=['DepCountryCode', 'ArrCountryCode', 'DepLocalDate'],
        right_on=['DepCountryCode', 'ArrCountryCode', 'futureDate'],
        suffixes=('', '_PrevYear'))
    data.loc[pd.isnull(data['DepLocalDate']), 'DepLocalDate'] = data['futureDate_PrevYear']

    # Fill nulls with zeros for dates for which we have previous data for
    prev_cols = ['SchedFlightCount_PrevYear', 'CancellationCount_PrevYear']

    # mask = data.previousDate_PrevYear >= data.DepLocalDate.min()
    # data.loc[mask, prev_cols] = data.loc[mask, prev_cols].fillna(0)

    data[prev_cols] = data[prev_cols].fillna(0)

    # Keep and rename necessary columns
    data = data[['DepLocalDate', 'DepCountryCode', 'ArrCountryCode', 'SchedFlightCount',
                 'CancellationCount', 'SchedFlightCount_PrevYear', 'CancellationCount_PrevYear']]
    data.columns = ['date', 'country_code_destination', 'country_code_origin', 'sched_flight_count',
                    'cancellation_count', 'sched_flight_count_prev_year',
                    'cancellation_count_prev_year']

    return data

# def merge_innovata_data(
# 	data: pd.DataFrame,
# 	innovata_data: pd.DataFrame,
# 	innovata_country_mappings: pd.DataFrame) -> pd.DataFrame:

# 	# Add country code mappings
# 	innovata_data = innovata_data \
# 		.merge(
# 			innovata_country_mappings,
# 			how='left',
# 			left_on='Origin Country',
# 			right_on='country_name') \
# 		.merge(
# 			innovata_country_mappings,
# 			how='left',
# 			left_on='Destination Country',
# 			right_on='country_name',
# 			suffixes=('_origin', '_destination'))

# 	# Aggregate Innovata data
# 	innovata_data[['Flights', 'Seats']] = innovata_data[['Flights', 'Seats']].astype(float)
# 	innovata_data['Date'] = pd.to_datetime(innovata_data['Date'])

# 	innovata_agg = innovata_data \
# 		.groupby(['Date', 'country_code_origin', 'country_code_destination']) \
# 		.agg({
# 			'Flights': np.sum,
# 			'Seats': np.sum,
# 		}) \
# 		.rename(columns={
# 			'Flights': 'Flights_Innovata',
# 			'Seats': 'Seats_Innovata',
# 		}) \
# 		.reset_index()
# 	innovata_agg.columns = [col.lower() for col in innovata_agg.columns]

# 	# TODO: Remove once the data has actual dates rather than a monthly view
# 	# Adds zeroes for foward filling
# 	dates = innovata_agg[['date']].drop_duplicates().dropna()
# 	dates['dummy'] = 1
# 	ond = innovata_agg[['country_code_origin', 'country_code_destination']].drop_duplicates(
# 	).dropna()
# 	ond['dummy'] = 1

# 	idx_df = dates \
# 		.merge(ond) \
# 		.drop('dummy', axis=1)

# 	innovata_agg = innovata_agg.merge(
# 		idx_df,
# 		how='outer',
# 		on=['date', 'country_code_origin', 'country_code_destination']) \
# 		.fillna(0)

# 	# Merge datasets
# 	data = data.merge(
# 		innovata_agg,
# 		how='outer',
# 		on=['date', 'country_code_origin', 'country_code_destination'])

# 	# TODO: Remove once the data has actual dates rather than a monthly view
# 	dates = data[['date']].drop_duplicates().dropna()
# 	dates['dummy'] = 1

# 	idx_df = dates \
# 		.merge(ond) \
# 		.drop('dummy', axis=1)

# 	idx = ['country_code_origin', 'country_code_destination', 'date']
# 	cols = ['flights_innovata', 'seats_innovata']

# 	filled = idx_df.merge(
# 		data[idx + cols],
# 		how='left',
# 		on=idx)
# 	filled = filled.sort_values(idx)
# 	filled[cols] = filled[cols].fillna(method='ffill')

# 	data = data \
# 		.drop(cols, axis=1) \
# 		.merge(
# 			filled,
# 			how='outer',
# 			on=idx)

# 	# Return data
# 	return data

def merge_schedule_data(
        data: pd.DataFrame,
        schedule_data: pd.DataFrame) -> pd.DataFrame:
    # Change column names to match actual flight OAG data
    schedule_data = schedule_data.rename(columns={
        'Dep IATA Country Code': 'country_code_destination',
        'Arr IATA Country Code': 'country_code_origin',
        'Frequency': 'frequency',
        'Seats (Total)': 'seats',
        'Time series': 'date'
    })
    schedule_data.date = pd.to_datetime(schedule_data.date)

    # Merge datasets
    data = data.merge(
        schedule_data,
        how='outer',
        on=['date', 'country_code_origin', 'country_code_destination'])

    # Return data
    return data

def add_geographical_mappings(data: pd.DataFrame, country_mappings: pd.DataFrame) -> pd.DataFrame:
    # .rename(columns={
    # 	'DepCountryCode': 'country_code_destination',
    # 	'ArrCountryCode': 'country_code_origin',
    # 	'DepLocalDate': 'date',
    # 	'SchedFlightCount': 'sched_flight_count',
    # 	'CancellationCount': 'cancellation_count'}) \

    # Add mappings
    data = data \
        .merge(
        country_mappings[['code_2', 'country', 'region']],
        how='left',
        left_on='country_code_origin',
        right_on='code_2') \
        .drop(['code_2'], axis=1) \
        .merge(
        country_mappings[['code_2', 'country', 'region']],
        how='left',
        left_on='country_code_destination',
        right_on='code_2',
        suffixes=('_origin', '_destination')) \
        .drop(['code_2'], axis=1) \
 \
    # Domestic v. international
    data['travel_type'] = np.where(
        data['country_code_origin'] == data['country_code_destination'],
        'Domestic',
        np.where(
            data['region_origin'] == data['region_destination'],
            'Continental',
            'Intercontinental'
        ))

    # Capitalize columns
    data.columns = ['_'.join([word.capitalize() for word in col.split('_')]) for col in
                    data.columns]

    return data

def aggregate_data_by_orig_dest(data: pd.DataFrame) -> pd.DataFrame:
    # Aggregate data
    orig = data.groupby(['Country_Code_Origin', 'Region_Origin', 'Date', 'Travel_Type']) \
        .agg({
        'Sched_Flight_Count': np.sum,
        'Cancellation_Count': np.sum
    }) \
        .reset_index()

    dest = data.groupby(['Country_Code_Destination', 'Region_Destination', 'Date', 'Travel_Type']) \
        .agg({
        'Sched_Flight_Count': np.sum,
        'Cancellation_Count': np.sum
    }) \
        .reset_index()

    # Join data
    joined = orig \
        .merge(
        dest,
        how='outer',
        left_on=['Country_Code_Origin', 'Region_Origin', 'Date', 'Travel_Type'],
        right_on=['Country_Code_Destination', 'Region_Destination', 'Date', 'Travel_Type'],
        suffixes=('_(Origin)', '_(Destination)')) \
        .rename(columns={
        'Country_Code_Origin': 'Country_Code',
        'Region_Origin': 'Region'}) \
        .drop(['Country_Code_Destination', 'Region_Destination'], axis=1)

    # Compute totals
    joined['Sched_Flight_Count'] = joined['Sched_Flight_Count_(Origin)'] + joined[
        'Sched_Flight_Count_(Destination)']
    joined['Cancellation_Count'] = joined['Cancellation_Count_(Origin)'] + joined[
        'Cancellation_Count_(Destination)']

    # Pivot data
    pivot = pd.pivot_table(
        joined,
        index=['Country_Code', 'Region', 'Date'],
        columns=['Travel_Type'],
        values=['Sched_Flight_Count_(Origin)', 'Cancellation_Count_(Origin)'],
        aggfunc='sum'
    )

    pivot.columns = ['%s - %s' % (a, b) for a, b in pivot.columns]

    # Compute totals
    pivot['Sched_Flight_Count_(Origin)'] = pivot[
        [col for col in pivot.columns if 'Sched_Flight_Count_(Origin)' in col]].sum(axis=1)
    pivot['Cancellation_Count_(Origin)'] = pivot[
        [col for col in pivot.columns if 'Cancellation_Count_(Origin)' in col]].sum(axis=1)
    result = pivot.reset_index()

    return result


import io
import os
dir_name = os.path.expanduser('~')
import shutil
import time

from data_handling import get_data_from_git_annex, delete_unnecessary_file_links, split_list_in_n_sized_chunks, extract_tar
from ocrd_workflow import init_img_dir, init_ocrd_workspace, add_img_to_mets, set_ocrd_workspace_identifier, run_ocrd_workflow, run_ocrd_workflow_parallel

if __name__ == '__main__':
    parallel_processing = True
    today = '_2020_02_18'

    root_path = dir_name + '/Qurator/qurator-data/DTA/images'
    ocrd_workflow_path = dir_name + '/Qurator/my_ocrd_workflow'
    data_target_path = dir_name + '/Qurator/used_data/ocr/dta_ocr' + today
    data_readme_path = data_target_path + '/README.md'
    ocrd_workspaces = dir_name + '/Qurator/used_data/ocrd_workspaces/workspaces' + today
    cwd = os.getcwd()
    calamari_target_path = os.path.join(data_target_path, 'calamari_ocr')
    tesseract_target_path = os.path.join(data_target_path, 'tesseract_ocr')

    if not os.path.isdir(calamari_target_path):
        os.makedirs(calamari_target_path)
    if not os.path.isdir(tesseract_target_path):
        os.makedirs(tesseract_target_path)
    if not os.path.isdir(ocrd_workspaces):
        os.mkdir(ocrd_workspaces)
    if not os.path.isfile(data_readme_path):
        with io.open(data_readme_path, mode='w') as f_in:
            f_in.write('# List of Processed Files\n\n')

    filename_chunks = list(split_list_in_n_sized_chunks(os.listdir(root_path), 10))
    sub_chunks = filename_chunks[2:3]
    start_time = time.time()

    for index, filename_chunk in enumerate(sub_chunks):

        os.chdir(root_path)

        #####################################################################
        #                                                                   #
        #  Get data from git annex, copy tar to workspace and delete links  #
        #                                                                   #
        #####################################################################

        for filename in filename_chunk:
            file_path = os.path.join(root_path, filename)

            get_data_from_git_annex(file_path)
            shutil.copy(file_path, ocrd_workspaces)
            delete_unnecessary_file_links(file_path)

        os.chdir(ocrd_workspaces)

        #################################
        #                               #
        #  Extract tar and delete them  #
        #                               #
        #################################

        for filename in filename_chunk:
            new_file_path = os.path.join(ocrd_workspaces, filename)
            extract_tar(new_file_path)
            os.remove(new_file_path)

        os.chdir(cwd)

        #######################################################
        #                                                     #
        #  Run OCRD workflow, save OCR and delete workspaces  #
        #                                                     #
        #######################################################

        for filename in filename_chunk:
            foldername = os.path.splitext(filename)[0]
            workspace_path = os.path.join(ocrd_workspaces, foldername)

            if not os.path.exists(os.path.join(workspace_path, 'OCR-D-IMG')):
                init_img_dir(workspace_path)

            # check: process only short documents
            #img_path = os.path.join(workspace_path, 'OCR-D-IMG')
            #if len(os.listdir(img_path)) > 10:
            #    continue

            init_ocrd_workspace(workspace_path)

            add_img_to_mets(workspace_path)

            set_ocrd_workspace_identifier(workspace_path, id='1234567890')

        # check if parallel processing is needed
        if parallel_processing:
            run_ocrd_workflow_parallel(ocrd_workspaces, ocrd_workflow_path)

        for workspace_path in os.listdir(ocrd_workspaces):
            workspace_name = workspace_path
            workspace_path = os.path.join(ocrd_workspaces, workspace_path)

            # check: process only short documents
            #img_path = os.path.join(workspace_path, 'OCR-D-IMG')
            #if len(os.listdir(img_path)) > 10:
            #    continue

            # if no parallel processing, run ocrd workflow on single workspace
            if not parallel_processing:
                run_ocrd_workflow(workspace_path, ocrd_workflow_path)

            try:
                calamari_path = os.path.join(workspace_path, 'OCR-D-OCR-CALAMARI')
                calamari_doc_target_path = os.path.join(calamari_target_path, workspace_name)
                tesseract_path = os.path.join(workspace_path, 'OCR-D-OCR-TESS')
                tesseract_doc_target_path = os.path.join(tesseract_target_path, workspace_name)

                mets_path = os.path.join(workspace_path, 'mets.xml')

                # save OCR to Calamari and Tesseract paths, respectively
                shutil.copytree(calamari_path, calamari_doc_target_path)
                shutil.copytree(tesseract_path, tesseract_doc_target_path)

                # save METS to Calamari and Tesseract paths, respectively
                os.path.copy(mets_path, calamari_doc_target_path)
                os.path.copy(mets_path, tesseract_doc_target_path)
            except FileNotFoundError as fnfe:
                print(fnfe)

            #shutil.rmtree(workspace_path)

            with io.open(data_readme_path, mode='a') as f_in:
                f_in.write(workspace_name + '\n')

        if index == 0:
            break

    end_time = time.time()
    print('Runtime: {}sec'.format(round(end_time-start_time, 2)))


import os
import time
import unittest
from unittest.mock import Mock
from threading import Thread
from shutil import rmtree

from src.scanner import FileScanner
from utils.settings import Settings

"""
    Testing thread to run scanner in separate thread to test
"""
class TestThread(Thread):
    def __init__(self, scanner):
        super(TestThread, self).__init__()
        self.scanner = scanner

    def run(self):
        self.scanner.run()

class TestScanner(unittest.TestCase):

    def setUp(self):
        # Set up testing directory
        self.test_dir = os.path.relpath("testFileDir", start=os.curdir)
        if not os.path.exists(self.test_dir):
            os.mkdir(self.test_dir)

        # Setup test file
        with open(os.path.join(self.test_dir, 'test.txt'), 'w') as f:
            f.write("Hello world!")
        f.close()

        # Setup event handler and file scanner
        Settings.client_dir = self.test_dir  # File scanner to watch testing directory
        dummy_handler = Mock()
        self.scanner = FileScanner(dummy_handler)
        self.thread = TestThread(self.scanner)

    def test_create(self):
        # Scanner should detect when a file is created
        self.thread.start()
        time.sleep(1)
        with open(os.path.join(self.test_dir, 'create.txt'), 'w') as f:
            f.write("Hello world!")
        time.sleep(Settings.scanner_interval + 1)
        self.scanner.handler.handle_create.assert_called()

    def test_update(self):
        # Scanner should detect when a file is updated
        self.thread.start()
        time.sleep(1)
        with open(os.path.join(self.test_dir, 'test.txt'), 'w') as f:
            f.write("New content")
        time.sleep(Settings.scanner_interval + 1)
        self.scanner.handler.handle_update.assert_called()

    def test_delete(self):
        # Scanner should detect when a file is deleted
        self.thread.start()
        time.sleep(1)
        file_path = os.path.join(self.test_dir, 'test.txt')
        os.remove(file_path)
        self.assertFalse(os.path.exists(file_path))
        time.sleep(Settings.scanner_interval + 1)
        self.scanner.handler.handle_delete.assert_called()

    def tearDown(self):
        self.scanner.running = False
        rmtree(self.test_dir)
        self.thread.join()


import pyodbc
import pandas as pd
import csv
import os

'''
mostrar_info_equipo
————————
conexion: Connection
equipo_seleccionado: string
————————
Toma el equipo seleccionado y se obtiene toda su informacion de la tabla, excluyendo su nombre e ID.
'''
def mostrar_info_equipo(conexion, equipo_seleccionado):
    cursor = conexion.cursor()

    try:
        cursor.execute("""
        SELECT *
        FROM FIFA
        WHERE Team = ?
        ORDER BY Year;
        """, equipo_seleccionado)

        info_equipo = cursor.fetchall()

        if info_equipo:
            print(f"\nInformacion del equipo {equipo_seleccionado}:\n")
            df = pd.DataFrame.from_records(info_equipo, columns=[desc[0] for desc in cursor.description])
            columns_to_exclude = ["ID", "Team"]
            df.drop(columns=columns_to_exclude, inplace=True)
            for year, grupo_df in df.groupby("Year"):
                print(f"Año {year}:")
                print(grupo_df.to_string(index=False))
                print('\n')
    except Exception as e:
        print("error", e)

'''
mostrar_menu_equipos
————————
conexion: Connection
————————
Toma los nombres de los equipos, los enumera y ordena alfabeticamente, luego los muestra por pantalla y permite escoger uno
de ellos. Se retorna el equipo escogido o 's' en caso de salir.
'''
def mostrar_menu_equipos(conexion):
    cursor = conexion.cursor()
    cursor.execute("""
    SELECT DISTINCT Team
    FROM FIFA;
    """)

    equipos = cursor.fetchall()
    print("Seleccione un equipo segun su numero:")

    contador = 0
    for i, equipo in enumerate(equipos, start=1):
        equipo_str = ', '.join(str(elemento) for elemento in equipo)
        print(f"{i}. {equipo_str}")
        contador = i
    print('\n')

    while True:
        numero_equipo = input("Ingrese su respuesta('s' para salir): ")
        if numero_equipo == 's':
            return 's'
        elif numero_equipo.isdigit():
            numero_equipo = int(numero_equipo)

            if numero_equipo > 0  and numero_equipo <= contador:
                return equipos[numero_equipo-1][0]
            else:
                print("Opcion no valida")
                print('\n')

        else:
            print("Opcion no valida")
            print('\n')

'''
verificar_tablas_existen
————————
conexion: Connection
table_names: lista de strings
————————
Toma table_names y la recorre para verificar que los nombres dentro de esta coincidan con las tablas
existentes en la base de datos, si existen ambas retorna True en caso contrario False.
'''
def verificar_tablas_existen(conexion, table_names):
    cursor = conexion.cursor()

    existing_tables = [table[2] for table in cursor.tables(tableType='TABLE')]
    existen = 0

    for table_name in table_names:
        if table_name in existing_tables:
            existen += 1

    if existen == 2:
        return True
    else:
        return False

'''
crear_tablas
————————
conexion: Connection
————————
Se hace un query para crear dos tablas, SUMMARY y FIFA respectivamente, cada una representada con
la informacion correspondiente, en el caso de FIFA se agrega una columna ID para crear una llave
primaria a esa tabla.
'''
def crear_tablas(conexion):
    cursor = conexion.cursor()
    create_table_query = '''
    CREATE TABLE SUMMARY (
        Year SMALLINT PRIMARY KEY NOT NULL,
        Host NVARCHAR(50) NOT NULL,
        Champion NVARCHAR(50) NOT NULL,
        Runner_up NVARCHAR(50) NOT NULL,
        Third_place NVARCHAR(50) NOT NULL,
        Teams SMALLINT NOT NULL,
        Matches_played SMALLINT NOT NULL,
        Goals_scored SMALLINT NOT NULL,
        Avg_goals_per_game FLOAT NOT NULL
    );

    CREATE TABLE FIFA (
        Position SMALLINT NOT NULL,
        Team NVARCHAR(50) NOT NULL,
        Games_Played SMALLINT NOT NULL,
        Win SMALLINT NOT NULL,
        Draw SMALLINT NOT NULL,
        Loss SMALLINT NOT NULL,
        Goals_For SMALLINT NOT NULL,
        Goals_Against SMALLINT NOT NULL,
        Goal_Difference SMALLINT NOT NULL,
        Points SMALLINT NOT NULL,
        Year SMALLINT NOT NULL FOREIGN KEY REFERENCES SUMMARY(Year),
        ID SMALLINT NOT NULL PRIMARY KEY
    );
    '''
    cursor.execute(create_table_query)
    conexion.commit()
    cursor.close()

'''
insertar_datos_SUMMARY
————————
conexion: Connection
ruta: string
————————
Lee el archivo de sumario usando ruta, salta la primera linea y luego hacee el query insertando
los datos a la tabla SUMMARY.
'''
def insertar_datos_SUMMARY(conexion,ruta):
    cursor = conexion.cursor()

    try:
        with open(ruta, 'r') as archivo:
            leer_csv = csv.reader(archivo)
            next(leer_csv)  # Salta la primera linea

            for fila in leer_csv:
                Year, Host, Champion, Runner_up, Third_place, Teams, Matches_played, Goals_scored, Avg_goals_per_game = fila
                Year = int(Year)

                insert_query = f'''
                INSERT INTO SUMMARY (Year, Host, Champion, Runner_up, Third_place, Teams, Matches_played, Goals_scored, Avg_goals_per_game)
                VALUES ({Year}, '{Host}', '{Champion}', '{Runner_up}', '{Third_place}', {Teams}, {Matches_played}, {Goals_scored}, {Avg_goals_per_game})
                '''
                cursor.execute(insert_query)

            conexion.commit()

    except pyodbc.Error as ex:
        print("Ocurrió un error al insertar datos:", ex)

    finally:
        cursor.close()

'''
insertar_datos_FIFA
————————
conexion: Connection
ruta: string
year: int
i: int
————————
Abre el archivo usando ruta, lee cada fila sin contar la primera, corrobora que esté bien escrita
y hace el query para insertar los datos a la tabla FIFA, se hace un contador de las filas el cual
incrementa por fila insertada (esto para tener una llave primaria llamada ID, que es el contador).
Finalmente retorna el contador para los archivos siguientes.
'''
def insertar_datos_FIFA(conexion,ruta,year,i):
    cursor = conexion.cursor()
    j = i # variable incremento para ID

    try:
        with open(ruta, 'r') as archivo:
            leer_csv = csv.reader(archivo)
            next(leer_csv)  # Salta la primera linea

            for fila in leer_csv:
                position, team, games_played, win, draw, loss, goals_for, goals_against, goal_difference, points = fila

                # Verifica que tenga el simbolo especial y reemplazarlo
                if 'âˆ’' in goal_difference:
                    goal_difference = goal_difference.replace('âˆ’','-')
                    goal_difference = int(goal_difference)

                if '*' in team:
                    team = team.replace('*','')

                insert_query = f'''
                INSERT INTO FIFA (Position, Team, Games_Played, Win, Draw, Loss, Goals_For, Goals_Against, Goal_Difference, Points, Year, ID)
                VALUES ({position}, '{team}', {games_played}, {win}, {draw}, {loss}, {goals_for}, {goals_against}, {goal_difference}, {points}, {year}, {j})
                '''
                cursor.execute(insert_query)
                j += 1

            conexion.commit()

    except pyodbc.Error as ex:
        print("Ocurrió un error al insertar datos:", ex)

    finally:
        cursor.close()
        return j


#Client Credentials Grant method
import config
import http.client
import json

connection = http.client.HTTPSConnection('www.com')
headers = {"Content-type":"application/x-www-form-urlencoded"}
body='grant_type=client_credentials&client_id=%s&client_secret=%s'%(config.client_id, config.client_secret )

#POST
print("getting token")
connection.request('POST', '/api/v2/oauth2/token.json', body, headers)
print("token ok")
responsepost = connection.getresponse()
result=json.loads(responsepost.read().decode("utf-8"))
token=result['access_token']

#GET
print("getting stats")
connection._send_request('GET', '/api/v2/campaigns.json',[], {"Authorization":"Bearer " + token})

responseget = connection.getresponse()
result=json.loads(responseget.read().decode("utf-8"))
print(result['items'])
print(responseget.read())


#This code will fetch the weather details for the given city name.
#It will make use of geopy module to get the location details.
#It will make use of weatherapi.com to get the current weather details of the city.

from geopy.geocoders import Nominatim
from constants import *
import requests
import streamlit as st
import pandas as pd

url = "https://weatherapi-com.p.rapidapi.com/current.json"
headers = {
	"X-RapidAPI-Key": f"{API_KEY}",
	"X-RapidAPI-Host": "weatherapi-com.p.rapidapi.com"
}

def get_location_lat_lon(city_name):
    '''
    This function will return the latitude and longitude of the given city name.
    '''
    geolocator = Nominatim(user_agent="MyApp")
    location = geolocator.geocode(f"{city_name}")
    if location != None:
        return location.latitude, location.longitude
    else:
        st.warning("Error Occured: Please Enter the City Name Again")
        st.stop()

def get_weather_data(url, headers, lat, lon):
    '''
    This function will return the weather details of the given latitude and longitude.
    '''
    querystring = {"q": f"{lat},{lon}"}
    response = requests.get(url, headers=headers, params=querystring)
    if response.status_code == 200:
        return response.json()
    else:
        st.error("Error Occured: Please try Again")
        st.stop()

def get_location_details(location):
    '''
    This function will return the location details of the given latitude and longitude.
    '''
    location_data = ''
    for k, v in location.items():
        if "epoch" in k:
            continue
        elif k in CHANGABLE_KEY_ITEMS and type(CHANGABLE_KEY_ITEMS[k]) == list:
            st.write(f"{CHANGABLE_KEY_ITEMS[k][0].title()}: {v} {CHANGABLE_KEY_ITEMS[k][1]}")
            location_data += f"{CHANGABLE_KEY_ITEMS[k][0]} is {v} {CHANGABLE_KEY_ITEMS[k][1]} and "
        elif k in CHANGABLE_KEY_ITEMS and type(CHANGABLE_KEY_ITEMS[k]) == str:
            st.write(f"{CHANGABLE_KEY_ITEMS[k].title()}: {v}")
            location_data += f"{CHANGABLE_KEY_ITEMS[k]} is {v} and "
        else:
            st.write(f"{k.capitalize()}: {v}")
            location_data += f"{k} is {v} and "
    st.title("Location Summary")
    st.write(location_data[:-5])

def get_current_weather_details(current):
    '''
    This function will return the current weather details of the given latitude and longitude.
    '''
    current_data = ''
    for k, v in current.items():
        if "epoch" in k or type(v) == dict:
            continue
        elif k in CHANGABLE_KEY_ITEMS and type(CHANGABLE_KEY_ITEMS[k]) == list:
            st.write(f"{CHANGABLE_KEY_ITEMS[k][0].title()}: {v} {CHANGABLE_KEY_ITEMS[k][1]}")
            current_data += f"{CHANGABLE_KEY_ITEMS[k][0]} is {v} {CHANGABLE_KEY_ITEMS[k][1]} and "
        elif k in CHANGABLE_KEY_ITEMS and type(CHANGABLE_KEY_ITEMS[k]) == str:
            st.write(f"{CHANGABLE_KEY_ITEMS[k].title()}: {v}")
            current_data += f"{CHANGABLE_KEY_ITEMS[k]} is {v} and "
        else:
            st.write(f"{k.capitalize()}: {v}")
            current_data += f"{k} is {v} and "
    st.title("Current Weather Summary")
    st.write(current_data[:-5])

if __name__ == "__main__":
    st.title("Weather Forecast App")
    city_name = st.text_input("Please Enter the City Name:",key=1)
    state = st.button("Go!..")
    if state:
        lat,lon = get_location_lat_lon(city_name)
        st.write("Hey There! You have entered the city name as: ", city_name)
        st.write("Fetching the Weather Details for the City: ", city_name)
        my_weather_data = get_weather_data(url, headers, lat, lon)
        st.map(pd.DataFrame(my_weather_data["location"],index=[0]))
        st.image(f'https:{my_weather_data["current"]["condition"]["icon"]}',caption=f"Current Weather in {city_name.capitalize()} is {my_weather_data['current']['condition']['text']}")
        st.title("Location Details")
        get_location_details(my_weather_data["location"])
        st.json(my_weather_data["location"])
        st.title("Current Weather Details")
        get_current_weather_details(my_weather_data["current"])
        st.json(my_weather_data["current"])