import config
import pygame
import math

class static_shape_object:
    def __init__(self, position, vertices) -> None:
        ''' vertices are relative to given position'''
        self.vertices = vertices
        self.position = position
        self.colliding = False
        self.color = config.DEFAULT_SHAPE_COLOR
        self.world_vertices = list(map(lambda x: (x[0] + self.position[0], x[1] + self.position[1]), self.vertices))
        self.centralize_polygon_position()

    def draw(self, surface):
        pygame.draw.polygon(surface, self.color, self.world_vertices, config.POLYGON_BORDER_WIDTH)
        pygame.draw.circle(surface, (255,0,0), self.position, 2)
        fill_polygon_with_alpha(surface, self.world_vertices, self.color)

    def centralize_polygon_position(self):
        self.position = get_centroid(self.world_vertices)
        self.vertices = []
        for vertex in self.world_vertices:
            self.vertices.append(calc_vector(self.position, vertex))

class dynamic_shape_object(static_shape_object):
    def __init__(self, position, vertices) -> None:
        super().__init__(position, vertices)
        self.facing = [0, 1]
        self.moved = False

    def udpdate_world_verices(self):
        self.world_vertices = list(map(lambda x: (x[0] + self.position[0], x[1] + self.position[1]), self.vertices))

    def rotate(self, angle):
        ''' rotates vertices cunter-clockwise by given angle in radians'''
        cos = math.cos(angle)
        sin = math.sin(angle)
        for vertex in self.vertices:
            x, y = vertex
            vertex[0] = x * cos - y * sin
            vertex[1] = x * sin + y * cos

        x, y = self.facing
        self.facing[0] = x * cos - y * sin
        self.facing[1] = x * sin + y * cos
        self.moved = True

    def move_forwards(self):
        self.position[0] += self.facing[0] * config.MOVEMENT_MULTIPLIER
        self.position[1] += self.facing[1] * config.MOVEMENT_MULTIPLIER
        self.moved = True

    def move_backwards(self):
        self.position[0] -= self.facing[0] * config.MOVEMENT_MULTIPLIER
        self.position[1] -= self.facing[1] * config.MOVEMENT_MULTIPLIER
        self.moved = True

    def update_variables(self, world_shapes):
        if self.moved:
            self.handle_collisions(world_shapes)
            self.udpdate_world_verices()
            self.moved = False

    def handle_collisions(self, world_shapes):
        self_edges = generate_edge_data(self)
        moved = False
        for shape in world_shapes:
            if shape is not self:
                shape_edges = generate_edge_data(shape)
                for diag in self.vertices:
                    for point, dir in shape_edges:
                        vector = line_intersection(self.position, diag, point, dir)
                        if vector != None:
                            self.position[0] -= (diag[0] - vector[0])
                            self.position[1] -= (diag[1] - vector[1])
                            moved = True

                if moved:
                    self_edges = generate_edge_data(self)

                for diag in shape.vertices:
                    for point, dir in self_edges:
                        vector = line_intersection(shape.position, diag, point, dir)
                        if vector != None:
                            self.position[0] += (diag[0] - vector[0])
                            self.position[1] += (diag[1] - vector[1])
                            self_edges = generate_edge_data(self)
                            moved = True
        if moved:
            self.udpdate_world_verices()

def get_centroid(vertices):
    x = y = 0
    area = 0
    n = len(vertices)
    for i in range(n):
        p1 = vertices[i]
        p2 = vertices[(i + 1) % n]
        tmp = p1[0] * p2[1] - p2[0] * p1[1]
        area += tmp
        x += (p1[0] + p2[0]) * tmp
        y += (p1[1] + p2[1]) * tmp

    return [x / (3*area), y / (3*area)]

def generate_edge_data(shape):
    edge_data = []
    vertices = shape.vertices
    n = len(vertices)
    for i in range(n):
        vector = calc_vector(vertices[i], vertices[(i + 1) % n])
        position = (vertices[i][0] + shape.position[0], vertices[i][1] + shape.position[1])
        edge_data.append((position, vector))
    return edge_data

def line_intersection(p, r, q, s):
    ''' <p,q> - points,  <r,s> - vectors, if lines do intersect with each other return vetor from p to intersection point'''
    a = cross_product_2d(r, s)
    vec = calc_vector(p, q)
    b = cross_product_2d(vec, s)
    if a == 0:
        if b == 0:
            v1 = calc_vector(p, q)
            v2 = calc_vector(p, add_vectors(q, s))
            r_length = get_vector_length(r)
            if r_length != 0:
                r_versor = calc_versor(r, r_length)
                d_p = min(dot_product(r_versor, v1), dot_product(r_versor, v2))
                if 0 < d_p < r_length:
                    return multiply_vector_by_scalar(d_p, r)
    else:
        t = b / a
        s = cross_product_2d(vec, r) / a
        if 0 < t < 1 and 0 < s < 1:
            return multiply_vector_by_scalar(t, r)
    return None

def fill_polygon_with_alpha(surface, polygon, color):
    x_range = [math.inf, -math.inf]
    y_range = [math.inf, -math.inf]
    for vertex in polygon:
        x_range[0] = min(x_range[0], vertex[0])
        x_range[1] = max(x_range[1], vertex[0])
        y_range[0] = min(y_range[0], vertex[1])
        y_range[1] = max(y_range[1], vertex[1])

    surface_rect = (x_range[0], y_range[0], x_range[1] - x_range[0], y_range[1] - y_range[0])
    relative_vertices = list(map(lambda x: (x[0] - surface_rect[0], x[1] - surface_rect[1]), polygon))
    polygon_surface = pygame.Surface(surface_rect[2:])
    polygon_surface.set_colorkey((0, 0, 0))
    polygon_surface.set_alpha(config.POLYGON_FILL_ALPHA)
    pygame.draw.polygon(polygon_surface, color, relative_vertices)
    surface.blit(polygon_surface, surface_rect[:2])

def calc_vector(p1, p2):
    return [p2[0] - p1[0], p2[1] - p1[1]]

def get_vector_length(v):
    return math.sqrt(v[0]**2 + v[1]**2)

def calc_versor(v, length):
    length = math.sqrt(v[0]**2 + v[1]**2)
    return (v[0] / length , v[1] / length)

def add_vectors(v1, v2):
    return (v1[0] + v2[0], v1[1] + v2[1])

def subtract_vectors(v1, v2):
    return (v1[0] - v2[0], v1[1] - v2[1])

def multiply_vector_by_scalar(s, v):
    return (s * v[0], s * v[1])

def cross_product_2d(v1, v2):
    return v1[0]* v2[1] - v1[1]* v2[0]

def dot_product(v0, v1):
    return v0[0] * v1[0] + v0[1] * v1[1]


from collections import defaultdict, deque

def solve(path):
#只能解决小数据集
    border = 10 ** 9
    w = h = 0
    dictionary = defaultdict(int)
    d = {'N': 0, "S": 1, "E": 2, "W": 3}
    stack = deque()
    N = len(path)
    tmp = ''
    for i in range(N):
        if tmp=='':
            tmp+=path[i]
        elif path[i]=='(':
            stack.append(tmp)
            tmp=''
        elif tmp.isdigit() != path[i].isdigit():
            stack.append(tmp)
            tmp=path[i]
        elif path[i]==')':
            while stack and stack[-1].isdigit() == tmp.isdigit():
                tmp=stack.pop()+tmp
            if stack:
                tmp=int(stack.pop())*tmp
        else:
            tmp+=path[i]

    while stack:
        tmp=stack.pop()+tmp
    north=tmp.count('N')
    south=tmp.count('S')
    east=tmp.count('E')
    west=tmp.count('W')
    w=(east-west+1) % border
    h=(south-north+1) % border
    if w==0:w=border
    if h==0:h=border
    return w, h

T = int(input())
for t in range(T):
    path = input()
    w, h = solve(path)
    print('Case #%d: %d %d' % (t + 1, w, h))
#---------------------------------------------------------

def f(s):
    i = 0
    x, y = 0, 0
    while i < len(s):
        c = s[i]
        if c == 'N': y -= 1
        elif c == 'S': y += 1
        elif c == 'E': x += 1
        elif c == 'W': x -= 1
        else:
            r = int(c)
            j = i+2
            lvl = 1
            sub = ''
            while lvl:
                if s[j] == '(': lvl += 1
                if s[j] == ')': lvl -= 1
                sub += s[j]
                j += 1
            dx, dy = f(sub[:-1])
            x += dx * r
            y += dy * r
            i = j-1
        i += 1
    return x, y

M = int(1e9)
for t in range(input()):
    s = raw_input().strip()
    x, y = f(s)
    x %= M
    y %= M
    print('Case #%d: %d %d' % (t+1, x+1, y+1))


from openerp import fields, models, api
from datetime import datetime
from openerp.osv import osv
from openerp.addons.report_xlsx.report.report_xlsx import ReportXlsx
from dateutil.relativedelta import relativedelta
from dateutil import tz
from pytz import timezone

class AccountsReportWizard(models.TransientModel):
    _name = 'accounts.report.wizard'

    from_date = fields.Date('Date From')
    to_date = fields.Date('Date To')
    # location_id = fields.Many2one('stock.location', 'Location')

    month = fields.Selection([('January', 'January'),
                              ('February', 'February'),
                              ('March', 'March'),
                              ('April', 'April'),
                              ('May', 'May'),
                              ('June', 'June'),
                              ('July', 'July'),
                              ('August', 'August'),
                              ('September', 'September'),
                              ('October', 'October'),
                              ('November', 'November'),
                              ('December', 'December')], 'Month', )

    @api.onchange('month')
    def onchange_month(self):
        if self.month:
            date = '1 ' + self.month + ' ' + str(datetime.now().year)

            date_object = datetime.strptime(date, '%d %B %Y')
            self.from_date = date_object
            end_date = date_object + relativedelta(day=31)

            self.to_date = end_date

    @api.multi
    def generate_xls_report(self):

        return self.env["report"].get_action(self, report_name='AccountsReport.xlsx')

class BillReportXlsx(ReportXlsx):
    def generate_xlsx_report(self, workbook, data, invoices):
        worksheet = workbook.add_worksheet("AccountsReport")
        # raise UserError(str(invoices.invoice_no.id))

        boldc = workbook.add_format({'bold': True, 'align': 'center', 'bg_color': '#D3D3D3', 'font': 'height 10'})
        heading_format = workbook.add_format({'bold': True, 'align': 'center', 'size': 10})
        bold = workbook.add_format({'bold': True})
        rightb = workbook.add_format({'align': 'right', 'bold': True})
        right = workbook.add_format({'align': 'right'})
        regular = workbook.add_format({'align': 'center', 'bold': False, 'size': 8})
        merge_format = workbook.add_format({
            'bold': 1,
            'border': 1,
            'align': 'center',
            'valign': 'vcenter',
            'bg_color': '#D3D3D3',
            'font_color': '#000000',
        })
        format_hidden = workbook.add_format({
            'hidden': True
        })
        align_format = workbook.add_format({
            'align': 'right',
        })
        row = 6
        col = 1
        new_row = row

        worksheet.set_column('A:A', 13)
        worksheet.set_column('B:B', 25)
        worksheet.set_column('D:S', 13)

        worksheet.merge_range('A1:S1', 'Palathra Constructions', boldc)
        worksheet.merge_range('A3:S3', 'Accounts Details From %s To %s' % (
        datetime.strptime(invoices.from_date, "%Y-%m-%d").strftime("%d-%m-%Y"),
        datetime.strptime(invoices.to_date, "%Y-%m-%d").strftime("%d-%m-%Y")), boldc)
        worksheet.merge_range('A4:A5', 'Sl.NO', regular)
        worksheet.merge_range('B4:D4', 'Accounts', regular)
        worksheet.merge_range('E4:E5', 'Available Balance', regular)

        count = 1
        for rec in invoices:
            date_from = datetime.strptime(invoices.from_date, "%Y-%m-%d")
            date_to = datetime.strptime(invoices.to_date, "%Y-%m-%d")
            accounts = self.env['res.partner.bank'].search([('common_usage','=',True)])
            count =1
            for acc in accounts:
                if acc.account_balance>0:

                    worksheet.write('A%s' % (new_row), count, regular)
                    worksheet.merge_range('B%s:D%s'% (new_row,new_row), acc.bank_name + '-' + acc.acc_number, regular)
                    worksheet.write('E%s' % (new_row), acc.account_balance, regular)
                    count += 1
                    new_row += 1

            new_row +=3
            worksheet.merge_range('A%s:A%s'% (new_row,new_row), 'Sl.NO', regular)
            worksheet.merge_range('B%s:D%s'% (new_row,new_row), 'Accounts', regular)
            worksheet.merge_range('E%s:E%s'% (new_row,new_row), 'Available Balance', regular)
            worksheet.merge_range('F%s:F%s' % (new_row, new_row), 'Availed Balance', regular)
            accounts = self.env['res.partner.bank'].search([])
            count = 1
            for acc in accounts:

                worksheet.write('A%s' % (new_row), count, regular)
                worksheet.merge_range('B%s:D%s' % (new_row, new_row), acc.bank_name and acc.bank_name or ''  + '-' + acc.acc_number and acc.acc_number or '', regular)
                worksheet.write('E%s' % (new_row), acc.account_balance, regular)
                worksheet.write('F%s' % (new_row), acc.usable_balance, regular)
                count += 1
                new_row += 1

BillReportXlsx('report.AccountsReport.xlsx', 'accounts.report.wizard')


from plyer import notification
from rich.table import Table
from rich import print
from datetime import time
from config.config import *
from requests import get

import json
import os
import rich
import psutil
import platform
import pyaudio
import pyttsx3
import datetime
import requests
import random
import requests

try:
    from pyfirmata import Arduino, util, STRING_DATA

    board = Arduino('/dev/ttyUSB0') # Informe a porta do arduino
    arduino = True
except:
    arduino = False

#Configurações de dados
plataforma = platform.system()

if 'Windows' in plataforma:
    # Trás a função letícia voz
    sara_voz=pyttsx3.init('sapi5')

    # Função de ajuste de voz da sara
    voz = sara_voz.getProperty('voices')
    sara_voz.setProperty('voice', voz[2].id)
    rate = sara_voz.getProperty('rate')
    sara_voz.setProperty('rate', rate-50)

else:
    # Trás a função letícia voz # Se não for a voz leticia padrão é Espeak
    sara_voz=pyttsx3.init()

    # Função de ajuste de voz da sara
    sara_voz.setProperty('voice', 'en')
    # No 'm2'(masculino) pode colocar 'f2'(feminino) e números até 7
    rate = sara_voz.getProperty('rate')
    sara_voz.setProperty('rate', rate-50)

# Preparando o microfone para captura
p = pyaudio.PyAudio()

stream = p.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True, frames_per_buffer=8192)

def resposta(audio):
    notification.notify(title = "SARA",message = audio,timeout = 3)

    stream.stop_stream()
    print(f'[bold purple]SARA:[/] [cyan]{audio}[/]')
    sara_voz.say(audio)

    if arduino == True:
        try:
            board.send_sysex(STRING_DATA, util.str_to_two_byte_iter(audio) )
        except:
            pass
    sara_voz.runAndWait()
    stream.start_stream ()

def respostalonga(textofala):
    stream.stop_stream ()
    print(f'[bold purple]SARA:[/] [cyan]{textofala}[/]')
    sara_voz.say(textofala)
    if arduino == True:
        try:
            board.send_sysex( STRING_DATA, util.str_to_two_byte_iter(textofala) )
        except:
            pass
    sara_voz.runAndWait()
    stream.start_stream ()

def horario():
	from datetime import datetime
	hora = datetime.now()
	horas= hora.strftime('%H horas e %M minutos')
	resposta(f'Agora são {horas}')

def datahoje():
    from datetime import date
    dataatual = date.today()
    diassemana = ('Segunda-feira','Terça-feira','Quarta-feira','Quinta-feira','Sexta-feira','Sábado','Domingo')
    meses = ('Zero','Janeiro','Fevereiro','Março','Abril','Maio','Junho','Julho','Agosto','Setembro','Outubro','Novembro','Dezembro')
    resposta(f"Hoje é {diassemana[dataatual.weekday()]}")
    diatexto = f'{dataatual.day} de '
    mesatual = (meses[dataatual.month])
    datatexto = dataatual.strftime(" de %Y")
    resposta(f'Dia {diatexto} {mesatual} {datatexto}')

def bateria(): # No momento funcionameto em notebook
    try:
        bateria = psutil.sensors_battery()
        carga = bateria.percent
        bp = str(bateria.percent)
        bpint = "{:.0f}".format(float(bp))
        resposta("A bateria está em:" +bpint +'%')
        if carga <= 20:
            resposta('Ela está em nivel crítico')
            resposta('Por favor, coloque o carregador')
        elif carga == 100:
            resposta('Ela está totalmente carregada')
            resposta('Retire o carregador da tomada')
    except:
        resposta('Desculpa')
        resposta('Seu dispositivo atual não está usando bateria')
        resposta('Por isso é impossivel informar a quantidade de carga')

def cpu ():
    usocpuinfo = str(psutil.cpu_percent())
    usodacpu  = "{:.0f}".format(float(usocpuinfo))
    resposta('O uso do processador está em ' +usodacpu +'%')

def temperaturadacpu():
    tempcpu = psutil.sensors_temperatures()
    cputemp = tempcpu['coretemp'][0]
    temperaturacpu = cputemp.current
    cputempint = "{:.0f}".format(float(temperaturacpu))
    if temperaturacpu >= 20 and temperaturacpu < 40:
        resposta('Estamos trabalhado em um nível agradavel')
        resposta('A temperatura está em ' +cputempint +'°')

    elif temperaturacpu >= 40 and temperaturacpu < 58:
        resposta('Estamos operando em nivel rasoável')
        resposta('A temperatura é de ' +cputempint +'°')

    elif temperaturacpu >= 58 and temperaturacpu < 70:
        resposta('A temperatura da CPU está meio alta')
        resposta('Algum processo do sistema está causando aquecimento')
        resposta('Fique de olho')
        resposta('A temperatura está em ' +cputempint +'°')

    elif temperaturacpu >= 70 and temperaturacpu != 80:
        resposta('Atenção')
        resposta('Temperatura de ' +cputempint +'°')
        resposta('Estamos em nivel crítico')
        resposta('Desligue o sistema imediatamente')

# função de boas vindas, fases do dia
def BoasVindas():
    Horario = int(datetime.datetime.now().hour)
    if Horario >= 0 and Horario < 12:
        resposta('Bom dia')

    elif Horario >= 12 and Horario < 18:
        resposta('Boa tarde')

    elif Horario >= 18 and Horario != 0:
        resposta('Boa noite')

def tempo():
    try:
        #Procure no google maps as cordenadas da sua cidade e coloque no "lat" e no "lon"(Latitude,Longitude)
        api_url = "https://fcc-weather-api.glitch.me/api/current?lat=14º 13 30&lon=42º 46 53"
        data = requests.get(api_url)
        data_json = data.json()
        if data_json['cod'] == 200:
            main = data_json['main']
            wind = data_json['wind']
            weather_desc = data_json['weather'][0]
            temperatura =  str(main['temp'])
            tempint = "{:.0f}".format(float(temperatura))
            vento = str(wind['speed'])
            ventoint = "{:.0f}".format(float(vento))
            dicionario = {
                'Rain' : 'chuvoso',
                'Clouds' : 'nublado',
                'Thunderstorm' : 'com trovoadas',
                'Drizzle' : 'com garoa',
                'Snow' : 'com possibilidade de neve',
                'Mist' : 'com névoa',
                'Smoke' : 'com muita fumaça',
                'Haze' : 'com neblina',
                'Dust' : 'com muita poeira',
                'Fog' : 'com névoa',
                'Sand' : 'com areia',
                'Ash' : 'com cinza vulcanica no ar',
                'Squall' : 'com rajadas de vento',
                'Tornado' : 'com possibilidade de tornado',
                'Clear' : 'com céu limpo'
                }
            tipoclima =  weather_desc['main']
            if data_json['name'] == "Shuzenji":
                resposta('Erro')
                resposta('Não foi possivel verificar o clima')
                resposta('Tente novamente o comando')
            else:
                resposta(f'Verificando clima para a cidade de {data_json["name"]}')
                resposta(f'O clima hoje está {dicionario[tipoclima]}')
                resposta(f'A temperatura é de {tempint}°')
                resposta(f'O vento está em {ventoint} kilometros por hora')
                resposta(f'E a umidade é de {str(main["humidity"])}%')

    except:
        resposta('Não foi possível realizar essa tarefa')
        resposta('Erro na conexão')

def AteMais():
    Horario = int(datetime.datetime.now().hour)
    if Horario >= 0 and Horario < 12:
        resposta('Tenha um ótimo dia')

    elif Horario >= 12 and Horario < 18:
        resposta('Tenha uma ótima tarde')

    elif Horario >= 18 and Horario != 0:
        resposta('Boa noite')

def linha_sara(): # Linha para menu

    #Tabela Sara >>
    table = Table(title=f"»»»»»»»»»»»»»»»»»»»»»»»»»»»»»> {baner} <««««««««««««««««««««««««««««««", title_justify='center', title_style='bold blue')

    table.add_column('Informação', justify='center', style=' purple')
    table.add_column('Versão', justify='center', style='bold red')
    table.add_column('Suporte', justify='center', style='bold green')

    #Adicionar linhas nas colunas >>
    table.add_row('Sara, assistente virtual pessoal', '--Beta v1.0--   Compatível: Windows >> Sim;   linux >> Sim;   ',  'Contado: Telegram >> https://t.me/Lasstll')

    print(table)

    return 0

def localizacao():
	try:
		EndereçoIP = get('https://api.ipify.org').text
		url = 'https://get.geojs.io/v1/ip/geo/'+EndereçoIP+'.json'
		geo_reqeust = get(url)
		geo_data = geo_reqeust.json()
		city = geo_data['city']
		resposta('Sua localização é '+str(city))
	except:
		resposta('Falha ao verificar a localização')

def primeira_vez():
    ler = open('build/cache.txt', 'r')
    leitura = json.loads(ler.read())
    if leitura == 0:
        resposta('Oie, Meu nome é Sara')
        resposta('A parte de agora sou sua nova assistente pessoal')
        resposta('Estou pronta para atender o seus comandos')
        resposta('Vai ser um prazer trabalha com você')
        resposta('Vamos lá me diga um comando :)')
        resposta('Se você não souber basta digitar ou falar')
        resposta('"help comandos" para ver os meus comandos')
        dicionario = 1
        f = open('build/cache.txt', 'w+')
        f.write(json.dumps(dicionario))

    elif leitura == 1:
        None


import os
import logging

import begin
from tqdm import tqdm
import numpy as np
from skimage.transform import resize as sk_resize

@begin.start(auto_convert=True)
@begin.logging
def main(in_file: 'Input npz file containing the dataset'='.', out_folder: 'Output folder for the npz file'='.',
         imagenet_size: 'IF True, resizes to 224x224 (and crops'=False):
    """
    Rescales an existing dataset in npz format
    """
    logging.info("Input file: {}".format(in_file))
    logging.info("Output folder: {}".format(out_folder))
    if imagenet_size:
        logging.info("Using imagenet size (224, 224, 3)")

    with np.load(os.path.abspath(os.path.expanduser(in_file))) as data:
        X_old = data['X']
        Y = data['Y']

    if imagenet_size:
        X = np.zeros((X_old.shape[0], 224, 224, 3))
    else:
        X = np.zeros((X_old.shape[0], 224, 224))

    # Resize images
    for i in tqdm(range(X_old.shape[0]), total=X_old.shape[0]):
        if imagenet_size:
            X[i, :, :, 0] = sk_resize(X_old[i, :, :], (224, 299), anti_aliasing=True, mode='constant')[:, 37:261]
            X[i, :, :, 1] = X[i, :, :, 0]
            X[i, :, :, 2] = X[i, :, :, 0]
        else:
            X[i, :, :] = sk_resize(X_old[i, :, :], (180, 240), anti_aliasing=True, mode='constant')

    # Save files
    np.savez_compressed(os.path.join(out_folder, 'data-resized{}.npz'.format('-imagenet' if imagenet_size else '')),
                        X=X, Y=Y)


import imp
import time
import json
import uvicorn
from threading import Thread
from datetime import datetime
from kafka import KafkaProducer

from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware

from app.config.settings import config
from app.config.docs import documentation_config
from app.config.cors import cors_config
from app.osm.osm_adapter import OSM_Adapter
from app.routers import metrics, service_discovery, db_helper, images, collectors, hosts, perfsonar
from app.data.postgres import postgres_manager

from sqlalchemy import select

from app.data.models.collector_type import CollectorType
from app.data.models.collector import Collector
from app.data.models.image import Image
from app.data.models.host_state import HostState
from app.gnocchi.gnocchi_puller import GnocchiPuller
from app.grafana_authorization.grafana_authorization import GrafanaAuthorizationMiddleware, GrafanaDashboard

app = FastAPI(**documentation_config)

app.add_middleware(
    CORSMiddleware,
    **cors_config
)

# Service Discovery
def service_discover_start():
    print("---------Initiating Service Discovery----------")
    osm_adapter = OSM_Adapter(config.OSM_HOST, config.OSM_USER, config.OSM_PWD, config.OSM_PROJECT)

    while True:
        time.sleep(30)
        new_vdus_ips = osm_adapter.get_vdus_ip()

# Service Discovery
def gnocchi_puller_start():
    print("---------Initiating Gnocchi Puller----------")
    gnocchi_puller = GnocchiPuller(config.OPENSTACK_HOST, config.OPENSTACK_ID, config.OPENSTACK_SECRET)

    producer = KafkaProducer(
        bootstrap_servers=['%s:%s' % (config.KAFKA_HOST, config.KAFKA_LISTENER_PORT)],
        value_serializer=lambda x: json.dumps(x).encode('utf-8')
    )

    while True:
        time.sleep(30)
        hosts = gnocchi_puller.get_gnocchi_targets()

        if hosts:
            for host in hosts:
                metrics_list = gnocchi_puller.get_metrics_list(host)

                metrics = {
                    "instance": host.ip
                }
                for metric_name, metric_id in metrics_list.items():
                    values = gnocchi_puller.get_metric_value(metric_id)
                    value = values[len(values) - 1]

                    # Gnocchi's date
                    date = value[0].split('+')[0]
                    dt = datetime.strptime(date, "%Y-%m-%dT%H:%M:%S")

                    # Actual date
                    actual_timestamp = time.time()

                    metrics[metric_name] = [
                        int(actual_timestamp),
                        value[1],
                        value[2]
                    ]

                print(metrics)
                # escrever no kafka
                producer.send('gnocchi', value=metrics)

# Initialize Database
def db_init():
    host_states = [
        HostState('ACTIVE'),
        HostState('INACTIVE'),
    ]
    postgres_manager.save_all(host_states)

    collectors_type = [
        CollectorType('PULL'),
        CollectorType('PUSH'),
    ]
    postgres_manager.save_all(collectors_type)

    pull = postgres_manager.get_by_name(CollectorType, 'PULL')
    push = postgres_manager.get_by_name(CollectorType, 'PUSH')

    collectors = [
        Collector('Prometheus', 'metrics', 9100, pull.id),
        Collector('Gnocchi', '', 0, pull.id),
        Collector('Telegraf', '', 0, push.id),
    ]
    postgres_manager.save_all(collectors)

    os_images = [
        Image('ubuntu_2004_5gasp_monitoring_prometheus'),
        Image('ubuntu_2004_5gasp_monitoring_telegraf'),
        Image('ubuntu-20.04-server-cloudimg-amd64')
    ]
    postgres_manager.save_all(os_images)

    image = postgres_manager.get_by_name(Image, 'ubuntu_2004_5gasp_monitoring_prometheus')
    prometheus = postgres_manager.get_by_name(Collector, 'Prometheus')

    image.collectors.append(prometheus)
    postgres_manager.commit()

    image = postgres_manager.get_by_name(Image, 'ubuntu_2004_5gasp_monitoring_telegraf')
    telegraf = postgres_manager.get_by_name(Collector, 'Telegraf')

    image.collectors.append(telegraf)
    postgres_manager.commit()

    image = postgres_manager.get_by_name(Image, 'ubuntu-20.04-server-cloudimg-amd64')
    gnocchi = postgres_manager.get_by_name(Collector, 'Gnocchi')

    image.collectors.append(gnocchi)
    postgres_manager.commit()

# Init Grafana Dashboard
GrafanaAuthorizationMiddleware.get_and_create_dashboard_for_user_if_not_exists(config.GRAFANA_USER, GrafanaDashboard.ESIGHT_INTERFACES)
GrafanaAuthorizationMiddleware.get_and_create_dashboard_for_user_if_not_exists(config.GRAFANA_USER, GrafanaDashboard.ESIGHT_SLOTS)

app.include_router(metrics.router)
app.include_router(service_discovery.router)
app.include_router(db_helper.router)
app.include_router(images.router)
app.include_router(collectors.router)
app.include_router(hosts.router)
app.include_router(perfsonar.router)

if not postgres_manager.get_by_name(CollectorType, 'PULL'):
    print("---------Initiating Empty DataBase----------")
    db_init()

# Threads
thread_service_discovery = Thread(target=service_discover_start)
thread_service_discovery.start()

time.sleep(5)

thread_gnocchi_puller = Thread(target=gnocchi_puller_start)
thread_gnocchi_puller.start()


# update_data.py
# Author : Irfan TOOR <email@irfantoor.com>
#

# imports
import os
import numpy as np
import pandas as pd

# -----------------------------------------------------------------------------------------
# data paths -- adjust these paths and PCA parameters

globo_source = "../data/source/news-portal-user-interactions-by-globocom/"
data_source = "../data/source/"
data_cleaned = "../data/cleaned/"
data_target = "../irfantoor-recommend/recommend/data/"

# if you want to apply PCA for dimentional reduction
apply_pca = True
pca_components = 100 # originally in the embedding : 250

# -----------------------------------------------------------------------------------------
# item_metadata_nunique
print('preparing : item_metadata', end=' ')
item_metadata = pd.read_csv(os.path.join(globo_source, 'articles_metadata.csv'))
item_metadata.rename(columns={"article_id": "item", "category_id": "group"}, inplace=True)
print ('ok')

print('preparing : item_metadata_unique', end=' ')
item_metadat_nunique = item_metadata.nunique()
item_metadat_nunique.to_csv(os.path.join(data_cleaned, 'item_metadata_nunique.csv'))
print ('ok')

# -----------------------------------------------------------------------------------------
# users_clicks : ∑ clicks_hour_{i}.csv, where i=000 to 384

# initialize
print('preparing : user_clicks', end=' ')
data_path = os.path.join(globo_source, 'clicks')
file_ids = range(385)
clicks = []

# read all files
for i in file_ids:
    file = f'clicks_hour_%03d.csv'%(i)
    file_path = os.path.join(data_path, file)
    ds = pd.read_csv(file_path)

    for r in ds.to_numpy():
        clicks.append(r)

# convert to DataFrame
clicks = np.array(clicks)
users_clicks = pd.DataFrame(clicks, columns=ds.columns)
users_clicks = users_clicks[['user_id', 'click_article_id', 'click_timestamp']].rename(columns={"user_id":"user", "click_article_id":"item", "click_timestamp":"timestamp"})
print('ok')

# -----------------------------------------------------------------------------------------
# item_clicks.csv

print('preparing : item_clicks', end=' ')
item_clicks = users_clicks.groupby('item').count().sort_values(by='user', ascending=False)['user']
item_clicks = pd.DataFrame(
    {
        'items': item_clicks.keys(),
        'clicks': item_clicks.values,
    }
)
item_clicks.to_csv(os.path.join(data_cleaned, 'item_clicks.csv'), index=False)
print('ok')

# -----------------------------------------------------------------------------------------
# user_interactions_with_groups.csv

print('preparing : user_interactions_with_groups', end=' ')
user_interactions_with_groups = users_clicks.join(item_metadata.set_index('item'), on='item')[['user', 'item', 'timestamp', 'group']]
user_interactions_with_groups.to_csv(
    os.path.join(data_cleaned, 'user_interactions_with_groups.csv'),
    index=False
)
print('ok')

# -----------------------------------------------------------------------------------------
# user_interactions_nunique.csv

print('preparing : user_interactions_nunique', end=' ')
user_interactions_nunique = user_interactions_with_groups.nunique()
user_interactions_nunique.to_csv(os.path.join(data_cleaned, 'user_interactions_nunique.csv'))
user_interactions_nunique
print('ok')

# -----------------------------------------------------------------------------------------
# group_clicks.csv

print('preparing : group_clicks', end=' ')
group_clicks = user_interactions_with_groups[['group', 'item']].groupby(by='group').count().sort_values(by='item', ascending=False)['item']
group_clicks = pd.DataFrame(
    {
        'group': group_clicks.keys(),
        'clicks': group_clicks.values
    }
)
group_clicks.to_csv(os.path.join(data_cleaned, 'group_clicks.csv'), index=False)
print('ok')

# -----------------------------------------------------------------------------------------
# group_items.csv

print('preparing : group_items', end=' ')
item_metadata[['group', 'item']].to_csv(os.path.join(data_cleaned, 'group_items.csv'), index=False)
print('ok')

# -----------------------------------------------------------------------------------------
# item_features.csv

print('preparing : item_features', end=' ')
item_features = pd.DataFrame(
    pd.read_pickle(os.path.join(globo_source, 'articles_embeddings.pickle'))
)

# from sklearn.linear_model import LogisticRegression
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler

if apply_pca:
    print(' applying PCA ...', end=' ')
    # intialize pca and logistic regression model
    pca = PCA(n_components=pca_components)
    # lr = LogisticRegression(multi_class='auto', solver='liblinear')

    # fit and transform data
    sc = StandardScaler()
    X, y = item_features.iloc[:, 1:].values, item_features.iloc[:, 0].values

    X_std = sc.fit_transform(X)

    X_pca = pca.fit_transform(X_std)
    X_pca.shape

    item_features = pd.DataFrame(X_pca)
    # lr.fit(X_pca, y)

item_features.to_csv(os.path.join(data_cleaned, 'item_features.csv'), index=False)
print('ok')
# -----------------------------------------------------------------------------------------
# item_features_shape.csv

print('preparing : item_features_shape', end=' ')
item_features_shape = pd.DataFrame(
    {
        'shape':item_features.shape
    }
)
item_features_shape.to_csv(os.path.join(data_cleaned, 'item_features_shape.csv'), index=False)
print('ok')

# -----------------------------------------------------------------------------------------
# top_group_clicks.csv

print('preparing : top_group_clicks', end=' ')
top_group_clicks = group_clicks[group_clicks['clicks']>=100000]
top_group_clicks.to_csv(
    os.path.join(data_cleaned, 'top_group_clicks.csv'),
    index=False
)
print('ok')

# -----------------------------------------------------------------------------------------
# top_item_clicks.csv

print('preparing : top_item_clicks', end=' ')
top_item_clicks = item_clicks[item_clicks['clicks']>=10000]
top_item_clicks.to_csv(
    os.path.join(data_cleaned, 'top_item_clicks.csv'),
    index=False
)
print('ok')

# -----------------------------------------------------------------------------------------
# copier les fichier

import shutil

print('')
print('copying files: ')
source_list = [
    [data_source, '100k', '100k.txt'],

    [globo_source, '', 'articles_embeddings.pickle'],
    [globo_source, '', 'articles_metadata.csv'],

    [data_cleaned, '', 'group_clicks.csv'],
    [data_cleaned, '', 'group_items.csv'],
    [data_cleaned, '', 'item_clicks.csv'],
    [data_cleaned, '', 'item_features_shape.csv'],
    [data_cleaned, '', 'item_features.csv'],
    [data_cleaned, '', 'item_metadata_nunique.csv'],
    [data_cleaned, '', 'top_group_clicks.csv'],
    [data_cleaned, '', 'top_item_clicks.csv'],
    [data_cleaned, '', 'user_interactions_nunique.csv'],
    [data_cleaned, '', 'user_interactions_with_groups.csv'],
]

for item in source_list:
    src = os.path.join(item[0], item[1], item[2])
    dst = os.path.join(data_target, item[2])
    if not os.path.exists(dst):
        print(f"creating file: {dst}")
    else:
        print(f"file: {dst}, already exists, overwriting")

# -----------------------------------------------------------------------------------------
# summary.json

import json

print('')
print('preparing : summary', end=' ')
n_items = item_metadat_nunique['item']
n_clicked_items = user_interactions_nunique['item']
n_groups = item_metadat_nunique['group']
n_clicked_groups = user_interactions_nunique['group']

summary = {
    "n_items": int(n_items),
    "n_groups": int(n_groups),
    "n_features": int(item_features.shape[1]),
    "n_users": int(user_interactions_nunique['user']),
    "n_clicked_items": int(n_clicked_items),
    "n_clicked_items_percent": f"%2.2f%%"%(n_clicked_items/n_items*100),
    "n_clicked_groups": int(n_clicked_groups),
    "n_clicked_groups_percent": f"%2.2f%%"%(n_clicked_groups/n_groups*100),
}

with open(os.path.join(data_target, "summary.json"), "w+") as fp:
    json.dump(summary, fp)
print('ok')

print(json.dumps(summary, indent=4))
print("Azure-function deployment trigger ...")


from flask import render_template, redirect, url_for,flash, abort
from . import main
from .forms import SubscriberForm, BlogPostForm, CommentForm
from ..models import User, Subscribers, BlogPost, Comment
from ..email import mail_message
from .. import db
from flask_login import login_required, current_user

@main.route('/',methods=['GET', 'POST'])
def index():

    '''
    View root page function that returns the index page and its data
    '''
    title= "Personal Blog"

    subscribers = SubscriberForm()
    if subscribers.validate_on_submit():
        subscriber = Subscribers(email = subscribers.email.data)
        db.session.add(subscriber)
        db.session.commit()
        flash('You are now subscribed')
        mail_message("Welcome to my blog","email/welcome",subscriber.email,subscriber=subscriber)
        return redirect(url_for('main.index'))

    all_blogs = BlogPost.get_all_blogs()
    if all_blogs:
        blogs= all_blogs
        return render_template('index.html',title=title, all_blogs= blogs, subscribers=subscribers)
    elif not all_blogs:
        blog_status= 'There are no blogs'
        return render_template('index.html', title=title, blog_status=blog_status,subscribers=subscribers)

@main.route('/create_blogpost', methods = ['GET', 'POST'])
@login_required
def create_blogpost():
    title="Create BlogPost"

    blog_form = BlogPostForm()

    if blog_form.validate_on_submit():
        blog = BlogPost(title= blog_form.title.data, blog_post= blog_form.blog_content.data)
        db.session.add(blog)
        db.session.commit()
        return redirect(url_for('main.create_blogpost'))

    comment_form = CommentForm()
    if comment_form.validate_on_submit():
        new_comment = Comment(name= comment_form.name.data, comment_content= comment_form.comment_content.data)
        db.session.add(new_comment)
        db.session.commit()
        return redirect(url_for('main.create_blogpost'))

    return render_template('blog.html', BlogPost = blog_form, title=title, comment_form=comment_form)

@main.route('/blogpost/<id>', methods=['POST','GET'])
def blogpost(id):

    title= f'Posts'
    blog = BlogPost.query.filter_by(id=id).first()
    comment_form = CommentForm()
    if comment_form.validate_on_submit():
        new_comment = Comment(name= comment_form.name.data,comment_content = comment_form.comment_content.data, blog_id=id)
        db.session.add(new_comment)
        db.session.commit()
        return redirect(url_for('main.blogpost',id=blog.id))

    all_comments = Comment.query.all()
    blog_comments = Comment.query.filter_by(blog_id=id).all()

    return render_template('blogpost.html', title = title, blog=blog, comment_form=comment_form,all_comments=all_comments,blog_comments=blog_comments)

@main.route('/blog/<int:id>/<int:id_comment>/delete_comment')
@login_required
def delete_comment(id,id_comment):
    comment = Comment.get_single_comment(id,id_comment)

    db.session.delete(comment)
    db.session.commit()

    flash('Comment has been deleted')

    return redirect(url_for('main.blogpost',id=id))

@main.route('/index/<int:id>/delete_blog')
@login_required
def delete_blog(id):
    blog = BlogPost.get_blog(id)

    db.session.delete(blog)
    db.session.commit()

    flash('Blog has been deleted')

    return redirect(url_for('main.index', id=id))


import torch
INFERENCE_SIZE = (244, 244)
import sys

def get_model_graph(model):

    OLD_RECURSION_LIMIT = sys.getrecursionlimit()
    sys.setrecursionlimit(1500)
    dummy_input = torch.randn(1, 3, INFERENCE_SIZE[0], INFERENCE_SIZE[1])
    dummy_output = model(dummy_input)
    params = model.state_dict()
    param_map = {id(v): k for k, v in params.items()}

    seen = set()

    nodes = {}

    edges = {}

    def size_to_str(size):
        return '('+(', ').join(['%d'% v for v in size])+')'

    def add_edge(f, t):
        fromid = str(id(f))
        if fromid in edges:
            edges[fromid].append(str(id(t)))
        else:
            edges[fromid] = [str(id(t))]

    def add_nodes(var, d=0):
        if var not in seen:
            if torch.is_tensor(var):
                nodes[str(id(var))] = size_to_str(var.size())
            elif hasattr(var, 'variable'):
                subvar = var.variable
                node_name = '%s\n %s' % (param_map.get(id(subvar)), size_to_str(subvar.size()))
                nodes[str(id(var))] = node_name
            else:
                nodes[str(id(var))] = str(type(var).__name__)
            seen.add(var)
            if hasattr(var, 'next_functions'):
                for u in var.next_functions:
                    if u[0] is not None:
                        add_edge(u[0], var)
                        if u[0] not in seen:
                            add_nodes(u[0], d+1)
            if hasattr(var, 'saved_tensors'):
                for t in var.saved_tensors:
                    add_edge(t, var)
                    if t not in seen:
                        add_nodes(t, d+1)

    try:
        add_nodes(dummy_output.grad_fn)
    except BaseException:
        # some models have a weird structure on the output
        add_nodes(dict(dummy_output)['out'].grad_fn)

    # print("Model graph complete", len(edges), len(nodes), flush=True)
    sys.setrecursionlimit(OLD_RECURSION_LIMIT)
    return nodes, edges

def __topo_sort(edge_lookup, nodeid, visited, stack):
    visited[nodeid] = True
    node_edges = edge_lookup[nodeid] if nodeid in edge_lookup else []
    for node in node_edges:
        if node not in visited:
            __topo_sort(edge_lookup, node, visited, stack)
    stack.insert(0, nodeid)

def topological_sort(e):
    # print("Sort edge map", flush=True)
    visited = {}
    stack = []
    for idx, nodeid in enumerate(e.keys()):
        if nodeid not in visited:
            __topo_sort(e, nodeid, visited, stack)
    # print("Edge map sorted", flush=True)
    return stack

def get_critical_path(model):
    nodes, edges = get_model_graph(model)
    from_nodes_sorted = topological_sort(edges)
    latencies = {}

    for node in from_nodes_sorted:
        e = edges[node] if node in edges  else []
        if node not in latencies:
            latencies[node] = 0
        for edge in e:
            edge_latency = latencies[edge] if edge in latencies else 0
            latencies[edge] = max(edge_latency, latencies[node] + 1)
    return max(latencies.values()), latencies, [latencies[k] for k in from_nodes_sorted]


import torch

class Single_net(torch.nn.Module):
    def __init__(self, n_feature, n_hidden, n_output):
        super(Single_net, self).__init__()
        self.hidden = torch.nn.Linear(n_feature, n_hidden)   # hidden layer
        self.predict = torch.nn.Linear(n_hidden, n_output)   # output layer

    def forward(self, x):
        tanh = torch.nn.ReLU()
        x = tanh(self.hidden(x))      # activation function for hidden layer
        x = self.predict(x)             # linear output
        return x

class Network(torch.nn.Module):
    def __init__(self,layers):
        super(Network, self).__init__()
        self.nLayers= len(layers)
        self.layers = layers
        self.linear = torch.nn.ModuleList()
        for i in range(len(layers)-1):
            self.linear.append(torch.nn.Linear(layers[i],layers[i+1]))

    def forward(self,out):
        tanh = torch.nn.ReLU()
        for i in range(self.nLayers-2):
            out = self.linear[i](out)
            out = tanh(out)
        out = self.linear[-1](out)
        return out

def train(net,X,y,no_epochs,loss_func,optimizer):
    losses=[]
    for t in range(no_epochs):
        optimizer.zero_grad()   # clear gradients for next train
        prediction = net(X)     # input X and predict based on X
        loss = loss_func(prediction, y)
        loss.backward()         # backpropagation, compute gradients
        optimizer.step()        # apply gradients
        if t % 5 == 0:
            losses.append(loss.data.numpy())
    return(losses)

def test_network(net=None): # this code simply checks with only test data
    ## creating artificial data
    import sklearn.datasets as datasets
    import matplotlib.pyplot as plt
    X_,y_=datasets.make_regression(n_samples=1000,n_features=2,noise=0.1)
    X = torch.from_numpy(X_).float()
    y = torch.from_numpy(y_).float()
    y=y.view(1000,1)

    #net = Net(n_feature=2, n_hidden=10, n_output=1)     # define the network
    if net==None:
        net=Network([2,10,1])
    print(net)  # net architecture

    optimizer = torch.optim.Adam(net.parameters(), lr=0.01)
    loss_func = torch.nn.MSELoss()  # this is for regression mean squared loss

    loss_func2 = torch.nn.functional.smooth_l1_loss
    optimizer2 = torch.optim.RMSprop(net.parameters())

    losses=train(net,X,y,200,loss_func2,optimizer2)
    prediction=net(X)
    plt.plot(losses)
    plt.show()
    plt.figure()
    plt.scatter(range(len(y_)),y_,s=2,label="actual")
    plt.scatter(range(len(y_)),prediction.data.numpy(),s=2,label="predicted")
    plt.legend()
    plt.show()


#!/Library/Frameworks/Python.framework/Versions/3.6/bin/python3
import praw
import re
from imgurpython import ImgurClient
import requests
import time
import os
import urllib.request

import config

#TODO fix for reddit images redd.it/###.jpg

# max number of submissions to check
LIMIT = 200

# subreddit being checked
#TODO allow for multireddits
SUBREDDIT = config.subreddit

img = (".png", ".jpg", ".jpeg", ".gif", ".webm", ".bmp")

file_name_queue = []
download_queue = []
reddit_id_cache = []

# the directory this file is in
#CURRENT_PATH = (os.path.dirname(os.path.realpath(__file__)))
CURRENT_PATH = os.getcwd()

# create the reddit instance in read-only mode
reddit = praw.Reddit(
	client_id = config.reddit_id,
	client_secret = config.reddit_secret,
	user_agent = config.user_agent)

# create the imgur instance or something
imgur = ImgurClient(
	client_id = config.imgur_id,
	client_secret = config.imgur_secret)

# get the extension of the file
def get_file_name(url, file_name):
	# get the extension from the url and append it to the file name
	f = file_name + '.' + url.split('.')[-1]
	f = CURRENT_PATH + "/pictures/" + "/" + config.subreddit + "/" + f
	return f # string

# add the url to download_queue and file name to file_name_queue
def add_to_queue(url, file_name):
	download_queue.append(url)
	file_name_queue.append(file_name)
	print(url + " added to the download queue " + "(" + str(len(download_queue )) + " items)")

# download files from the queue then clear the queue
def download_files():
	q = 0
	print("downloading " + str(len(download_queue )) + " items")
	while q < len(download_queue):
		# download the file
		print(str(len(download_queue) - q) + " downloads left in queue")
		urllib.request.urlretrieve(download_queue[q], file_name_queue[q])
		print("downloading " + download_queue[q] + " as " + file_name_queue[q])
		q += 1
		time.sleep(1)
	print(str(len(download_queue)) + " images have been downloaded")
	# clear download_queue and file_name_queue
	download_queue[:] = []
	file_name_queue[:] = []

def check_if_album(url, file_name):
	# check if the url is an imgur album or gallery
	if "imgur.com/a/" in url or "imgur.com/gallery/" in url:
		# checks if the url is actually an album? idk i copied it from the internet
		match = re.match("(https?)\:\/\/(www\.)?(?:m\.)?imgur\.com/(a|gallery)/([a-zA-Z0-9]+)(#[0-9]+)?", url)
		imgur_album_id = match.group(4) # id of the imgur album
		print(url + " is an album containing " + str(len(imgur.get_album_images(imgur_album_id))) + " images")
		i = 0 # counter for file names
		n = 0 # counter for new files
		if len(imgur.get_album_images(imgur_album_id)) == 1:
			for image in imgur.get_album_images(imgur_album_id):
				if not os.path.isfile(get_file_name(image.link, file_name)):
					add_to_queue(image.link, get_file_name(image.link, file_name))
		else:
			album_dir = CURRENT_PATH + "/pictures/" + config.subreddit + "/" + file_name
			if not os.path.isdir(album_dir):
				os.mkdir(album_dir)
				print("creating /" + file_name + "/ in " + album_dir)
			for image in imgur.get_album_images(imgur_album_id):
				i += 1
				# append a number if the image is part of an album
				f = "/" + file_name + "/" + file_name + "_" + str(i)
				f = get_file_name(image.link, f)
				if not os.path.isfile(f):
					n += 1
					add_to_queue(image.link, f)
			print("album contained " + str(n) + " new images")
		return True
	else:
		return False

# checks if the url is an image or is an imgur album
def check_submission(url, file_name):
	if url.endswith(img):
		f = get_file_name(url, file_name)
		if not os.path.isfile(f):
			add_to_queue(url, f)
	else:
		check_if_album(url, file_name)

# get reddit submissions
def get_submissions():
	c = 0
	# check SUBREDDIT for LIMIT number of submissions
	print("checking " + SUBREDDIT + " for " + str(LIMIT) + " submissions")
	# can change to either hot/new/top etc.
	for submission in reddit.subreddit(SUBREDDIT).hot(limit = LIMIT):
		url = submission.url
		id = submission.id
		if id not in reddit_id_cache: #TODO change this to url so the same url on different subs doesnt get dled
			reddit_id_cache.append(id)
			print(str(c) + ": " + id + " added to reddit_id_cache")
			check_submission(url, id)
			c += 1
		else:
			print(id + " has already been checked")
		print("sleeping for 1 second...")
		time.sleep(1)

	print(str(c) + " new submissions found")

if not os.path.isdir(CURRENT_PATH + "/pictures/"):
	os.mkdir(CURRENT_PATH + "/pictures/")
	print("creating /pictures/ in " + CURRENT_PATH)

if not os.path.isdir(CURRENT_PATH + "/pictures/" + config.subreddit + "/"):
	os.mkdir(CURRENT_PATH + "/pictures/" + config.subreddit + "/")
	print("creating /"  + config.subreddit + "/ in " + CURRENT_PATH)

get_submissions()
download_files()


from celery import shared_task
from django.conf import settings
from django.db import transaction
from .models import Project, Reference, VAF
from .serializers import MetadataSerializer
import maptide
import time
import math

def entropy(probabilities, normalised=False):
    ent = sum([-(x * math.log2(x)) if x != 0 else 0 for x in probabilities])

    if normalised:
        return ent / math.log2(len(probabilities))
    else:
        return ent

def get_stats(counts, decimals=3):
    coverage = sum(counts)
    probabilities = [count / coverage if coverage > 0 else 0.0 for count in counts]
    percentages = [100 * probability for probability in probabilities]
    ent = entropy(probabilities, normalised=True)
    secondary_count = list(counts)
    secondary_count.pop(counts.index(max(counts)))
    secondary_coverage = sum(secondary_count)
    secondary_probabilities = [
        count / secondary_coverage if secondary_coverage > 0 else 0.0
        for count in secondary_count
    ]
    secondary_ent = entropy(secondary_probabilities, normalised=True)
    return (
        [coverage]
        + counts
        + [round(x, decimals) for x in percentages + [ent, secondary_ent]]
    )

def iterate(data, region=None, stats=False, decimals=3):
    if region:
        chrom, start, end = maptide.parse_region(region)
        for (pos, ins_pos), row in sorted(data[chrom].items()):
            if (not start or pos >= start) and (not end or pos <= end):
                if stats:
                    yield [chrom, pos, ins_pos] + get_stats(row, decimals=decimals)
                else:
                    yield [chrom, pos, ins_pos, sum(row)] + row
    else:
        for chrom, chrom_data in data.items():
            for (pos, ins_pos), row in sorted(chrom_data.items()):
                if stats:
                    yield [chrom, pos, ins_pos] + get_stats(row, decimals=decimals)
                else:
                    yield [chrom, pos, ins_pos, sum(row)] + row

@shared_task
def generate(code, metadata):
    start = time.time()

    # Get the project
    project = Project.objects.get(code=code)

    # Run maptide
    mp = maptide.query(
        metadata["bam_path"],
        region=project.region,
        bai=metadata.get("bai_path"),
        mapping_quality=project.mapping_quality,
        base_quality=project.base_quality,
    )

    # Maptide output columns
    columns = [
        "reference",
        "position",
        "insertion",
        "coverage",
        "a",
        "c",
        "g",
        "t",
        "ds",
        "n",
        "pc_a",
        "pc_c",
        "pc_g",
        "pc_t",
        "pc_ds",
        "pc_n",
        "entropy",
        "secondary_entropy",
    ]

    references = {}
    vafs = []
    num_vafs = 0

    # Create VAF instances
    for record in iterate(mp, region=project.region, stats=True):
        record = dict(zip(columns, record))

        # If the project is not recording insertions, skip any non-ref VAFS
        if (not project.insertions) and record["insertion"] != 0:
            continue

        # Check the VAF's coverage and entropies fit the project requirements
        if (
            record["coverage"] < project.min_coverage
            or record["entropy"] < project.min_entropy
            or record["secondary_entropy"] < project.min_secondary_entropy
        ):
            continue

        # Find the dominant base and its confidence
        base_counts = {
            "A": record["a"],
            "C": record["c"],
            "G": record["g"],
            "T": record["t"],
            "DS": record["ds"],
        }
        if record["coverage"] > 0:
            record["base"] = max(
                base_counts,
                key=base_counts.get,  # type: ignore
            )
            record["confidence"] = (
                100 * base_counts[record["base"]] / sum(base_counts.values())
            )
        else:
            record["base"] = None
            record["confidence"] = 0

        # Get the corresponding reference object for the VAF, if not already retrieved
        if record["reference"] not in references:
            references[record["reference"]] = Reference.objects.get(
                project=project, name=record["reference"]
            )

        # Check for difference between ref_base and base
        ref_base = references[record["reference"]].sequence[record["position"] - 1]
        diff = bool(record["base"] and record["base"] != ref_base)

        # If the project requires VAFS with a difference from the reference, above a certain confidence,
        # check that there is a difference, above the diff confidence threshold.
        # If not, skip
        if project.diff_confidence and not (
            diff and record["confidence"] >= project.diff_confidence
        ):
            continue

        vafs.append(
            {
                "reference": record["reference"],
                "position": record["position"],
                "insertion": record["insertion"],
                "ptype": "REF" if record["insertion"] == 0 else "INS",
                "coverage": record["coverage"],
                "ref_base": ref_base,
                "base": record["base"],
                "confidence": round(
                    record["confidence"], settings.FLOATFIELD_DECIMAL_PLACES
                ),
                "diff": diff,
                "a": record["a"],
                "c": record["c"],
                "g": record["g"],
                "t": record["t"],
                "ds": record["ds"],
                "pc_a": round(record["pc_a"], settings.FLOATFIELD_DECIMAL_PLACES),
                "pc_c": round(record["pc_c"], settings.FLOATFIELD_DECIMAL_PLACES),
                "pc_g": round(record["pc_g"], settings.FLOATFIELD_DECIMAL_PLACES),
                "pc_t": round(record["pc_t"], settings.FLOATFIELD_DECIMAL_PLACES),
                "pc_ds": round(record["pc_ds"], settings.FLOATFIELD_DECIMAL_PLACES),
                "entropy": round(record["entropy"], settings.FLOATFIELD_DECIMAL_PLACES),
                "secondary_entropy": round(
                    record["secondary_entropy"], settings.FLOATFIELD_DECIMAL_PLACES
                ),
            }
        )
        num_vafs += 1

    end = time.time()

    print(
        f"[GENERATE] [SAMPLE_ID] {metadata['sample_id']} [SUCCESS] time: {round(end - start, settings.FLOATFIELD_DECIMAL_PLACES)} s"
    )
    return metadata, vafs, num_vafs, None, None, None, None

@shared_task
def store(args):
    metadata, vafs, num_vafs, num_reads, mean_coverage, mean_entropy, references = args
    start = time.time()

    # Validate the metadata (again because we cant pass the object to the task) before doing anything else
    serializer = MetadataSerializer(data=metadata)  # type: ignore
    serializer.is_valid(raise_exception=True)

    # All changes to the db for this task are wrapped in a single transaction
    # So if anything goes wrong, any changes made are rolled back
    with transaction.atomic():
        # Create Metadata instance
        instance = serializer.save()

        # Create VAF instances
        for vaf in vafs:
            print(vaf)
            VAF.objects.create(metadata=instance, **vaf)

        # Update Metadata instance with VAF summary statistics
        # instance.num_reads = sum(bc.read_counts.values())
        # instance.mean_coverage = bc.mean_coverage()
        instance.num_vafs = num_vafs
        # instance.mean_entropy = bc.mean_entropy()
        # instance.references = ",".join(bc.references)
        instance.save(
            update_fields=[
                "num_reads",
                "mean_coverage",
                "num_vafs",
                "mean_entropy",
                "references",
            ]
        )
    end = time.time()

    print(
        f"[STORE] [SAMPLE_ID] {metadata['sample_id']} [SUCCESS] time: {round(end - start, settings.FLOATFIELD_DECIMAL_PLACES)} s"
    )


import os
import torch
import torch.nn as nn
from torch.nn import functional as F

# Initialize some hyperparameters
# Hyperparameters control various aspects of training, such as the batch size,
# learning rate, dropout ratio, etc.
batch_size = 132 # Number of sequences to process in parallel.
block_size = 16 # Maximum context length for predictions.
max_iters = 500000 # Maximum number of iterations to train.
eval_interval = 100 # Frequency at which to evaluate the model.
learning_rate = 1e-3 # Learning rate for the optimizer.
device = 'cuda' if torch.cuda.is_available() else 'cpu' # Device to use for computations, use GPU if available else CPU.
eval_iters = 100 # Number of evaluation iterations.
n_embd = 64 # Size of the embeddings.
n_head = 4 # Number of attention heads.
n_layer = 4 # Number of transformer layers.
dropout = 0.1 # Dropout ratio.

torch.manual_seed(1337) # Set the seed for reproducibility.

# Load the training text file.
with open('input.txt', 'r', encoding='utf-8') as f:
    text = f.read()

# Extract the unique characters in the text.
chars = sorted(list(set(text)))
vocab_size = len(chars) # Size of the vocabulary.

# Map characters to indices and vice versa for easy conversion.
stoi = { ch:i for i,ch in enumerate(chars) }
itos = { i:ch for i,ch in enumerate(chars) }
encode = lambda s: [stoi[c] for c in s] # Function to convert characters to indices.
decode = lambda l: ''.join([itos[i] for i in l]) # Function to convert indices to characters.

# Convert the whole dataset to indices and split it into training and validation sets.
data = torch.tensor(encode(text), dtype=torch.long)
n = int(0.9*len(data)) # first 90% will be train, rest val
train_data = data[:n]
val_data = data[n:]

# A function to get a batch of data with inputs x and targets y.
def get_batch(split):
    data = train_data if split == 'train' else val_data
    ix = torch.randint(len(data) - block_size, (batch_size,)) # Random indices for getting the batches.
    x = torch.stack([data[i:i+block_size] for i in ix]) # Inputs for the model.
    y = torch.stack([data[i+1:i+block_size+1] for i in ix]) # Corresponding targets for the model.
    x, y = x.to(device), y.to(device) # Move tensors to the computation device.
    return x, y

# This function estimates the loss on both training and validation data without updating the model parameters.
def estimate_loss():
    out = {}
    model.eval() # Set the model to evaluation mode.
    for split in ['train', 'val']:
        losses = torch.zeros(eval_iters)
        for k in range(eval_iters):
            X, Y = get_batch(split)
            logits, loss = model(X, Y)
            losses[k] = loss.item()
        out[split] = losses.mean()
    model.train() # Set the model back to training mode.
    return out

# Transformer architecture components.

# Self-attention head.
class Head(nn.Module):
    def __init__(self, head_size):
        super().__init__()
        # Key, query and value matrices for attention mechanism.
        self.key = nn.Linear(n_embd, head_size, bias=False)
        self.query = nn.Linear(n_embd, head_size, bias=False)
        self.value = nn.Linear(n_embd, head_size, bias=False)
        # Lower triangle matrix for masking future tokens.
        self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))
        # Dropout layer.
        self.dropout = nn.Dropout(dropout)

    # Forward pass of the attention head.
    def forward(self, x):
        B, T, C = x.shape # Batch size, sequence length, embedding dimension.
        k = self.key(x) # Key.
        q = self.query(x) # Query.
        # Compute attention scores.
        wei = q @ k.transpose(-2,-1) * C**-0.5 # Scaling factor for the scores.
        # Mask the future tokens.
        wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf'))
        # Compute attention weights using softmax.
        wei = F.softmax(wei, dim=-1)
        wei = self.dropout(wei)
        # Compute the output by weighted aggregation of values.
        v = self.value(x)
        out = wei @ v
        return out

# Multiple heads of self-attention in parallel.
class MultiHeadAttention(nn.Module):
    def __init__(self, num_heads, head_size):
        super().__init__()
        self.heads = nn.ModuleList([Head(head_size) for _ in range(num_heads)]) # List of attention heads.
        self.proj = nn.Linear(n_embd, n_embd) # Final projection layer.
        self.dropout = nn.Dropout(dropout)

    # Forward pass of the multi-head attention layer.
    def forward(self, x):
        out = torch.cat([h(x) for h in self.heads], dim=-1) # Concatenate the outputs of all attention heads.
        out = self.dropout(self.proj(out)) # Final projection and dropout.
        return out

# Feed-forward network.
class FeedFoward(nn.Module):
    def __init__(self, n_embd):
        super().__init__()
        # A simple linear layer followed by a ReLU non-linearity and another linear layer.
        self.net = nn.Sequential(
            nn.Linear(n_embd, 4 * n_embd),
            nn.ReLU(),
            nn.Linear(4 * n_embd, n_embd),
            nn.Dropout(dropout),
        )

    # Forward pass of the feed-forward network.
    def forward(self, x):
        return self.net(x)

# Transformer block: multi-head attention followed by feed-forward network.
class Block(nn.Module):
    def __init__(self, n_embd, n_head):
        super().__init__()
        head_size = n_embd // n_head # Size of each head.
        self.sa = MultiHeadAttention(n_head, head_size) # Self-attention layer.
        self.ffwd = FeedFoward(n_embd) # Feed-forward network.
        self.ln1 = nn.LayerNorm(n_embd) # Layer normalization.
        self.ln2 = nn.LayerNorm(n_embd) # Layer normalization.

    # Forward pass of the transformer block.
    def forward(self, x):
        x = x + self.sa(self.ln1(x)) # Add & Norm for self-attention.
        x = x + self.ffwd(self.ln2(x)) # Add & Norm for feed-forward network.
        return x

# The main model: a bigram language model.
class BigramLanguageModel(nn.Module):
    def __init__(self):
        super().__init__()
        # Embedding layers for tokens and positions.
        self.token_embedding_table = nn.Embedding(vocab_size, n_embd)
        self.position_embedding_table = nn.Embedding(block_size, n_embd)
        # Multiple transformer blocks.
        self.blocks = nn.Sequential(*[Block(n_embd, n_head=n_head) for _ in range(n_layer)])
        self.ln_f = nn.LayerNorm(n_embd) # Final layer norm.
        self.lm_head = nn.Linear(n_embd, vocab_size) # Linear layer for generating logits for the next token.

    # Forward pass of the language model.
    def forward(self, idx, targets=None):
        B, T = idx.shape # Batch size, sequence length.
        tok_emb = self.token_embedding_table(idx) # Token embeddings.
        pos_emb = self.position_embedding_table(torch.arange(T, device=device)) # Position embeddings.
        x = tok_emb + pos_emb # Add the token and position embeddings.
        x = self.blocks(x) # Pass through the transformer blocks.
        x = self.ln_f(x) # Final layer norm.
        logits = self.lm_head(x) # Compute the logits for the next token.

        # If targets are provided, compute the loss.
        if targets is not None:
            B, T, C = logits.shape
            logits = logits.view(B*T, C)
            targets = targets.view(B*T)
            loss = F.cross_entropy(logits, targets) # Compute cross-entropy loss.
        else:
            loss = None
        return logits, loss

    # Function for generating new text.
    def generate(self, idx, max_new_tokens):
        for _ in range(max_new_tokens):
            idx_cond = idx[:, -block_size:] # Current context.
            logits, loss = self(idx_cond) # Get predictions.
            logits = logits[:, -1, :] # Focus on the last time step.
            probs = F.softmax(logits, dim=-1) # Convert logits to probabilities.
            idx_next = torch.multinomial(probs, num_samples=1) # Sample the next token.
            idx = torch.cat((idx, idx_next), dim=1) # Update the current context.
        return idx

# Create the model and move it to the device.
model = BigramLanguageModel()
m = model.to(device)

# Print the number of parameters in the model.
print(sum(p.numel() for p in m.parameters())/1e6, 'M parameters')

# Create an optimizer.
optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)

# Main training loop.
for iter in range(max_iters):
    # Evaluate the loss on train and val sets periodically.
    if iter % eval_interval == 0 or iter == max_iters - 1:
        losses = estimate_loss()
        print(f"step {iter}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}")

    # Sample a batch of data.
    xb, yb = get_batch('train')

    # Evaluate the loss.
    logits, loss = model(xb, yb)
    # Zero gradients, perform a backward pass, and update the weights.
    optimizer.zero_grad(set_to_none=True)
    loss.backward()
    optimizer.step()

# Generate new text from the model.
torch.save(model.state_dict(), './model/model.pt')
context = torch.zeros((1, 1), dtype=torch.long, device=device)
print(decode(m.generate(context, max_new_tokens=2000)[0].tolist()))


import logging
import pickle
import sys
from pathlib import Path
from timeit import timeit

import pandas as pd
import yaml
from nyx_space.cosmic import Cosm, Orbit, Spacecraft, SrpConfig
from nyx_space.mission_design import (
    Event,
    SpacecraftDynamics,
    StateParameter,
    TrajectoryLoader,
    propagate,
    two_body,
)
from nyx_space.monte_carlo import StateParameter, generate_orbits
from nyx_space.plots import plot_traj_errors
from nyx_space.time import Duration, Epoch, TimeSeries, Unit

def test_propagate():
    # Initialize logging
    FORMAT = "%(levelname)s %(name)s %(filename)s:%(lineno)d\t%(message)s"
    logging.basicConfig(format=FORMAT)
    logging.getLogger().setLevel(logging.INFO)

    # Base path
    root = Path(__file__).joinpath("../../../").resolve()

    config_path = root.joinpath("./data/tests/config/")

    sc = Spacecraft.load(str(config_path.joinpath("spacecraft.yaml")))
    # Check that we have loaded this correctly by checking the values from the YAML file
    assert sc.value_of(StateParameter.X) == -9042.862234
    assert sc.value_of(StateParameter.Y) == 18536.333069
    assert sc.value_of(StateParameter.Z) == 6999.957069
    assert sc.value_of(StateParameter.VX) == -3.288789
    assert sc.value_of(StateParameter.VY) == -2.226285
    assert sc.value_of(StateParameter.VZ) == 1.646738
    assert sc.value_of(StateParameter.Cr) == 1.0
    assert sc.value_of(StateParameter.Cd) == 2.2
    assert sc.value_of(StateParameter.FuelMass) == 50.0
    assert sc.epoch.timedelta(Epoch("2018-09-15T00:15:53.098 UTC")) == Duration.zero()
    # Check other stuff that is computed on request
    assert sc.value_of(StateParameter.SMA) == 21999.99774705774

    dynamics = SpacecraftDynamics.load_named(str(config_path.joinpath("dynamics.yaml")))
    # So far, there are no accessors on the dynamics, so we will check what they print out =(
    assert (
        f"{dynamics['lofi']}"
        == "Spacecraft dynamics (with guidance = false): No force models; Orbital dynamics: Point masses of Sun J2000, Earth J2000"
    )
    assert (
        f"{dynamics['hifi']}"
        == "Spacecraft dynamics (with guidance = false): SRP with φ = 1367 W/m^2 and eclipse light-source: Sun J2000, shadows casted by: Sun J2000, Moon J2000;  Orbital dynamics: Point masses of Sun J2000, Earth J2000, Moon J2000; IAU Earth gravity field 10x10 (order x degree)"
    )

    rslt, traj = propagate(sc, dynamics["lofi"], Unit.Day * 5.159)
    # Check that we propagated for the correct duration
    assert rslt.epoch.timedelta(sc.epoch) == Duration("5 days 3 h 48 min 57 s 600 ms")
    # We aren't in two body dynamics so the SMA has changed among other things.
    assert rslt.value_of(StateParameter.SMA) == 22000.024381883788
    # Grab a specific epoch from the trajectory
    sc_state = traj.at(Epoch("2018-09-16T00:16:53 TDB"))
    # Check that we got the right epoch
    assert sc_state.epoch.timedelta(Epoch("2018-09-16T00:16:53 TDB")) == Duration.zero()
    # Save the file to parquet
    traj.to_parquet("lofi.parquet")

    # We can also propagate with a different method
    rslt, traj = propagate(sc, dynamics["lofi"], Unit.Day * 5.159, method="Dormand78")
    assert rslt.epoch.timedelta(sc.epoch) == Duration("5 days 3 h 48 min 57 s 600 ms")

    # Let's now propagate the original spacecraft to its apoapsis, but let's search no more than 2 orbit periods
    rslt_apo, _ = propagate(
        sc,
        dynamics["lofi"],
        sc.orbit.period() * 2,
        event=Event(StateParameter.Apoapsis, 0.0),
    )
    # Let's make sure that worked to within the default precision
    assert abs(rslt_apo.value_of(StateParameter.TrueAnomaly) - 180.0) < 0.001

    event = Event(StateParameter.Apoapsis, 0.0, value_precision=1e-6)

    # But we can also be even more precise
    rslt_apo, traj = propagate(
        sc,
        dynamics["lofi"],
        sc.orbit.period() * 2,
        event=event,
    )
    assert abs(rslt_apo.orbit.ta_deg() - 180.0) <= 1e-6

    # Resample the trajectory at fixed step size of 25 seconds
    traj = traj.resample(Unit.Second * 25.0)

    # Export this trajectory with additional metadata and the events
    # Base path
    root = Path(__file__).joinpath("../../../").resolve()
    outpath = root.joinpath("output_data/")

    # Note: Python interface only supports strings for paths, not Path objects.
    traj.to_parquet(
        str(outpath.joinpath("./lofi_with_events.parquet")),
        metadata={"dynamics": str(dynamics["lofi"])},
        events=[event],
    )

    # Propagate until the lo-fi epoch to compare
    _, traj_hifi = propagate(
        sc,
        dynamics["hifi"],
        epoch=rslt_apo.epoch,
    )

    # Plot both trajectories in RIC frame
    if sys.platform != "win32":
        diff_path = str(outpath.joinpath("./lofi_hifi_ric_diff.parquet"))
        traj.ric_diff_to_parquet(traj_hifi, diff_path)

        traj_diff_ric = pd.read_parquet(diff_path)

        plot_traj_errors(
            traj_diff_ric,
            "LoFi vs HiFi",
            html_out=outpath.joinpath("./md_ric_lofi_hifi_diff.html"),
            show=False,
        )

        plot_traj_errors(
            traj_diff_ric,
            "LoFi vs HiFi",
            html_out=outpath.joinpath("./md_ric_lofi_hifi_diff_velocity.html"),
            vertical=True,
            velocity=True,
            show=False,
        )

    # Resample the trajectory at fixed step size of 25 seconds
    traj = traj.resample(Unit.Second * 25.0)

    # Also export this ground track
    cosm = Cosm.de438()
    iau_earth = cosm.frame("IAU Earth")
    traj.to_parquet("iau_earth_lofi.parquet", groundtrack=iau_earth)

    # Let's also search for this event in the trajectory
    for sc_at_event in traj.find(event):
        print(sc_at_event)
        assert abs(sc_at_event.value_of(StateParameter.TrueAnomaly) - 180.0) <= 1e-6

def test_build_spacecraft():
    """
    Tests that we can build a spacecraft from scratch without an input file
    """
    # Initialize logging
    FORMAT = "%(levelname)s %(name)s %(filename)s:%(lineno)d\t%(message)s"
    logging.basicConfig(format=FORMAT)
    logging.getLogger().setLevel(logging.INFO)

    cosm = Cosm.de438()
    eme2k = cosm.frame("EME2000")  # Earth Mean Equator J2000
    orbit = Orbit.from_keplerian_altitude(
        400.0, 0.01, 15.6, 45.0, 90.0, 75.0, Epoch.system_now(), eme2k
    )
    # Define the SRP
    srp = SrpConfig(2.0)  # 2.0 will be the area
    print(srp)
    assert srp.cr == 1.8  # Default value
    assert srp.area_m2 == 2.0
    sc = Spacecraft(orbit, 150.0, 15.0, srp)
    print(sc)

    assert sc.srp() == srp
    assert sc.drag().area_m2 == 0.0
    assert (
        sc.drag().cd == 2.2
    )  # Default value, but the area is zero, so it doesn't have any effect

    # Using this spacecraft as a template, let's load an OEM file, convert it to Parquet, and ensure we can load it back in.
    # The orbit data will be overwritten with data from the OEM file.
    root = Path(__file__).joinpath("../../../").resolve()

    config_path = root.joinpath("./data/tests/ccsds/oem/LEO_10s.oem")
    output_path = root.joinpath("./output_data/LEO_10s.parquet")
    # Convert from OEM to Parquet will happen on load
    traj = TrajectoryLoader(str(config_path), "oem", str(output_path), sc)
    print(traj)
    # Check that we can pickle the trajectory loader object
    traj_pkl = pickle.dumps(traj)
    traj_unpkl = pickle.loads(traj_pkl)
    assert traj_unpkl == traj
    # Check that we can convert this to a spacecraft trajectory
    traj_sc = traj.to_spacecraft_traj()
    traj_orbit = traj.to_orbit_traj()
    traj_orbit_dc = traj_sc.downcast()
    # Check that we can query it (will raise an exception if we can't, thereby failing the test)
    ts = TimeSeries(
        Epoch("2020-06-01T12:00:00.000000"),
        Epoch("2020-06-01T13:00:00.000000"),
        step=Unit.Minute * 17 + Unit.Second * 13.8,
        inclusive=True,
    )
    for epoch in ts:
        orbit = traj_orbit.at(epoch)
        dc_orbit = traj_orbit_dc.at(epoch)
        sc_orbit = traj_sc.at(epoch).orbit
        # Check params individually
        assert orbit.epoch == sc_orbit.epoch
        assert orbit.x_km == sc_orbit.x_km
        assert orbit.vx_km_s == sc_orbit.vx_km_s
        # Check the downcasted version
        assert dc_orbit.x_km == sc_orbit.x_km
        assert dc_orbit.vx_km_s == sc_orbit.vx_km_s
        assert dc_orbit == sc_orbit

def test_two_body():
    # Build a demo orbit
    cosm = Cosm.de438()
    eme2k = cosm.frame("EME2000")

    assert eme2k.gm() == 398600.435392  # SPICE data, not GMAT data (398600.4415)
    assert eme2k.is_geoid()
    assert eme2k.equatorial_radius() == 6378.1363

    e = Epoch.system_now()

    orbit = Orbit.from_keplerian_altitude(
        400,
        ecc=1e-4,
        inc_deg=30.5,
        raan_deg=35.0,
        aop_deg=65.0,
        ta_deg=590,
        epoch=e,
        frame=eme2k,
    )

    orbits = generate_orbits(
        orbit,
        [
            (StateParameter("SMA"), 0.05),
            (StateParameter.Eccentricity, 0.1),
            (StateParameter.Inclination, 0.1),
        ],
        1000,
        kind="prct",
    )

    # And propagate in parallel using a single duration
    proped_orbits = two_body(orbits, durations=[Unit.Day * 531.5])
    assert len(proped_orbits) == len(orbits)

    # And propagate in parallel using many epochs
    ts = TimeSeries(e, e + Unit.Day * 1000, step=Unit.Day * 1, inclusive=False)
    epochs = [e for e in ts]
    proped_orbits = two_body(orbits, new_epochs=epochs)
    # Allow up to two to fail
    assert len(proped_orbits) >= len(orbits) - 2

    timing = timeit(lambda: two_body(orbits, new_epochs=epochs), number=1)
    print(f"two body propagation of {len(orbits)} orbits in {timing} s")

def test_merge_traj():
    # Initialize logging
    FORMAT = "%(levelname)s %(name)s %(filename)s:%(lineno)d\t%(message)s"
    logging.basicConfig(format=FORMAT)
    logging.getLogger().setLevel(logging.INFO)

    # Base path
    root = Path(__file__).joinpath("../../../").resolve()

    config_path = root.joinpath("./data/tests/config/")

    sc1 = Spacecraft.load(str(config_path.joinpath("spacecraft.yaml")))

    dynamics = SpacecraftDynamics.load_named(
        str(config_path.joinpath("dynamics.yaml"))
    )["lofi"]

    # Check loading from the YAML read from Python
    with open(config_path.joinpath("dynamics.yaml")) as fh:
        data = yaml.safe_load(fh)

    loaded = SpacecraftDynamics.loads(data["lofi"])
    assert loaded == dynamics

    sc2, traj1 = propagate(sc1, dynamics, Unit.Day * 5)
    # And propagate again
    sc3, traj2 = propagate(sc2, dynamics, Unit.Day * 5)
    # Add the trajectories
    traj = traj1 + traj2

    assert traj.last().epoch == sc3.epoch, f"{traj.last()} != {sc3}"

    # Convert into another frame and try to add them too.
    # We only check the epoch this time.
    traj1_moon = traj1.to_frame("Moon J2000")
    traj2_moon = traj2.to_frame("Moon J2000")

    traj_moon = traj1_moon + traj2_moon

    assert traj_moon.last().epoch == sc3.epoch
    print(traj_moon)

if __name__ == "__main__":
    # test_propagate()
    test_merge_traj()


import pandas as pd
import json
import datetime
from bdd import get_database
from fetch_air_quality_datas import fetch_air_quality_datas

# Fonction permettant de convertir les données au format csv en format json qui sera plus adapté pour structurer notre
# MongoDB.
def convert_air_quality_datas_to_json() :
    #Récupération des données
    fetch_air_quality_datas('2021-01-01')

    df = pd.read_csv("df.csv",delimiter=';')
    df = df.replace('Bld Périphérique Est','Bld Peripherique Est')
    df = df.replace("Place de l'Opéra","Place de l Opera")
    #Convertion des timestamps unix en date lisible
    df["attributes.date_debut"] = df["attributes.date_debut"]/1000
    df["date"] = df["attributes.date_debut"].apply(lambda x : datetime.datetime.fromtimestamp(int(x)))
    df["year"] = df["attributes.date_debut"].apply(lambda x : datetime.datetime.fromtimestamp(int(x)).year)
    df["month"] = df["attributes.date_debut"].apply(lambda x : datetime.datetime.fromtimestamp(int(x)).month)
    df["day"] = df["attributes.date_debut"].apply(lambda x : datetime.datetime.fromtimestamp(int(x)).day)

    def pd_to_station_dict(df):
        for (station), df_station_grouped in df.groupby(["attributes.nom_station"]):
            print(station)
            yield {station: {
                "Arr": df.loc[df['attributes.nom_station'] == station, 'attributes.nom_com'].iloc[0],
                "Id_com" : int(df.loc[df['attributes.nom_station'] == station, 'attributes.id_com'].iloc[0]),
                "Longitude" : df.loc[df['attributes.nom_station'] == station, 'geometry.x'].iloc[0],
                "Latitude" : df.loc[df['attributes.nom_station'] == station, 'geometry.y'].iloc[0],
                "Mesures" : list(split_mesures(df_station_grouped))
                }
            }

    def split_mesures(df_station_grouped):
        for row in df_station_grouped.itertuples():
            yield {
                "Annee" : row[22],
                "Mois" : row[23],
                "Jour" : row[24],
                "Heure" : row[21].hour,
                "Date" : str(row[21]),
                "Date_exacte" : row[14],
                "Nom_Poll": row[9],
                "Id_Poll": int(row[10]),
                "Valeur": int(row[11]),
                "Unite": row[12],
                "Typologie": row[8]
            }

    station_list = list(pd_to_station_dict(df))
    station_dict = {}
    for sub_dict in station_list:
        station_dict.update(sub_dict)

    fichier = open("df_json.json", "w")
    json.dump(station_dict,fichier ,indent=4)
    fichier.close()

# Fonction permettant d'inserer les données au format json dans la MongoDB
def insert_datas_db() :
    dbname = get_database()
    convert_air_quality_datas_to_json()
    with open('df_json.json') as f:
        json_datas = json.load(f)
    collection_name = dbname["air_quality"]
    collection_name.insert_one(json_datas)

# convert_air_quality_datas_to_json()


import json
import jsonschema
from datetime import datetime
from models.models import Order, Customer, Driver
from django.db import connection
from django.core import serializers
from django.http import JsonResponse, HttpRequest
from django.views.decorators.http import require_http_methods

@require_http_methods(["GET", "POST"])
def handle_order(request: HttpRequest, customer_id: int):

    response_body = {}
    status_code = 200

    if request.method == 'GET':
        # Check if customer exist
        if not Customer.objects.filter(pk=customer_id).exists():
            status_code = 404
            response_body["status"] = "Error"
            response_body["description"] = "Customer does not exist"
            response_body["error"] = ""
        else:
            query = 'SELECT * FROM public.order WHERE customer_id = %s;'
            model = Order.objects.raw(raw_query=query, params=[customer_id])
            data = serializers.serialize("json", model)

            result = json.loads(data)
            response_body = [order["fields"] for order in result]

    elif request.method == 'POST':
        # Create dict from request
        payload = json.loads(request.body)

        # Creating the Order
        order = Order()

        customer = Customer()
        customer.customer_id = int(payload["customer_id"])
        customer.save()

        driver = Driver()
        driver.driver_id = int(payload["driver_id"])
        driver.save()

        order.order_number = payload["order_number"]
        order.order_datetime = payload["order_datetime"]
        order.store_name = payload["store_name"]
        order.customer_id = customer
        order.driver_id = driver
        order.item_name = payload["item_name"]
        order.item_price = float(payload["item_price"])
        order.item_picture = payload["item_picture"]
        order.total = float(payload["total"])
        order.status = payload["status"]

        # Posting order to DB
        order.save()
        status_code = 200
    elif request.method == "PATCH":
        # The only thing that can be changed is status and driver_id
        payload = json.loads(request.body)
        if not payload["order_id"]:
            status_code = 400
            response_body["status"] = "Error"
            response_body["description"] = "Failed to update order"
            response_body["error"] = "The order_id is missing"
            return JsonResponse(data=response_body, json_dumps_params={'indent': 2})

        order = Order.objects.get(int(payload["order_id"]))
        if payload["driver_id"]:
            order.driver_id = int(payload["driver_id"])
        if payload["status"]:
            order.status = payload["status"]

    response = JsonResponse(data=response_body, json_dumps_params={'indent': 2}, safe=False)
    response.status_code = status_code
    return response


from utilities import Course
import requests
from lxml import html
import json

headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.5",
    "Connection": "keep-alive",
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Firefox/91.0",
}

response = requests.get(url="https://catalog.purdue.edu/preview_program.php?catoid=14&poid=19156", headers=headers, verify=False)
doc = html.fromstring(response.text)
elements = doc.cssselect("li.acalog-course a")

multidis: list[Course] = []
for element in elements:
    text = element.text_content().strip()
    if (text != ""):
        subject = text[0:text.find(" ")]
        text = text[text.find(" ") + 1:]
        number = int(text[0: text.find(" ")])
        text = text[text.find("- ") + 2:]
        title = text
        multidis.append(
            {
                "type": "course",
                "value": Course(subject, number, title).get_dict()
            }
        )

multidis = {
    "title": "Multidisciplinary Experience",
    "type": "or",
    "value": multidis
}

with open("multidis.json", "w") as file:
    file.write(json.dumps(multidis, indent=4))


import abc
import binascii
import os
import sys
import warnings
from hashlib import sha384
from typing import Dict, Iterable

from hypothesis.configuration import mkdir_p, storage_directory
from hypothesis.errors import HypothesisException, HypothesisWarning
from hypothesis.utils.conventions import not_set

__all__ = [
    "DirectoryBasedExampleDatabase",
    "ExampleDatabase",
    "InMemoryExampleDatabase",
    "MultiplexedDatabase",
    "ReadOnlyDatabase",
]

def _usable_dir(path):
    """
    Returns True iff the desired path can be used as database path because
    either the directory exists and can be used, or its root directory can
    be used and we can make the directory as needed.
    """
    while not os.path.exists(path):
        # Loop terminates because the root dir ('/' on unix) always exists.
        path = os.path.dirname(path)
    return os.path.isdir(path) and os.access(path, os.R_OK | os.W_OK | os.X_OK)

def _db_for_path(path=None):
    if path is not_set:
        if os.getenv("HYPOTHESIS_DATABASE_FILE") is not None:  # pragma: no cover
            raise HypothesisException(
                "The $HYPOTHESIS_DATABASE_FILE environment variable no longer has any "
                "effect.  Configure your database location via a settings profile instead.\n"
                "https://hypothesis.readthedocs.io/en/latest/settings.html#settings-profiles"
            )

        path = storage_directory("examples")
        if not _usable_dir(path):  # pragma: no cover
            warnings.warn(
                HypothesisWarning(
                    "The database setting is not configured, and the default "
                    "location is unusable - falling back to an in-memory "
                    f"database for this session.  path={path!r}"
                )
            )
            return InMemoryExampleDatabase()
    if path in (None, ":memory:"):
        return InMemoryExampleDatabase()
    return DirectoryBasedExampleDatabase(str(path))

class _EDMeta(abc.ABCMeta):
    def __call__(self, *args, **kwargs):
        if self is ExampleDatabase:
            return _db_for_path(*args, **kwargs)
        return super().__call__(*args, **kwargs)

# This __call__ method is picked up by Sphinx as the signature of all ExampleDatabase
# subclasses, which is accurate, reasonable, and unhelpful.  Fortunately Sphinx
# maintains a list of metaclass-call-methods to ignore, and while they would prefer
# not to maintain it upstream (https://github.com/sphinx-doc/sphinx/pull/8262) we
# can insert ourselves here.
#
# This code only runs if Sphinx has already been imported; and it would live in our
# docs/conf.py except that we would also like it to work for anyone documenting
# downstream ExampleDatabase subclasses too.
if "sphinx" in sys.modules:
    try:
        from sphinx.ext.autodoc import _METACLASS_CALL_BLACKLIST

        _METACLASS_CALL_BLACKLIST.append("hypothesis.database._EDMeta.__call__")
    except Exception:
        pass

class ExampleDatabase(metaclass=_EDMeta):
    """An abstract base class for storing examples in Hypothesis' internal format.

    An ExampleDatabase maps each ``bytes`` key to many distinct ``bytes``
    values, like a ``Mapping[bytes, AbstractSet[bytes]]``.
    """

    @abc.abstractmethod
    def save(self, key: bytes, value: bytes) -> None:
        """Save ``value`` under ``key``.

        If this value is already present for this key, silently do nothing.
        """
        raise NotImplementedError(f"{type(self).__name__}.save")

    @abc.abstractmethod
    def fetch(self, key: bytes) -> Iterable[bytes]:
        """Return an iterable over all values matching this key."""
        raise NotImplementedError(f"{type(self).__name__}.fetch")

    @abc.abstractmethod
    def delete(self, key: bytes, value: bytes) -> None:
        """Remove this value from this key.

        If this value is not present, silently do nothing.
        """
        raise NotImplementedError(f"{type(self).__name__}.delete")

    def move(self, src: bytes, dest: bytes, value: bytes) -> None:
        """Move ``value`` from key ``src`` to key ``dest``. Equivalent to
        ``delete(src, value)`` followed by ``save(src, value)``, but may
        have a more efficient implementation.

        Note that ``value`` will be inserted at ``dest`` regardless of whether
        it is currently present at ``src``.
        """
        if src == dest:
            self.save(src, value)
            return
        self.delete(src, value)
        self.save(dest, value)

class InMemoryExampleDatabase(ExampleDatabase):
    """A non-persistent example database, implemented in terms of a dict of sets.

    This can be useful if you call a test function several times in a single
    session, or for testing other database implementations, but because it
    does not persist between runs we do not recommend it for general use.
    """

    def __init__(self):
        self.data = {}

    def __repr__(self) -> str:
        return f"InMemoryExampleDatabase({self.data!r})"

    def fetch(self, key: bytes) -> Iterable[bytes]:
        yield from self.data.get(key, ())

    def save(self, key: bytes, value: bytes) -> None:
        self.data.setdefault(key, set()).add(bytes(value))

    def delete(self, key: bytes, value: bytes) -> None:
        self.data.get(key, set()).discard(bytes(value))

def _hash(key):
    return sha384(key).hexdigest()[:16]

class DirectoryBasedExampleDatabase(ExampleDatabase):
    """Use a directory to store Hypothesis examples as files.

    Each test corresponds to a directory, and each example to a file within that
    directory.  While the contents are fairly opaque, a
    ``DirectoryBasedExampleDatabase`` can be shared by checking the directory
    into version control, for example with the following ``.gitignore``::

        # Ignore files cached by Hypothesis...
        .hypothesis/*
        # except for the examples directory
        !.hypothesis/examples/

    Note however that this only makes sense if you also pin to an exact version of
    Hypothesis, and we would usually recommend implementing a shared database with
    a network datastore - see :class:`~hypothesis.database.ExampleDatabase`, and
    the :class:`~hypothesis.database.MultiplexedDatabase` helper.
    """

    def __init__(self, path: str) -> None:
        self.path = path
        self.keypaths: Dict[str, str] = {}

    def __repr__(self) -> str:
        return f"DirectoryBasedExampleDatabase({self.path!r})"

    def _key_path(self, key):
        try:
            return self.keypaths[key]
        except KeyError:
            pass
        directory = os.path.join(self.path, _hash(key))
        self.keypaths[key] = directory
        return directory

    def _value_path(self, key, value):
        return os.path.join(self._key_path(key), _hash(value))

    def fetch(self, key: bytes) -> Iterable[bytes]:
        kp = self._key_path(key)
        if not os.path.exists(kp):
            return
        for path in os.listdir(kp):
            try:
                with open(os.path.join(kp, path), "rb") as i:
                    yield i.read()
            except OSError:
                pass

    def save(self, key: bytes, value: bytes) -> None:
        # Note: we attempt to create the dir in question now. We
        # already checked for permissions, but there can still be other issues,
        # e.g. the disk is full
        mkdir_p(self._key_path(key))
        path = self._value_path(key, value)
        if not os.path.exists(path):
            suffix = binascii.hexlify(os.urandom(16)).decode("ascii")
            tmpname = path + "." + suffix
            with open(tmpname, "wb") as o:
                o.write(value)
            try:
                os.rename(tmpname, path)
            except OSError:  # pragma: no cover
                os.unlink(tmpname)
            assert not os.path.exists(tmpname)

    def move(self, src: bytes, dest: bytes, value: bytes) -> None:
        if src == dest:
            self.save(src, value)
            return
        try:
            os.renames(self._value_path(src, value), self._value_path(dest, value))
        except OSError:
            self.delete(src, value)
            self.save(dest, value)

    def delete(self, key: bytes, value: bytes) -> None:
        try:
            os.unlink(self._value_path(key, value))
        except OSError:
            pass

class ReadOnlyDatabase(ExampleDatabase):
    """A wrapper to make the given database read-only.

    The implementation passes through ``fetch``, and turns ``save``, ``delete``, and
    ``move`` into silent no-ops.

    Note that this disables Hypothesis' automatic discarding of stale examples.
    It is designed to allow local machines to access a shared database (e.g. from CI
    servers), without propagating changes back from a local or in-development branch.
    """

    def __init__(self, db: ExampleDatabase) -> None:
        assert isinstance(db, ExampleDatabase)
        self._wrapped = db

    def __repr__(self) -> str:
        return f"ReadOnlyDatabase({self._wrapped!r})"

    def fetch(self, key: bytes) -> Iterable[bytes]:
        yield from self._wrapped.fetch(key)

    def save(self, key: bytes, value: bytes) -> None:
        pass

    def delete(self, key: bytes, value: bytes) -> None:
        pass

class MultiplexedDatabase(ExampleDatabase):
    """A wrapper around multiple databases.

    Each ``save``, ``fetch``, ``move``, or ``delete`` operation will be run against
    all of the wrapped databases.  ``fetch`` does not yield duplicate values, even
    if the same value is present in two or more of the wrapped databases.

    This combines well with a :class:`ReadOnlyDatabase`, as follows:

    .. code-block:: python

        local = DirectoryBasedExampleDatabase("/tmp/hypothesis/examples/")
        shared = CustomNetworkDatabase()

        settings.register_profile("ci", database=shared)
        settings.register_profile(
            "dev", database=MultiplexedDatabase(local, ReadOnlyDatabase(shared))
        )
        settings.load_profile("ci" if os.environ.get("CI") else "dev")

    So your CI system or fuzzing runs can populate a central shared database;
    while local runs on development machines can reproduce any failures from CI
    but will only cache their own failures locally and cannot remove examples
    from the shared database.
    """

    def __init__(self, *dbs: ExampleDatabase) -> None:
        assert all(isinstance(db, ExampleDatabase) for db in dbs)
        self._wrapped = dbs

    def __repr__(self) -> str:
        return "MultiplexedDatabase({})".format(", ".join(map(repr, self._wrapped)))

    def fetch(self, key: bytes) -> Iterable[bytes]:
        seen = set()
        for db in self._wrapped:
            for value in db.fetch(key):
                if value not in seen:
                    yield value
                    seen.add(value)

    def save(self, key: bytes, value: bytes) -> None:
        for db in self._wrapped:
            db.save(key, value)

    def delete(self, key: bytes, value: bytes) -> None:
        for db in self._wrapped:
            db.delete(key, value)

    def move(self, src: bytes, dest: bytes, value: bytes) -> None:
        for db in self._wrapped:
            db.move(src, dest, value)


from matplotlib import pyplot as plt
from matplotlib import font_manager

# 选用一个图表通用字体存为my_font，为显示中文
my_font = font_manager.FontProperties(fname="字体文件所在目录")

# 数据
x = range(11)
y = [1, 5, 6, 2, 11, 8, 9, 23, 5, 6, 8]

# 绘制折线图
plt.plot(x, y, label="图例名", color="折线颜色")
# 设置图形大小
plt.figure(figsize=(20,8),dpi=80)

# 设置坐标
_xtick_label = ["{}年".format(i) for i in x]
plt.xticks(x, _xtick_label, fontproperties=my_font)

# 绘制网格
plt.grid(alpha= 0.4, linestyle="--")

# 展示
plt.show()