#!/usr/bin/python2.7

from __future__ import division
import copy
import logging
import cgi
import datetime
import calendar
import urllib
import urllib2
import webapp2
import random
import re
import numpy as np
import copy
import pickle
import os
import json
import time
import math
import xml.etree.ElementTree as ET
import __builtin__
from google.appengine.ext import db
from google.appengine.api import users
from google.appengine.ext.webapp import template
from google.appengine.api import taskqueue
from google.appengine.ext import deferred
from backend.routeInfo import *
from backend.dataModel import *
from backend.buildMuniRoute import *
from backend.saveLocations import *
from backend.kdtree import *

tz_offset = 7 # UTC - PDT

class index(webapp2.RequestHandler):
    def get(self):
        self.slow_speed_limit = 5
        path = os.path.join(os.path.dirname(__file__), 'templates/bootstrap_template_index.html')
        t = time.time()
        last_key = getKeys()
        #self.response.out.write('Time elapsed: %s
' % (time.time() - t))
        BDA = BusDataAccess()

        routeTag = 'All'
        filter = False
        historic_count = 1000

        current_bus_locations = BDA.currentBusLocations(last_key, filter)
        #self.response.out.write('Time elapsed: %s
' % (time.time() - t))
        recent_bus_data = BDA.recentBusData(historic_count, routeTag, False, 'Array')
        #self.response.out.write('Time elapsed: %s
' % (time.time() - t))
        #logging.info(recent_bus_data)
        #self.response.out.write(BDA.recentBusBins())
        count_array             = recent_bus_data['count']
        avg_speed_kmhr_array    = recent_bus_data['avgSpeedKmHr']
        slow_bus_ratio_array    = recent_bus_data['ratioSlowBusses']

        current_route_data = BDA.recentBusData(100 , filter, last_key, 'Dict')
        all_route_data = BDA.recentBusData(100 , False, last_key, 'Dict')

        #self.response.out.write('Total render time: %s
' % (time.time() - t))

        self.response.out.write(template.render(path, {"title": "The Bad Bus",
                                                       "bus_array_str" : json.dumps(current_bus_locations),
                                                       "bus_history_json" : json.dumps(count_array),
                                                       "bus_count": count_array[-1][1],
                                                       "avg_speed_kmhr_array" : json.dumps(avg_speed_kmhr_array),
                                                       "slow_bus_ratio_array" : json.dumps(slow_bus_ratio_array),
                                                       "route_table" : self.dict_array_to_table(current_route_data),
                                                       "route_array" : json.dumps(all_route_data.keys())
                                                       }))
    def dict_array_to_table(self, json_array):
        xml_output = "

"
        xml_output += '	'.join(['Route','Number Busses','Avg Speed (Km/Hr)','Ratio Slow Busses'])
        xml_output += '

\n
'
        for routeTag in json_array:
            row = json_array[routeTag]
            logging.info('ROW FOR ROUTETAG %s' % routeTag)
            logging.info(row)
            xml_output += '
%s	' % (row['route'], row['route'])
            xml_output += '	'.join([str(s) for s in [row['count'],row['avgSpeedKmHr'],row['ratioSlowBusses']]]) + '
\n'

        xml_output += '

\n'
        return xml_output

class displayRoute(webapp2.RequestHandler):
    def get(self, routeTag):
        self.slow_speed_limit = 5
        path = os.path.join(os.path.dirname(__file__), 'templates/bootstrap_template_index.html')
        t = time.time()
        last_key = getKeys()
        #self.response.out.write('Time elapsed: %s
' % (time.time() - t))
        BDA = BusDataAccess()
        historic_count =  100

        current_bus_locations = BDA.currentBusLocations(last_key, routeTag)
        #self.response.out.write('Time elapsed: %s
' % (time.time() - t))
        recent_bus_data = BDA.recentBusData(historic_count, routeTag, False, 'Array')
        count_array             = recent_bus_data['count']
        avg_speed_kmhr_array    = recent_bus_data['avgSpeedKmHr']
        slow_bus_ratio_array    = recent_bus_data['ratioSlowBusses']

        current_route_data = BDA.recentBusData(100 , routeTag, last_key, 'Dict')
        all_route_data = BDA.recentBusData(100 , False, last_key, 'Dict') # All busses

        #self.response.out.write('Total render time: %s
' % (time.time() - t))
        self.response.out.write(template.render(path, {"title": "Route %s" % (routeTag),
                                                       "bus_array_str" : json.dumps(current_bus_locations),
                                                       "bus_history_json" : json.dumps(count_array),
                                                       "bus_count": count_array[-1][1],
                                                       "avg_speed_kmhr_array" : json.dumps(avg_speed_kmhr_array),
                                                       "slow_bus_ratio_array" : json.dumps(slow_bus_ratio_array),
                                                       "route_table" : self.dict_array_to_table(current_bus_locations),
                                                       "route_array" : json.dumps(all_route_data.keys())
                                                       }))

    def dict_array_to_table(self, array_of_dicts):
        xml_output = "

"
        xml_output += '	'.join(['Vehicle Id','Speed (Km/Hr)','Heading', 'Next Bus','Direction Tag','Distance Next Bus'])
        xml_output += '

\n
'
        for row in array_of_dicts:
            if type(row['distanceNextBus']) is float:
                dist = round(row['distanceNextBus'],2)
            else:
                dist = 'None'
            xml_output += '
%s	' % (row['vehicleId'], row['vehicleId'])
            xml_output += '	'.join([str(s) for s in [row['speedKmHr'],row['heading'],row['nextBusId'],row['dirTag'],dist]])
            xml_output += '
\n'

        xml_output += '

\n'
        return xml_output
class displayVehicleId(webapp2.RequestHandler):
    def get(self, vehicleId):
        path = os.path.join(os.path.dirname(__file__), 'templates/bootstrap_template_index.html')
        t = time.time()
        last_key = getKeys()
        #self.response.out.write('Time elapsed: %s
' % (time.time() - t))
        BDA = BusDataAccess()
        historic_count =  100

        current_bus_locations = BDA.recentLocationsVehicleId(vehicleId, 100)
        current_bus_locations_simple = filter_keys(current_bus_locations, ['lat','lon','text','speedKmHr','heading','type','vehicleId'])
        avg_speed_kmhr_array    = self.locations_to_chart_data(current_bus_locations)
        all_route_data = BDA.recentBusData(100 , False, last_key, 'Dict') # All busses
        route_table =  self.dict_array_to_table(current_bus_locations)
        #self.response.out.write('Total render time: %s
' % (time.time() - t))
        self.response.out.write(template.render(path, {"title": "Vehicle %s" % (vehicleId),
                                                       "bus_array_str" : json.dumps(current_bus_locations_simple),
                                                       "bus_count": len(current_bus_locations),
                                                       "avg_speed_kmhr_array" : json.dumps(avg_speed_kmhr_array),
                                                       "route_table" : route_table,
                                                       "route_array" : json.dumps(all_route_data.keys())
                                                       }))

    def dict_array_to_table(self, array_of_dicts):
        xml_output = "

"
        xml_output += '	'.join(['Route','Created','Speed (Km/Hr)','Heading', 'Next Bus','Distance To Next Bus (Km)'])
        xml_output += '

\n
'
        for row in array_of_dicts:
            if type(row['distanceNextBus']) is float:
                dist = round(row['distanceNextBus'],2)
            else:
                dist = 'None'
            xml_output += '
%s	' % (row['routeTag'], row['routeTag'])
            xml_output += '	'.join([str(s) for s in ['%s' % (row['created']),row['speedKmHr'],row['heading'],row['nextBusId'], dist]])
            xml_output += '
\n'

        xml_output += '

\n'
        return xml_output

    def locations_to_chart_data(self, data):
        speed_array = []
        for i in data:
            t = calendar.timegm(i['created'].utctimetuple()) * 1000 - tz_offset * 60 * 60 * 1000
            speed_array.append([t,i['speedKmHr']])
        speed_array.reverse()
        return speed_array

class BusDataAccess():
    def recentBusData(self, count = 1000, filter = 'All', ancestor_key = False, output = 'Array'):
        #get array with time stamps and number of busses
        time_start = time.time()
        q = ImportRouteMetrics.all()
        if filter: # Only take the 'ALL' Tag
            q.filter('routeTag =', filter) # Only Get the big imports, not the routes
        if ancestor_key:
            q.ancestor(ancestor_key)
        q.order('-created')

        bus_data =  q.run(limit = count)

        #self.response.out.write('Time elapsed: %s
' % (time.time() - time_start))
        if output == 'Array':
            return_data = {'count':[],'avgSpeedKmHr':[],"ratioSlowBusses":[],"created":[]}
            for i in bus_data:
                #logging.info(time.time() - time_start)
                t = calendar.timegm(i.created.utctimetuple()) * 1000 - tz_offset * 60 * 60 * 1000
                return_data['count'].append([t, i.count])
                return_data['avgSpeedKmHr'].append([t, i.avgSpeedKmHr])
                return_data['ratioSlowBusses'].append([t, i.ratioSlowBusses])
            return_data['count'].reverse()
            return_data['avgSpeedKmHr'].reverse()
            return_data['ratioSlowBusses'].reverse()
            #self.response.out.write('DATA TYPE BUS DATA %s
' % (cgi.escape(str(type(bus_data)))))
            #self.response.out.write('recent bus data Time elapsed: %s
' % (time.time() - time_start))
            return return_data
        elif output == 'Dict':
            return_data = {}
            for i in bus_data:
                return_data[i.routeTag] = {'route': i.routeTag, 'count':i.count, 'avgSpeedKmHr': round(i.avgSpeedKmHr,2), 'ratioSlowBusses' : round(i.ratioSlowBusses,2)}
            #self.response.out.write('Time elapsed: %s
' % (time.time() - time_start))
            return return_data

    def currentBusLocations(self, ancestor_key, filter =  False, count = 10):
        q = MuniVehicleLocations.all()
        q.ancestor(ancestor_key)
        return_array = []
        if filter != False:
            count = 1000
        fetched_results = q.fetch(limit = count)
        for s in fetched_results:
            if s.routeTag == filter or filter == False:
                return_array.append({"lat":s.lat, "lon":s.lon,
                                       "type" : "bus",
                                       "text" : "Speed %s Km/Hr" % (s.speedKmHr),
                                       "speedKmHr": s.speedKmHr,
                                       "heading" : s.heading,
                                       "vehicleId": s.vehicleId,
                                       "nextBusId": s.nextBusId,
                                       "distanceNextBus": s.distanceNextBus,
                                       "routeTag": s.routeTag,
                                       "dirTag": s.dirTag})
        return return_array
    def recentBusBins(self, count = 10000, bin_round = 0.01):
        q = MuniVehicleLocations.all()
        q.order('-created')
        fetched_results = q.fetch(limit = count)
        bin_dict = {}
        for s in fetched_results:
            lon_bin, lat_bin = math.floor(s.lon / bin_round) * bin_round, math.floor(s.lat / bin_round) * bin_round
            if lon_bin not in bin_dict:
                bin_dict[lon_bin] = {}
            if lat_bin not in bin_dict[lon_bin]:
                bin_dict[lon_bin][lat_bin]=[]
            bin_dict[lon_bin][lat_bin].append(s.speedKmHr)

        return_dict = {}
        for x in bin_dict:
            for y in bin_dict[x]:
                if x not in return_dict:
                    return_dict[x] = {}
                return_dict[x][y] = {'count' : len(bin_dict[x][y]), 'avg_speed': sum(bin_dict[x][y]) / len(bin_dict[x][y])}
        return return_dict
    def recentLocationsVehicleId(self, vehicleId, count = 1000):
        q = MuniVehicleLocations.all()
        q.filter('vehicleId = ', vehicleId)
        q.order('-created')
        fetched_results = q.fetch(limit = count)
        return_array = []
        for s in fetched_results:
            return_array.append({"lat":s.lat, "lon":s.lon,
                        "type" : "bus",
                        "text" : "Speed %s Km/Hr" % (s.speedKmHr),
                        "created" : s.created,
                        "speedKmHr": s.speedKmHr,
                        "heading" : s.heading,
                        "vehicleId":s.vehicleId,
                        "nextBusId":s.nextBusId,
                        "distanceNextBus":s.distanceNextBus,
                        "routeTag":s.routeTag})
        return return_array

class deleteMuniRoute(webapp2.RequestHandler):
    def get(self, routeName):
        stops =  db.GqlQuery("SELECT * from MuniStop where routeTag = :1",
                             routeName)
        logging.info("Stops deleted %s " % stops.count())
        self.response.out.write("
%s stops deleted

\n" %
                                (stops.count()))
        db.delete(stops)
        routes = db.GqlQuery("SELECT * from MuniRoute where tag = :1",
                                                        routeName)
        self.response.out.write("
%s directions deleted

\n" %
                                (routes.count()))
        db.delete(routes)

def findTotalDist(stop_dict):
    # Sum up distance along stop dist
    pass

def getKeys(offset = 0, limit = 1):
    q = VehicleImport.all()
    q.order('-created')
    ancestor_key =  None
    for p in q.run(offset = offset, limit = limit):
        ancestor_key = p.key()
    return ancestor_key

app = webapp2.WSGIApplication([
                               (r'/buildMuniRoute/(.*)',buildMuniRoute),
                               (r'/deleteMuniRoute/(.*)',deleteMuniRoute),
                               (r'/saveCurrentMuniLocations',saveCurrentMuniLocations),
                               (r'/route/(.*)',displayRoute),
                               (r'/vehicleId/(.*)',displayVehicleId),
                               ('/',index),
                               ],
                              debug=False)


import datetime
import traceback

from PyQt5.QtCore import pyqtSlot

from debug.qthread_with_logging import QThreadWithLogging
from data.schedule import schedule as entire_schedule
from threading import Lock
from music_downloader import MusicDownloader

class Scheduler(QThreadWithLogging):
    last = datetime.datetime.now()

    def __init__(self, main_platform):
        QThreadWithLogging.__init__(self)
        self.main_platform = main_platform
        self.schedule = []
        self.name_for_static_alarm = []
        self.name_for_static_alarm.append('입구폐쇄')
        self.name_for_static_alarm.append('점호준비')
        self.name_for_static_alarm.append('점호시작')
        self.name_for_static_alarm.append('점호종료')
        self.name_for_static_alarm.append('기숙사퇴실')
        self.name_for_static_alarm.append('복귀체크')

        self.tag_command_queue = []
        self.prev = datetime.datetime(2020, 8, 1)
        self.curr = datetime.datetime.now()
        self.debug_all_around = False

        self.music_downloader = MusicDownloader(self)

        self.lock = Lock()

        for grade in ['1', '2', '3']:
            for meal in ['아침', '점심', '저녁']:
                self.name_for_static_alarm.append(f'{grade}학년{meal}')
        self.name_for_static_alarm.append('저녁외출')
        self.name_for_static_alarm.append('점심외출')

    @pyqtSlot()
    def all_around_test(self):
        self.log(f'lock to all_around_test')
        with self.lock:
            now = datetime.datetime.now()
            self.prev = now.replace(hour=0, minute=0, second=0, microsecond=0)
            self.curr = now.replace(hour=23, minute=59, second=59, microsecond=999999)
            self.debug_all_around = True
        self.log(f'unlock to all_around_test')

    @pyqtSlot(str)
    def push_tag_command(self, tag: str):
        self.log(f'push {tag}')
        self.tag_command_queue.append(tag)

    def decode_tag(self, tag: str):
        self.log(f'decode {tag}')
        if tag == '기상송':
            with self.main_platform.storage_manager.lock:
                self.log(f'getlock 기상송')
                self.main_platform.music_player.push_to_playlist(
                    self.main_platform.storage_manager.files_to_play)
        elif tag == '기상송초기화':
            with self.main_platform.storage_manager.lock:
                self.main_platform.storage_manager.clear_internal_storage()
        elif tag == '기상송다운로드':
            with self.main_platform.storage_manager.lock:
                self.main_platform.storage_manager.clear_internal_storage()
                self.music_downloader.download()
        elif tag == '아침운동체크':
            self.main_platform.music_player.music_pause()
            self.main_platform.music_player.play_unstoppable_music('audio/아침운동체크.mp3')
            self.main_platform.music_player.music_resume()
        elif tag in self.name_for_static_alarm:
            self.main_platform.music_player.push_to_playlist([f'audio/{tag}.mp3'])
        else:
            self.log(f'unknown command {tag}')
        self.log(f'finish {tag}')

    def run(self):
        if self.curr.date().weekday() == 5:
            self.schedule = entire_schedule['토']
        elif self.curr.date().weekday() == 6:
            self.schedule = entire_schedule['일']
        else:
            self.schedule = entire_schedule['평일']
        while not self.isFinished():
            with self.lock:
                if not self.debug_all_around:
                    self.curr = datetime.datetime.now()
                if self.tag_command_queue:
                    self.decode_tag(self.tag_command_queue[0])
                    del self.tag_command_queue[0]
                try:
                    if self.curr.date() != self.prev.date():
                        self.prev = self.curr
                        if self.curr.date().weekday() == 5:
                            self.schedule = entire_schedule['토']
                        elif self.curr.date().weekday() == 6:
                            self.schedule = entire_schedule['일']
                        else:
                            self.schedule = entire_schedule['평일']

                    is_trying_to_play = False
                    for certain_time in self.schedule:
                        if self.prev.time() < certain_time <= self.curr.time():
                            self.prev = datetime.datetime.combine(self.curr.date(), certain_time)
                            self.decode_tag(self.schedule[certain_time])
                            is_trying_to_play = True
                            break

                    if self.debug_all_around and not is_trying_to_play:
                        self.debug_all_around = False
                        self.prev = datetime.datetime.now()
                        self.curr = datetime.datetime.now()
                except:
                    self.log(traceback.format_exc())


import cv2
import os
import sys

cam = cv2.VideoCapture(0, cv2.CAP_DSHOW)

start = False
counter = 4000
num_samples = int(sys.argv[1]) + counter
IMG_SAVE_PATH = 'images'
IMG_CLASS_PATH = ' '

try:
    os.mkdir(IMG_SAVE_PATH)
except FileExistsError:
    pass

while True:
    ret, frame = cam.read()
    if not ret:
        print("failed to grab frame")
        break
    if counter == num_samples:
       break

    cv2.rectangle(frame, (10, 30), (310, 330), (0, 255, 0), 2)

    k = cv2.waitKey(1)
    if k == ord('c'):
            name = 'OpenHand6'
            IMG_CLASS_PATH = os.path.join(IMG_SAVE_PATH, name)
            os.mkdir(IMG_CLASS_PATH)
    if k == ord('v'):
            name = 'ClosedFist7'
            IMG_CLASS_PATH = os.path.join(IMG_SAVE_PATH, name)
            os.mkdir(IMG_CLASS_PATH)
    if k == ord('b'):
            name = 'ThumbsUp7'
            IMG_CLASS_PATH = os.path.join(IMG_SAVE_PATH, name)
            os.mkdir(IMG_CLASS_PATH)
    if k == ord('m'):
            name = 'PeaceSignUpwards6'
            IMG_CLASS_PATH = os.path.join(IMG_SAVE_PATH, name)
            os.mkdir(IMG_CLASS_PATH)
    if k == ord('a'):
            name = 'PeaceSignDownwards6'
            IMG_CLASS_PATH = os.path.join(IMG_SAVE_PATH, name)
            os.mkdir(IMG_CLASS_PATH)
    if k == ord('d'):
            name = 'PointingLeft6'
            IMG_CLASS_PATH = os.path.join(IMG_SAVE_PATH, name)
            os.mkdir(IMG_CLASS_PATH)
    if k == ord('n'):
            name = 'Nothing6'
            IMG_CLASS_PATH = os.path.join(IMG_SAVE_PATH, name)
            os.mkdir(IMG_CLASS_PATH)

    if start:
        roi = frame[25:335, 8:315]
        save_path = os.path.join(IMG_CLASS_PATH, '{}.jpg'.format(counter + 1))
        print(save_path)
        cv2.imwrite(save_path, roi)
        counter += 1
    font = cv2.FONT_HERSHEY_SIMPLEX
    cv2.putText(frame,"Collecting {}".format(counter),
            (10, 20), font, 0.7, (0, 255, 255), 2, cv2.LINE_AA)
    cv2.imshow("Collecting images", frame)

    if k == ord('s'):
        start = not start

    if k == ord('q'):
            break

print("\n{} image(s) saved to {}".format(counter, IMG_CLASS_PATH))
cam.release()
cv2.destroyAllWindows()


import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.autograd as autograd
from .modelutils import SN, gcn_layer, init_weights
from loguru import logger
import NCE.toolbox as tb
from scipy.linalg import block_diag
import random
from utils.datautils import treeDataset, collate_wrapper, hdf5Dataset, hdf5_jsonDataset
from .RL_utils import get_advantages, ppo_loss
from scipy import stats
from sklearn  import metrics

class Crit(nn.Module):
    def __init__(self, config, sn=False, outlayer=True):
        """
        The initial func
        :param config: The dict containing all needed parameters
        """
        super(Crit, self).__init__()
        self.atom_size=config["atom_size"]
        self.bond_size=config["bond_size"]
        self.layers=config["dis_layers"]
        self.config=config
        self.grammar=config["grammar"]
        self.bonds=config["bonds"]
        self.atoms=config["atoms"]
        self.outlayer=outlayer
        self.clock=100

        self.emb_size=32
        self.bond_emb_size=8

        self.atom_linear=nn.Linear(self.atom_size, self.emb_size, bias=True)
        self.bond_linear=nn.Linear(self.bond_size, self.bond_emb_size, bias=True)
        self.layer=[]
        for _ in range(self.layers):
            if sn:
                self.layer.append(SN(gcn_layer(self.emb_size, self.emb_size, self.bond_emb_size, sn=True)))
            else:
                self.layer.append(gcn_layer(self.emb_size, self.emb_size, self.bond_emb_size, bias=True))

        self.gcn=nn.Sequential(*self.layer)
        if sn:
            self.out_layer=SN(nn.Linear(self.emb_size, 1))
        else:
            self.out_layer = nn.Linear(self.emb_size, 1)
        self.act=nn.Sigmoid()

        self.cl=nn.NLLLoss()
        self.acc=[0,0,0]
        if "dis_optimizer" in config:
            self.optimizer=config['dis_optimizer']
        else:
            lr=config["dis_lr"]
            self.optimizer=torch.optim.Adam(self.parameters(), lr=lr,
                                           weight_decay=1e-4)
        self.apply(init_weights)

    def forward(self, inp):
        mol, adj=inp
        maskmol=mol.clamp(0.0, 1.0).float().unsqueeze(2)
        mask=(maskmol.unsqueeze(1)*maskmol.unsqueeze(2))-torch.eye(adj.shape[1]).cuda(mol.device).unsqueeze(0).unsqueeze(3)
        mask=mask.clamp(0,1)
        adj=nn.functional.one_hot(adj, self.bond_size).float()
        adj=self.bond_linear(adj)*mask # not very sure to remain
        mol=nn.functional.one_hot(mol, self.atom_size).float()
        mol=self.atom_linear(mol)*maskmol
        assert (adj*(1-mask)).sum()==0
        adj=adj.permute(0,3,1,2)
        assert (mol*(1-maskmol)).sum()==0

        inp=[mol, adj, maskmol, mask]
        mol, adj, _, _=self.gcn(inp) #the mol is in the shape of n*self.embdsize
        mol=mol.mean(1)#torch.cumsum(mol, 0)

        x=self.out_layer(mol)
        if self.outlayer:
            x=nn.Sigmoid()(x)
        return x

    def reg_onestep(self, data):
        self.train()
        self.zero_grad()
        self.optimizer.zero_grad()
        mol, adj, score=data
        critdevice=next(self.parameters()).device
        mol=mol.cuda(critdevice)
        adj=adj.cuda(critdevice)
        score=(score.cuda(critdevice)<0.2).float().reshape(-1)
        pred=self.forward([mol, adj]).reshape(-1)
        loss=score*torch.log(pred+1e-12)+(1-score)*torch.log(1-pred+1e-12)
        loss=-loss.mean()
        loss.backward()
        self.optimizer.step()
        return loss

    def reg_eval(self, data):
        self.eval()
        Pred=[]
        Score=[]
        for x in data:
            mol, adj, score=x
            mol=mol.cuda()
            adj=adj.cuda()
            with torch.no_grad():
                pred=self.forward([mol, adj]).reshape(-1).cpu().numpy().tolist()
            Pred.extend(pred)
            Score.extend((score<0.2).float().numpy().tolist())
        fpr, tpr, _=metrics.roc_curve(Score, Pred)
        acc=[int(int(y>0.5)==x) for x,y in zip(Score, Pred)]
        acc=sum(acc)*1.0/len(acc)
        logger.info("Datasize is {} Pred size is {} Reg Evaluation auc is {}, acc is {}".format(len(Score), len(Pred), metrics.auc(fpr, tpr), acc))

    def get_score(self, mol):
        with torch.no_grad():
            critdevice = next(self.parameters()).device
            mol, adj = mol.get_mol_adjs([mol], [0])
            ret= self.forward(
                [torch.tensor(mol).to(critdevice), torch.tensor(adj).to(critdevice)])[0]

            ret=float(ret.cpu().numpy()[0])
            return ret, ret
    def get_single_score(self, mol):
        return self.get_score(mol)[0]
    def ft_onestep(self):
        fake=self.config["fake"]
        fakelabel=torch.ones(len(fake[0]))
        loss1=self.reg_onestep([fake[0], fake[1], fakelabel])
        try:
            pos_data=self.config["antib_train_data_pos"].next()
        except:
            self.config["antib_train_data_pos"]=iter(self.config["raw_atd_pos"])
            pos_data=self.config["antib_train_data_pos"].next()
        try:
            neg_data=self.config["antib_train_data_neg"].next()
        except:
            self.config["antib_train_data_neg"]=iter(self.config["raw_atd_neg"])
            neg_data=self.config["antib_train_data_neg"].next()
        loss2=self.reg_onestep(pos_data)
        loss3=self.reg_onestep(neg_data)


import asyncio
import logging

import aiohttp
from rapidfuzz import fuzz

from ...bot import Bot
from ...box import box
from ...command import argument
from ...event import Message
from ...event import YuiSystemStart
from ...utils.html import get_root

logger = logging.getLogger(__name__)

REF_URLS: dict[str, str] = {
    "html": "https://developer.mozilla.org/en-US/docs/Web/HTML/Element",
    "css": "https://developer.mozilla.org/en-US/docs/Web/CSS/Reference",
    "python": "https://docs.python.org/3/library/",
}

def parse(
    blob: bytes,
    selector: str,
    url_prefix: str,
) -> list[tuple[str, str]]:
    h = get_root(blob)
    a_tags = h.cssselect(selector)

    result = []
    for a in a_tags:
        name = str(a.text_content())
        link = f"{url_prefix}{a.get('href')}"
        result.append(
            (
                name,
                link,
            ),
        )
    return result

async def fetch_css_ref(bot: Bot):
    logger.info("fetch css ref start")

    url = "https://developer.mozilla.org/en-US/docs/Web/CSS/Reference"
    async with aiohttp.ClientSession() as session, session.get(url) as resp:
        blob = await resp.read()

    body = await bot.run_in_other_process(
        parse,
        blob,
        "a[href^=\\/en-US\\/docs\\/Web\\/CSS\\/]",
        "https://developer.mozilla.org",
    )

    await bot.cache.set("REF_CSS", body)

    logger.info("fetch css ref end")

async def fetch_html_ref(bot: Bot):
    logger.info("fetch html ref start")

    url = "https://developer.mozilla.org/en-US/docs/Web/HTML/Element"
    async with aiohttp.ClientSession() as session, session.get(url) as resp:
        blob = await resp.read()

    body = await bot.run_in_other_process(
        parse,
        blob,
        "a[href^=\\/en-US\\/docs\\/Web\\/HTML\\/Element\\/]",
        "https://developer.mozilla.org",
    )
    await bot.cache.set("REF_HTML", body)

    logger.info("fetch html ref end")

def parse_python(blob: bytes) -> list[tuple[str, str, str]]:
    h = get_root(blob)
    a_tags = h.cssselect("a.reference.internal")

    result = []
    for a in a_tags:
        code_els = a.cssselect("code.docutils.literal")
        name = str(a.text_content()).strip()
        link = f'https://docs.python.org/3/library/{a.get("href")}'
        if code_els:
            for code_el in code_els:
                result.append(
                    (
                        str(code_el.text_content()).strip(),
                        name,
                        link,
                    ),
                )
        else:
            result.append(
                (
                    "",
                    name,
                    link,
                ),
            )
    return result

async def fetch_python_ref(bot: Bot):
    logger.info("fetch python ref start")

    url = "https://docs.python.org/3/library/"
    async with aiohttp.ClientSession() as session, session.get(url) as resp:
        blob = await resp.read()

    body = await bot.run_in_other_process(
        parse_python,
        blob,
    )

    await bot.cache.set("REF_PYTHON", body)

    logger.info("fetch python ref end")

@box.on(YuiSystemStart)
async def on_start(bot):
    logger.info("on_start ref")
    tasks = [
        asyncio.create_task(fetch_css_ref(bot)),
        asyncio.create_task(fetch_html_ref(bot)),
        asyncio.create_task(fetch_python_ref(bot)),
    ]
    await asyncio.wait(tasks)
    return True

@box.cron("0 3 * * *")
async def refresh(bot):
    logger.info("refresh ref")
    tasks = [
        asyncio.create_task(fetch_css_ref(bot)),
        asyncio.create_task(fetch_html_ref(bot)),
        asyncio.create_task(fetch_python_ref(bot)),
    ]
    await asyncio.wait(tasks)

@box.command("html", ["htm"])
@argument("keyword", nargs=-1, concat=True, count_error="키워드를 입력해주세요")
async def html(bot, event: Message, keyword: str):
    """
    HTML 레퍼런스 링크

    `{PREFIX}html tbody` (`tbody` TAG에 대한 레퍼런스 링크)

    """

    data = await bot.cache.get("REF_HTML")
    if data is None:
        await bot.say(
            event.channel,
            "아직 레퍼런스 관련 명령어의 실행준비가 덜 되었어요. 잠시만 기다려주세요!",
        )
        return

    name = None
    link = None
    ratio = -100.0
    for _name, _link in data:
        _ratio = fuzz.ratio(keyword, _name)
        if _ratio > ratio:
            name = _name
            link = _link
            ratio = _ratio

    if ratio > 40:
        await bot.say(event.channel, f"HTML `{name}` - {link}")
    else:
        await bot.say(event.channel, "비슷한 HTML Element를 찾지 못하겠어요!")

@box.command("css")
@argument("keyword", nargs=-1, concat=True, count_error="키워드를 입력해주세요")
async def css(bot, event: Message, keyword: str):
    """
    CSS 레퍼런스 링크

    `{PREFIX}css color` (`color` 에 대한 레퍼런스 링크)

    """

    data = await bot.cache.get("REF_CSS")
    if data is None:
        await bot.say(
            event.channel,
            "아직 레퍼런스 관련 명령어의 실행준비가 덜 되었어요. 잠시만 기다려주세요!",
        )
        return

    name = None
    link = None
    ratio = -100.0
    for _name, _link in data:
        _ratio = fuzz.ratio(keyword, _name)
        if _ratio > ratio:
            name = _name
            link = _link
            ratio = _ratio

    if ratio > 40:
        await bot.say(event.channel, f"CSS `{name}` - {link}")
    else:
        await bot.say(event.channel, "비슷한 CSS 관련 요소를 찾지 못하겠어요!")

@box.command("php")
@argument("keyword", nargs=-1, concat=True, count_error="키워드를 입력해주세요")
async def php(bot, event: Message, keyword: str):
    """
    PHP 레퍼런스 링크

    `{PREFIX}php json_encode` (`json_encode` 에 대한 레퍼런스 링크)

    """

    raw_path = keyword.replace("::", ".")
    if "::" in keyword:
        superclass, func = (
            keyword.lower()
            .replace("$", "")
            .replace("__", "")
            .replace("_", "-")
            .split("::")
        )
    elif keyword.startswith("mysqli_stmt_"):
        superclass = "mysqli-stmt"
        func = keyword[12:].replace("_", "-")
    elif keyword.startswith("mysqli_"):
        superclass = "mysqli"
        func = keyword[7:].replace("_", "-")
    else:
        superclass, func = (
            "function",
            keyword.lower()
            .replace("$", "")
            .replace("__", "")
            .replace("_", "-"),
        )

    urls = [
        f"http://www.php.net/manual/en/{superclass}.{func}.php",
        f"http://php.net/{raw_path}",
    ]

    async with aiohttp.ClientSession() as session:
        for url in urls:
            async with session.get(url) as res, res:
                res_url = str(res.url)
                if "manual-lookup.php" in res_url:
                    continue
                if res.status == 200:
                    await bot.say(event.channel, f"PHP `{keyword}` - {res_url}")
                    break
        else:
            await bot.say(
                event.channel,
                "비슷한 PHP 관련 요소를 찾지 못하겠어요!",
            )

@box.command("python", ["py"])
@argument("keyword", nargs=-1, concat=True, count_error="키워드를 입력해주세요")
async def python(bot, event: Message, keyword: str):
    """
    Python library 레퍼런스 링크

    `{PREFIX}py re` (`re` 내장 모듈에 대한 레퍼런스 링크)

    """

    data = await bot.cache.get("REF_PYTHON")
    if data is None:
        await bot.say(
            event.channel,
            "아직 레퍼런스 관련 명령어의 실행준비가 덜 되었어요. 잠시만 기다려주세요!",
        )
        return

    name = None
    link = None
    ratio = -100.0
    for code, _name, _link in data:
        _ratio = (
            fuzz.ratio(keyword, code) if code else fuzz.ratio(keyword, _name)
        )
        if _ratio > ratio:
            name = _name
            link = _link
            ratio = _ratio

    if ratio > 40:
        await bot.say(event.channel, f"Python {name} - {link}")
    else:
        await bot.say(event.channel, "비슷한 Python library를 찾지 못하겠어요!")


"""
Various procedures used in the drawing
"""

from multiplicates import overlap, overlap_len
from collections import OrderedDict

class GFF_feature():
    """
    A GFF feature data class.
    It's only designed to work with a particular dialect of GFF and is by no
    means a general purpose GFF parser.
    """
    def __init__(self, **kwargs):
        self.id = kwargs['feat_id']
        for key in ('feature_class', 'start', 'end',
                    'strand', 'parent', 'source'):
                self.__setattr__(key, kwargs[key])

    def __str__(self):
        pass

    def get_id_prefix(self):
        """
        Return the first ':' or '-'-separated element of the ID. If neither
        char is present, return ID
        :return:
        """
        return self.id.split(':')[0].split('-')[0]

def parse_gff_line(line):
    """
    Create the GFF_feature object from a GFF line
    :param line:
    :return:
    """
    l = line.rstrip().split('\t')
    key_values = l[8].split(';')
    kv_dict = {}
    for x in key_values:
        pair = x.split('=')
        kv_dict[pair[0]] = pair[1]
    return GFF_feature(feat_id=kv_dict['ID'],
                       feature_class=l[2],
                       start=int(l[3]),
                       end=int(l[4]),
                       strand=l[6],
                       parent='Parent' in kv_dict and kv_dict['Parent'] or None,
                       source=l[0])

def convert_single_coord(exons, coord, strand='+'):
    """
    Convert a single coordinate using simplified exon representation
    Exons should be sorted by (source) coordinates, whatever their strand
    :param exons:
    :param coord:
    :return:
    """
    exon_lengths = [x[1] - x[0] + 1 for x in exons]
    # A pretty ugly, but working hack. Not sure if it's not missing a codon,
    # but whatever. I need a general feel anyway.
    if strand == '-':
        c = int(sum(exon_lengths)/3) - coord
    elif strand == '+':
        c = coord
    else:
        raise ValueError("Strand should be either \'+\' or \'-\'")
    x = 0
    for index, step in enumerate(exon_lengths):
        if x + step >= c * 3:
            return exons[index][0] - x + (c - 1) * 3 + 1
        else:
            x += step

def protein_coord_to_gene_coord(features, coord_set):
    """
    Take a set of gene features and a set of coordinates (in aminoacids),
    return these coordinates in nucleotides.

    Considers only 'exon' features for intron processing
    :param features:
    :param coord_set:
    :return:
    """
    exons = []
    for feature in features:
        if feature.feature_class == 'exon':
            exons.append((feature.start, feature.end))
    # Assuming all exons are on the same strand
    strand = feature.strand
    return [tuple((convert_single_coord(exons, x, strand) for x in y))
            for y in coord_set]

def convert_coord_dict(features, coords):
    """
    Same as `protein_coord_to_gene_coord` except that it takes an OrderedDict
    whose keys are coordinate pair and returns the OrderedDict where coordinates
    are converted and keys are preserved
    :param features:
    :param coords:
    :return:
    """
    r = OrderedDict()
    exons = []
    for feature in features:
        if feature.feature_class == 'exon':
            exons.append((feature.start, feature.end))
    strand = feature.strand
    for coord_pair in coords:
        r[(convert_single_coord(exons, coord_pair[0], strand),
           convert_single_coord(exons, coord_pair[1], strand))] =\
                    coords[coord_pair]
    return r

def reduce_coord_list(coord_set, initial_counts, overlap_cutoff):
    """
    Reduce a coordinate set, averaging overlapping coordinates.
    Coordinates are considered overlapping when at least `overlap_cutoff`
    of either pair of coordinates is overlapping with the other.
    :param coord_set:
    :param counts:
    :param overlap_cutoff:
    :return:
    """
    assert len(coord_set) == len(initial_counts)
    r = [coord_set[0]]
    counts = [initial_counts[0]]
    for hsp_index, hsp in enumerate(coord_set[1:]):
        merged = False
        for index, coord in enumerate(r):
            if overlap(coord, hsp):
                l = overlap_len(coord, hsp)
                hsp_len = hsp[1] - hsp[0]
                coord_len = coord[1] - coord[0]
                if l >= overlap_cutoff * hsp_len or \
                        l > overlap_cutoff * coord_len:
                    # Forget the smaller one if the other is at least twice
                    # bigger
                    if hsp_len >= coord_len * 2:
                        r[index] = hsp
                    elif coord_len >= hsp_len * 2:
                        break
                    # Average position if they are ~equal
                    else:
                        r[index] = round((hsp[0] + coord[0]) / 2), \
                                   round((hsp[1] + coord[1]) / 2)
                    counts[index] += initial_counts[hsp_index + 1]
                    merged = True
                    break
        if not merged:
            r.append(hsp)
            counts.append(initial_counts[hsp_index + 1])
    return OrderedDict((r[x], counts[x]) for x in range(len(r)))

def reduce_coords(coord_set, overlap_cutoff=0.8):
    """
    Wrapper around reduce_coord_list that converts BLAST hit data and runs
    a second reduction round
    :param coord_set: A list of lists of coord tuples
    :param overlap_cutoff:
    :return:
    """
    coords = [hsp for hit in coord_set for hsp in hit]
    c = [1 for x in coords]
    r = reduce_coord_list(coords, c, overlap_cutoff)
    # A se
    return reduce_coord_list(tuple(r.keys()), tuple(r.values()), overlap_cutoff)

    # r = list(coord_set[0])
    # counts = [1 for _ in r]
    # # A hit is a collection of HSP coordinate pairs
    # # Non-blast coordinates are treated similarly
    # for hit in coord_set[1:]:
    #     for hsp in hit:
    #         merged = False
    #         for index, coord in enumerate(r):
    #             if overlap(coord, hsp):
    #                 l = overlap_len(coord, hsp)
    #                 hsp_len = hsp[1] - hsp[0]
    #                 coord_len = coord[1] - coord[0]
    #                 if l >= overlap_cutoff * hsp_len or \
    #                         l > overlap_cutoff * coord_len:
    #                     # Forget the smaller one if the other is at least twice
    #                     # bigger
    #                     if hsp_len >= coord_len * 2:
    #                         r[index] = hsp
    #                     elif coord_len >= hsp_len * 2:
    #                         break
    #                     # Average position if they are ~equal
    #                     else:
    #                         r[index] = round((hsp[0] + coord[0]) / 2), \
    #                                    round((hsp[1] + coord[1]) / 2)
    #                     counts[index] += 1
    #                     merged = True
    #                     break
    #         if not merged:
    #             r.append(hsp)
    #             counts.append(1)
    # return {r[x]: counts[x] for x in range(len(r))}


import cv2
import numpy as np

def open_camera():
    cap = cv2.VideoCapture(0)
    while True:
        _, frame = cap.read()
        cv2.imshow('frame', frame)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
    cv2.destroyAllWindows()

open_camera()


from scholarly import scholarly
import urllib.request

def knowProf(query):
    search_query = scholarly.search_author(query)
    author = next(search_query)
    data = scholarly.fill(author, sections=[])
    fields = ['url_picture', 'name', 'affiliation', 'interests', 'homepage']

    urllib.request.urlretrieve(data[fields[0]], "./tmp/prof.jpg")

    with open("./tmp/prof.log", "w") as output:
        for i in range(1, len(fields)):
            if fields[i] in data.keys():
                line = f"{fields[i]}: {data[fields[i]]}\n"

            output.writelines(line)

def researchField(keyword):
    search_query = scholarly.search_keyword(keyword)

    # for i in range(10):
    data = scholarly.pprint(next(search_query))
    print(data)
        # print()


"""Script de précorrection pour l'évaluation à mi-semestre du S1 2019.

TOUS LES RENDUS ONT SUBI UNE CORRECTION MANUELLE, CE SCRIPT N'EST QU'UNE AIDE, NE PAS S'EN SERVIR
SEUL POUR ESTIMER UNE NOTE.
"""

import ast
import math
import os
import pathlib
import shutil
import subprocess
import tempfile
import typing as ty

import tqdm

data_source = pathlib.Path("AI08.txt")

def extract_answers(answer_script: os.PathLike):
    with tempfile.TemporaryDirectory() as tempdir:
        temp_answer = shutil.copy(answer_script, tempdir)
        shutil.copy(data_source, tempdir)
        answers_proc: ty.Union[subprocess.TimeoutExpired, subprocess.CompletedProcess]
        try:
            answers_proc = subprocess.run(
                ["python", temp_answer],
                capture_output=True,
                text=True,
                cwd=tempdir,
                timeout=10,
            )
            run_returned = not answers_proc.returncode
        except subprocess.TimeoutExpired as e:
            answers_proc = e
            run_returned = False
    if answers_proc.stdout is None:
        answers = [""]
    else:
        answers = [l.strip() for l in answers_proc.stdout.splitlines()]
    return run_returned, answers, answers_proc

class AnswerValidation(ty.NamedTuple):
    validable: bool
    correct: ty.Optional[float]
    answer: ty.Optional[str]
    canonical: str

def validate_answer1(answer):
    """Quantité d'ammoniac (en t/an) émise en Alsace en 2004"""
    canonical = "180.13"
    if answer is None:
        return AnswerValidation(
            validable=False, correct=0.0, answer=None, canonical=canonical
        )
    if answer.isspace():
        return AnswerValidation(
            validable=True, correct=0.0, answer=None, canonical=canonical
        )
    try:
        val = ast.literal_eval(answer)
        validable = isinstance(val, float)
    except SyntaxError:
        validable = False
    if validable:
        correct = 1.0 if math.isclose(val, 180.13) else 0.0
    else:
        correct = None
    return AnswerValidation(
        validable=validable, correct=correct, answer=answer, canonical=canonical
    )

def validate_answer2(answer):
    """Combien de noms de régions mentionnées dans ce fichier comportent un nombre de
    lettres supérieur ou égal à 10 (espaces et tirets non compris) ?"""
    canonical = "13"
    if answer is None:
        return AnswerValidation(
            validable=False, correct=0.0, answer=None, canonical=canonical
        )
    if answer.isspace():
        return AnswerValidation(
            validable=True, correct=0.0, answer=None, canonical=canonical
        )
    try:
        val = ast.literal_eval(answer)
        validable = isinstance(val, int)
    except SyntaxError:
        validable = False
    if validable:
        correct = 1.0 if val == 13 else 0.0
    else:
        correct = None
    return AnswerValidation(
        validable=validable, correct=correct, answer=answer, canonical=canonical
    )

def validate_answer3(answer):
    """Quelle est la région pour laquelle les émissions de monoxyde de carbone on été
    maximales en 2008 ?"""
    canonical = "LORRAINE"
    if answer is None:
        return AnswerValidation(
            validable=False, correct=0.0, answer=None, canonical=canonical
        )
    return AnswerValidation(
        validable=True,
        correct=1.0 if answer.lower() == canonical.lower() else 0.0,
        answer=answer,
        canonical=canonical,
    )

def validate_answer4(answer):
    """Même question que 3., mais pour le total des émissions de monoxyde de carbone
    sur l'ensemble de la période 1999 — 2009"""
    canonical = "AQUITAINE"
    if answer is None:
        return AnswerValidation(
            validable=False, correct=0.0, answer=None, canonical=canonical
        )
    return AnswerValidation(
        validable=True,
        correct=1.0 if answer.lower() == canonical.lower() else 0.0,
        answer=answer,
        canonical=canonical,
    )

def validate_answer5(answer):
    """Donner le contenu de la précision donnée entre parenthèses la plus longue du
    fichier."""
    canonical = "et ses composants"
    if answer is None:
        return AnswerValidation(
            validable=False, correct=0.0, answer=None, canonical=canonical
        )
    accepted = ("et ses composants", "NOX y compris NO2")
    if answer in accepted:
        correct = 1.0
    elif answer[1:-1] in accepted:
        correct = 0.9
    else:
        correct = 0.0
    return AnswerValidation(
        validable=True, correct=correct, answer=answer, canonical=canonical
    )

def validate_answer6(answer):
    """Donner, sous forme de liste séparée par des barres `|` et triée par ordre
    alphabétique l'ensemble des noms de composés chimiques ou de catégories de
    composés mentionnés dans ce fichier."""
    canonical = "1,2-dichloroéthane|acide cyanhydrique|ammoniac|arsenic|benzène|cadmium|chloroforme|chlorure d'hydrogène|chlorure de vinyle monomère|composés organiques volatils non méthaniques|dichlorométhane|dioxyde de carbone|fluorure d'hydrogène|mercure|monoxyde de carbone|oxyde d'azote|oxydes de soufre|perchloroéthylène|plomb|poussières|protoxyde d'azote|sulfure d'hydrogène|trichloroéthylène"
    if answer is None:
        return AnswerValidation(
            validable=False, correct=0.0, answer=None, canonical=canonical
        )
    if answer.isspace():
        return AnswerValidation(
            validable=True, correct=0.0, answer=None, canonical=canonical
        )
    parsed = answer.split("|")
    validable = len(parsed) > 1 and parsed == sorted(parsed)
    if validable:
        if answer == canonical:
            correct = 1.0
        # Facile à louper et pas important: virer les parenthèses peut laisser des double espaces
        elif answer.replace("  ", " ") == canonical:
            correct = 1.0
        else:
            correct = 0.0
    else:
        correct = None
    return AnswerValidation(
        validable=validable, correct=correct, answer=answer, canonical=canonical
    )

def validate_answer7(answer):
    """Donner, sous forme de liste séparée par des barres `|` et triée par ordre
    alphabétique l'ensemble des noms de régions présents dans ce fichier en
    capitalisant les débuts de mots"""
    canonical = "Alsace|Aquitaine|Auvergne|Basse Normandie|Bourgogne|Bretagne|Centre-Val De Loire|Champagne-Ardenne|Corse|D.O.M.|Franche Comte|Haute Normandie|Ile De France|Languedoc-Roussillon|Limousin|Lorraine|Midi-Pyrenees|Nord-Pas-De-Calais|Pays De La Loire|Picardie|Poitou-Charentes|Provence-Alpes-Cote D Azur|Rhone-Alpes"
    if answer is None:
        return AnswerValidation(
            validable=False, correct=0.0, answer=None, canonical=canonical
        )

    if answer.isspace():
        return AnswerValidation(
            validable=True, correct=0.0, answer=None, canonical=canonical
        )
    parsed = answer.split("|")
    validable = len(parsed) > 1 and parsed == sorted(parsed)
    if validable:
        correct = 1.0 if answer == canonical else 0.0
    else:
        correct = None
    return AnswerValidation(
        validable=validable, correct=correct, answer=answer, canonical=canonical
    )

def validate_answer8(answer):
    """Quelle est le composé dont pour lequel la moyenne annuelle des émissions en
    région Basse Normandie a été la plus élevée sur la période 1999–2009 ?"""
    canonical = "Dioxyde de carbone (CO2)"
    if answer is None:
        return AnswerValidation(
            validable=False, correct=0.0, answer=None, canonical=canonical
        )
    return AnswerValidation(
        validable=True,
        correct=1.0 if answer == canonical else 0.0,
        answer=answer,
        canonical=canonical,
    )

def autocorrect(
    answers: ty.Sequence[ty.Optional[str]]
) -> ty.Generator[AnswerValidation, None, None]:
    yield validate_answer1(answers[0])
    yield validate_answer2(answers[1])
    yield validate_answer3(answers[2])
    yield validate_answer4(answers[3])
    yield validate_answer5(answers[4])
    yield validate_answer6(answers[5])
    yield validate_answer7(answers[6])
    yield validate_answer8(answers[7])

results_file = pathlib.Path("results.tsv")
answers_dir = pathlib.Path("answers")
debugged_answers_dir = pathlib.Path("answers_dbug")
outputs_dir = pathlib.Path("outputs")
outputs_dir.mkdir(exist_ok=True)

with open(results_file, "w") as out_stream:
    # Header
    out_stream.write("name\treturned\tfull\t")
    out_stream.write("\t".join(f"q{i}\t" for i in range(1, 9)))
    out_stream.write("\n")
    pbar = tqdm.tqdm(list(answers_dir.glob("*.py")), unit="file")
    for answer_script in pbar:
        pbar.set_description(f"validating {answer_script.name}")
        full = True
        run_returned, answers, proc = extract_answers(answer_script)
        if not run_returned:
            _, answers, proc = extract_answers(
                debugged_answers_dir / answer_script.name
            )
            answers = [None if a == "DEBUGGED" else a for a in answers]
        if len(answers) < 8:
            answers = [*answers, *("" for _ in range(8 - len(answers)))]
            full = False
        elif len(answers) > 8:
            answers = answers[:8]
            full = False
        out_stream.write(f"{answer_script.stem}\t{int(run_returned)}\t{int(full)}\t")
        for validation in autocorrect(answers):
            out_stream.write(f"{int(validation.validable)}\t{validation.correct}\t")
        out_stream.write("\n")
        with open(
            outputs_dir / f"{answer_script.stem}.txt", "w"
        ) as script_output_stream:
            script_output_stream.write("========== stderr ===========\n")
            script_output_stream.write(proc.stderr if proc.stderr is not None else "")
            script_output_stream.write("\n========= stdout ===========\n")
            script_output_stream.write(proc.stdout if proc.stdout is not None else "")


# import sys
# sys.path.remove('/opt/ros/kinetic/lib/python2.7/dist-packages')
import os
import sys
sys.path.insert(0, './')
import gym
import rospy

from preddrl_env.environment_stage_3_bk import Env

from policy.td3 import TD3

from trainer import Trainer
print(os.getcwd())

# from tf2rl.algos.policy_base import OffPolicyAgent

if __name__ == '__main__':

    parser = Trainer.get_argument()

    parser = TD3.get_argument(parser)

    parser.set_defaults(batch_size=100)
    parser.set_defaults(n_warmup=3000) # 重新训练的话要改回 10000
    parser.set_defaults(max_steps=100000)
    parser.set_defaults(restore_checkpoint=False)
    parser.set_defaults(prefix='tf')
    args = parser.parse_args()
    print(vars(args))

    print(args.evaluate)
    # test param, modified by niraj
    if args.evaluate:
        args.test_episodes=50
        args.episode_max_steps = int(1e4)
        args.model_dir = './preddrl_td3/results/compare_network/1conv_2dnn_3input_dropout_1'
        args.show_test_progress=False
        args.save_model_interval = int(1e10)
        args.restore_checkpoint = True

        # parser.set_defaults(test_episodes=50)
        # parser.set_defaults(episode_max_steps=int(1e4))
        # parser.set_defaults(model_dir='./preddrl_td3/results/compare_network/1conv_2dnn_3input_dropout_1')
        # parser.set_defaults(show_test_progress=False)
        # parser.set_defaults(save_model_interval=int(1e10))

    rospy.init_node('turtlebot3_td3_stage_3', disable_signals=True)

    env = Env()
    test_env = Env()

    policy = TD3(
        state_shape=env.observation_space.shape,
        action_dim=env.action_space.high.size,
        gpu=0,
        memory_capacity=args.memory_capacity,
        max_action=env.action_space.high[0],
        batch_size=args.batch_size,
        actor_units=[400, 300],
        n_warmup=args.n_warmup)

    # print('offpolicy:', issubclass(type(policy), OffPolicyAgent))

    trainer = Trainer(policy, env, args, test_env=test_env)

    try:
        if args.evaluate:
            print('Evaluating policy ...')
            trainer.evaluate_policy(10000)  # 每次测试都会在生成临时文件，要定期处理

        else:
            print('Training policy ...')
            trainer()

    except KeyboardInterrupt: # this is to prevent from accidental ctrl + c
        sys.exit()
        print('-' * 89)
        print('Exiting from training early because of KeyboardInterrupt')


from flask import Flask, render_template, request
import datetime as dt
import requests

def kelvin_to_celsius(kelvin):
    return kelvin - 273.15

BASE_URL = "https://api.openweathermap.org/data/2.5/weather?"
API_KEY = open('api_key', 'r').read()

Flask_App = Flask(__name__)  # Creating our Flask Instance

@Flask_App.route('/', methods=['GET'])
def index():
    """ Displays the index page accessible at '/' """

    return render_template('index.html')

@Flask_App.route('/check_weather/', methods=['POST'])
def check_weather():
    """Route where we send calculator form input"""

    error = None
    result = None

    # request.form looks for:
    # html tags with matching "name= "
    city_input = request.form['city']

    try:
        CITY = city_input

        url = BASE_URL + "&q=" + CITY + "&appid=" + API_KEY

        response = requests.get(url).json()
        temp_kelvin = response['main']['temp']
        temp_celsius = round(kelvin_to_celsius(temp_kelvin), 3)

        result = temp_celsius

        return render_template(
            'index.html',
            city=city_input,
            result=result,
            is_success=True
        )

    except ValueError:
        return render_template(
            'index.html',
            result="Bad Input",
            is_success=False,
            error="Blad"
        )

if __name__ == '__main__':
    Flask_App.debug = True
    Flask_App.run()


import torch
from torch.utils.data.dataset import Dataset
import os
import numpy as np
from torch.utils.data import DataLoader
import cv2

import math
import random
from threading import Thread

from utils.utils import xywhn2xyxy
from utils.augmentations import random_perspective, augment_hsv, letterbox, DEFAULT_TRANSFORMS

IMG_FORMATS = 'bmp', 'dng', 'jpeg', 'jpg', 'mpo', 'png', 'tif', 'tiff', 'webp', 'pfm'  # include image suffixes
VID_FORMATS = 'asf', 'avi', 'gif', 'm4v', 'mkv', 'mov', 'mp4', 'mpeg', 'mpg', 'ts', 'wmv'  # include video suffixes

class CenterNetDataset(Dataset):
    def __init__(self, data_root, img_size=512, isTrain=True, augment=False):
        self.augment = augment
        self.image_size = img_size
        self.mosaic_border = [-img_size // 2, -img_size // 2]
        if isTrain:
            self.images_root = os.path.join(data_root, "train", "images")
        else:
            self.images_root = os.path.join(data_root, "val", "images")
        self.image_paths = [os.path.join(self.images_root, image_name) for image_name in os.listdir(self.images_root)]
        self.image_paths.sort()
        self.annotations_paths = self._img2label_paths(self.image_paths)
        self.indices = range(len(self.image_paths))  # 每张图片的索引
        self.labels = self._load_labels()  # 一次性将所有 labels 加载进内存

    def _img2label_paths(self, image_paths):
        annotations_paths = []
        for image_path in image_paths:
            image_name = os.path.split(image_path)[-1]
            label_name = os.path.splitext(image_name)[0] + '.txt'
            annotations_paths.append(os.path.join(os.path.dirname(self.images_root), 'labels', label_name))
        return annotations_paths

    def _load_labels(self):
        labels = []
        for path in self.annotations_paths:
            labels.append(np.loadtxt(path, dtype=np.float32).reshape(-1, 5))
        return labels

    def __getitem__(self, index):
        """
        image(Tensor): shape[3, 512, 512]
        targets:(Tensor[N, 6]): [batch_index, class_label, x1, y1, x2, y2]
        """
        if self.augment:
            img, labels = self.load_mosaic(index)
        else:
            img, (h0, w0), (h, w) = self.load_image(index)
            shape = self.image_size
            img, ratio, pad = letterbox(img, shape, auto=False, scaleup=self.augment)

            labels = self.labels[index].copy()
            if labels.size:  # normalized xywh to pixel xyxy format
                labels[:, 1:] = xywhn2xyxy(labels[:, 1:], ratio[0] * w, ratio[1] * h, padw=pad[0], padh=pad[1])

        # Albumentations
        if self.augment:
            new = DEFAULT_TRANSFORMS(image=img, bboxes=labels[:, 1:], class_labels=labels[:, 0])
            img, labels = new["image"], np.array([[c, *b] for c, b in zip(new['class_labels'], new['bboxes'])])

            # HSV color-space
            augment_hsv(img, hgain=0.1, sgain=0.7, vgain=0.4)

        # 第一个位置放当前图片在 batch 中的索引(在collate_fn中实现)
        nl = len(labels)
        labels_out = torch.zeros((nl, 6))
        if nl:
            labels_out[:, 1:] = torch.from_numpy(labels)

        img = img.transpose((2, 0, 1))[::-1]  # HWC to CHW, BGR to RGB
        img = np.ascontiguousarray(img)

        return torch.from_numpy(img), labels_out

    def __len__(self):
        return len(self.image_paths)

    def load_mosaic(self, index):
        """
        mosaic4 数据增强, 读取 4 张图片进行拼接, 并进行 random_perspective 数据增强
        """
        labels4 = []
        s = self.image_size
        yc, xc = (int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border)
        indices = [index] + random.choices(self.indices, k=3)  # 再随机取 3 张图片
        random.shuffle(indices)
        for i, index in enumerate(indices):
            img, _, (h, w) = self.load_image(index)

            # place img in img4
            if i == 0:  # top left
                img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8)  # base image with 4 tiles
                x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc  # xmin, ymin, xmax, ymax (large image)
                x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h  # xmin, ymin, xmax, ymax (small image)
            elif i == 1:  # top right
                x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
                x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h
            elif i == 2:  # bottom left
                x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
                x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, w, min(y2a - y1a, h)
            elif i == 3:  # bottom right
                x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
                x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)

            img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b]  # img4[ymin:ymax, xmin:xmax]
            padw = x1a - x1b
            padh = y1a - y1b

            labels = self.labels[index].copy()
            if labels.size:
                labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padw, padh)
            labels4.append(labels)

        labels4 = np.concatenate(labels4, 0)
        x = labels4[:, 1:]
        np.clip(x, 0, 2 * s, out=x)

        img4, labels4 = random_perspective(img4,
                                           labels4,
                                           degrees=0.0,
                                           translate=0.1,
                                           scale=0.5,
                                           shear=0.0,
                                           perspective=0.0,
                                           border=self.mosaic_border)

        return img4, labels4

    def load_image(self, i):
        """
        读取一张图片, 并将其长变 resize 到 self.image_size(保持长宽比不变)
        """
        im = cv2.imread(self.image_paths[i])
        assert im is not None, f"Image Not Found {self.image_paths[i]}"
        h0, w0 = im.shape[:2]
        r = self.image_size / max(h0, w0)
        if r != 1:
            im = cv2.resize(im, (math.ceil(w0 * r), math.ceil(h0 * r)), interpolation=cv2.INTER_LINEAR)
        return im, (h0, w0), im.shape[:2]

    def collate_fn(self, batch):
        imgs, targets = list(zip(*batch))
        imgs = torch.stack(imgs)

        # add sample index to targets
        # to identify which image the boxes belong to
        for i, boxes in enumerate(targets):
            boxes[:, 0] = i
        targets = torch.cat(targets, 0)

        return imgs, targets

class LoadStream:
    """
    网络/摄像头 dataloader
    """
    def __init__(self, source='file.stream', img_size=512, vid_stride=1):
        torch.backends.cudnn.benchmark = True
        self.mode = 'stream'
        self.img_size = img_size
        self.vid_stride = vid_stride
        self.current_img, self.fps, self.frames, self.thread = None, 0, 0, None
        self.source = eval(source) if source.isnumeric() else source

        cap = cv2.VideoCapture(self.source)
        assert cap.isOpened(), f"Failed to open {self.source}"
        w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
        h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
        fps = cap.get(cv2.CAP_PROP_FPS)
        self.frames = max(int(cap.get(cv2.CAP_PROP_FRAME_COUNT)), 0) or float('inf')
        self.fps = max((fps if math.isfinite(fps) else 0) % 100, 0) or 30
        _, self.current_img = cap.read()
        self.thread = Thread(target=self.update, args=[cap, self.source], daemon=True)
        self.thread.start()

    def update(self, cap, stream):
        """
        不断从 stream 中读取下一帧, 每隔 self.vid_stride 帧将其存入 self.current_img 中
        保证当前存的是摄像头中最新的图片
        """
        n = 0
        while cap.isOpened() and n < self.frames:
            n += 1
            cap.grab()
            if n % self.vid_stride == 0:
                success, im = cap.retrieve()
                if success:
                    self.current_img = im
                else:
                    print('WARNING Video stream unresponsive, please check your IP camera connection.')
                    self.current_img = np.zeros_like(self.current_img)
                    cap.open(stream)

    def __iter__(self):
        self.count = -1
        return self

    def __next__(self):
        self.count += 1
        if not self.thread.is_alive() or cv2.waitKey(1) == ord('q'):
            cv2.destroyAllWindows()
            raise StopIteration

        im0 = self.current_img.copy()
        im = letterbox(im0, self.img_size, auto=False)[0]
        im = im.transpose((2, 0, 1))[::-1]
        im = np.ascontiguousarray(im)

        return self.source, im, im0

class LoadVideo:
    """
    video dataloader
    """
    def __init__(self, path, img_size=512, vid_stride=1):
        assert(os.path.isfile(path)), f"path: {path} is not an exist file"
        assert(path.split('.')[-1].lower() in VID_FORMATS)

        self.path = path
        self.img_size = img_size
        self.vid_stride = vid_stride
        self.cap = cv2.VideoCapture(path)
        self.frame = 0
        self.frames = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT) / self.vid_stride)

    def __iter__(self):
        self.count = 0
        return self

    def __next__(self):
        for _ in range(self.vid_stride):
            self.cap.grab()
        ret_val, im0 = self.cap.retrieve()
        while not ret_val:
            self.count += 1
            self.cap.release()
            raise StopIteration
        self.frame += 1

        im = letterbox(im0, self.img_size, auto=False)[0]
        im = im.transpose((2, 0, 1))[::-1]
        im = np.ascontiguousarray(im)

        return self.path, im, im0

if __name__ == '__main__':
    import torchvision
    import matplotlib.pyplot as plt
    from draw_boxes_utils import draw_objs
    data = CenterNetDataset('./DroneVsBirds', isTrain=True, augment=False)
    dataloader = DataLoader(data, 4, False, num_workers=4, collate_fn=data.collate_fn)

    for img, target in data:
        target = target.numpy()
        print(target)
        img = torchvision.transforms.ToPILImage()(img)
        img = draw_objs(img, target[:, 2:], np.ones_like(target[:, 1]))
        plt.imshow(img)
        plt.show()


import yfinance as yf
import pandas as pd

def download_data(ticker):
    # Download historical data as a DataFrame
    df = yf.download(ticker,'2015-01-01','2023-04-30')

    # Select relevant columns
    df = df[['Open', 'High', 'Low', 'Close', 'Volume']]

    # Save to a CSV file
    df.to_csv(f"{ticker}.csv")

# List of stock tickers you are interested in
tickers = ['AAPL', 'GOOGL', 'MSFT', 'AMZN', 'DVA']

# Download data for each stock
for ticker in tickers:
    download_data(ticker)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
import datetime
from django.utils.timezone import utc

class Migration(migrations.Migration):

    dependencies = [
        ('recursos', '0004_recurso_averiado'),
    ]

    operations = [
        migrations.AddField(
            model_name='recurso',
            name='creado',
            field=models.DateTimeField(default=datetime.datetime(2017, 5, 20, 10, 21, 1, 238467, tzinfo=utc), auto_now_add=True),
            preserve_default=False,
        ),
        migrations.AddField(
            model_name='recurso',
            name='mantenimiento_preventivo',
            field=models.IntegerField(default=30, verbose_name=b'mantenimiento preventivo (dias)'),
        ),
    ]


import copy
from abc import ABC, abstractmethod
from functools import partial
from typing import Callable, Optional, Union
import warnings

import jax
import jax.numpy as jnp
import numpy as np
import numpyro
import pandas
import tqdm
from jax.random import PRNGKey
from numpyro import deterministic, infer, optim
from numpyro.handlers import condition, trace
from numpyro.infer import SVI, Trace_ELBO
from numpyro.infer.svi import SVIRunResult
from pysersic.exceptions import ShapeMatchError,KernelError
from pysersic.priors import PySersicMultiPrior, PySersicSourcePrior, base_profile_params
from pysersic.rendering import BaseRenderer, HybridRenderer
from pysersic.results import PySersicResults

from .loss import gaussian_loss

ArrayLike = Union[np.array, jax.numpy.array]

class BaseFitter(ABC):
    """
    Base class for Pysersic Fitters
    """
    def __init__(self,
        data: ArrayLike,
        rms: ArrayLike,
        psf: ArrayLike,
        mask: Optional[ArrayLike] = None,
        loss_func: Optional[Callable] = gaussian_loss,
        renderer: Optional[BaseRenderer] =  HybridRenderer,
        renderer_kwargs: Optional[dict] = {}) -> None:
        """Initialze BaseFitter class

        Parameters
        ----------
        data : ArrayLike
            Science image to be fit
        weight_map : ArrayLike
            Weight map (one over the variance) corresponding to `data`, must be the same shape
        psf_map : ArrayLike
            Pixelized PSF
        mask : Optional[ArrayLike], optional
            Array specifying the mask, `True` or 1 signifies a pixel should be masked, must be same shape as `data`
        sky_model : Optional[str], optional
            One of None, 'flat' or 'tilted-plane' specifying how to model the sky background
        renderer : Optional[BaseRenderer], optional
            The renderer to be used to generate model images, by default HybridRenderer
        renderer_kwargs : Optional[dict], optional
            Any additional arguments to pass to the renderer, by default {}
        """

        self.loss_func = loss_func

        self.data = jnp.array(data.astype(np.float32))
        self.rms = jnp.array(rms.astype(np.float32))
        self.psf = jnp.array(psf.astype(np.float32))
        self.mask = parse_mask(mask,self.data)
        data_isgood = check_input_data(self.data,rms=self.rms,psf=self.psf,mask=jnp.logical_not(self.mask))
        self.renderer = renderer(data.shape, self.psf, **renderer_kwargs)
        self.prior = None
        self.prior_dict = {}

    def set_loss_func(self, loss_func: Callable) -> None:
        """Set loss function to be used for inference

        Parameters
        ----------
        loss_func : Callable
            Functions which takes samples the loss function, see utils/loss.py for some examples.
        """
        self.loss_func = loss_func

    def set_prior(self,parameter: str,
        distribution: numpyro.distributions.Distribution) -> None:
        """Set the prior for a specific parameter

        Parameters
        ----------
        parameter : str
            Parameter to be set
        distribution : numpyro.distributions.Distribution
            Numpyro distribution object corresponding to the prior
        """
        self.prior_dict[parameter] = distribution

    def sample(self,
                num_samples: int = 1000,
                num_warmup: int = 1000,
                num_chains: int = 2,
                init_strategy: Optional[Callable] = infer.init_to_sample,
                sampler_kwargs: Optional[dict] ={},
                mcmc_kwargs: Optional[dict] = {},
                return_model: Optional[bool] = True,
                reparam_func: Optional[Callable] = lambda x: x,
                rkey: Optional[jax.random.PRNGKey] = jax.random.PRNGKey(3)
        ) -> pandas.DataFrame:
        """ Perform inference using a NUTS sampler

        Parameters
        ----------
        num_samples : int, optional
            Number of samples to draw, by default 1000
        num_warmup : int, optional
            Number of warmup samples, by default 1000
        num_chains : int, optional
            Number of chains to run, by default 2
        init_strategy : Optional[Callable], optional
            Initialization strategy for the sampler, by default infer.init_to_sample. See numpyro.infer.initialization for more options
        sampler_kwargs : Optional[dict], optional
            Arguments to pass to the numpyro NUTS kernel
        mcmc_kwargs : Optional[dict], optional
            Arguments to pass to the numpyro MCMC sampler
        return_model : Optional[bool]
            Whether to return the model images but adds a small memory/time overhead, by default True
        rkey : Optional[jax.random.PRNGKey], optional
            PRNG key to use, by default jax.random.PRNGKey(3)

        Returns
        -------
        pandas.DataFrame
            ArviZ summary of posterior
        """

        model =  self.build_model(return_model = return_model)
        model = reparam_func(model)
        sampler =infer.MCMC(infer.NUTS(model,init_strategy=init_strategy, **sampler_kwargs),num_chains=num_chains, num_samples=num_samples, num_warmup=num_warmup,  **mcmc_kwargs)
        sampler.run(rkey)
        self.sampling_results = PySersicResults(data=self.data,rms=self.rms,psf=self.psf,mask=self.mask,loss_func=self.loss_func,renderer=self.renderer)
        self.sampling_results.add_prior(self.prior)
        self.sampling_results.injest_data(sampler = sampler)
        return self.sampling_results

    def _train_SVI(self,
            autoguide: numpyro.infer.autoguide.AutoContinuous,
            method:str,
            ELBO_loss: Optional[Callable] = infer.Trace_ELBO(1),
            lr_init: Optional[int] = 1e-2,
            num_round: Optional[int] = 3,
            SVI_kwargs: Optional[dict]= {},
            train_kwargs: Optional[dict] = {},
            return_model: Optional[bool] = True,
            num_sample: Optional[int] = 1_000,
            rkey: Optional[jax.random.PRNGKey] = jax.random.PRNGKey(6),
            )-> pandas.DataFrame:
        """
        Internal function to perform inference using stochastic variational inference.

        Parameters
        ----------
        autoguide : numpyro.infer.autoguide.AutoContinuous
            Function to build guide
        method: str
            name of method being used; for saving results
        Elbo_loss : Optional[Callable], optional
            Loss function to use, by default infer.Trace_ELBO(1), see numpyro.infer.elbo for more options
        lr_init : Optional[int], optional
            Initial learning rate, by default 1e-2
        num_round : Optional[int], optional
            Number of rounds for training, lr decreases each round, by default 3
        SVI_kwargs : Optional[dict], optional
            Additional arguments to pass to numpyro.infer.SVI, by default {}
        train_kwargs : Optional[dict], optional
            Additional arguments to pass to utils.train_numpyro_svi_early_stop, by default {}
        return_model : Optional[bool]
            Whether to return the model images but adds a small memory/time overhead, by default True
        num_sample: Optional[int]
            Number of samples to draw from trained SVI posterior
        rkey : Optional[jax.random.PRNGKey], optional
            PRNG key, by default jax.random.PRNGKey(6)

        Returns
        -------
        pandas.DataFrame
            ArviZ summary of posterior
        """
        model_cur = self.build_model(return_model=return_model)
        guide = autoguide(model_cur)

        svi_kernel = SVI(model_cur,guide, optim.Adam(0.01), loss = ELBO_loss, **SVI_kwargs)
        numpyro_svi_result = train_numpyro_svi_early_stop(svi_kernel,rkey=rkey,lr_init=lr_init,
                                                        num_round=num_round, **train_kwargs)

        svi_res_dict =  dict(guide = guide, model = model_cur, svi_result = numpyro_svi_result)
        self.svi_results = PySersicResults(data=self.data,rms=self.rms,psf=self.psf,mask=self.mask,loss_func=self.loss_func,renderer=self.renderer)
        self.svi_results.injest_data(svi_res_dict=svi_res_dict,purge_extra=True, num_sample = num_sample)
        self.svi_results.add_prior(self.prior)
        self.svi_results.add_method_used(method)
        return self.svi_results

    def find_MAP(self,
                return_model: Optional[bool] = True,
                rkey: Optional[jax.random.PRNGKey] = jax.random.PRNGKey(3),
                purge_extra: Optional[bool] = True):
        """Find the "best-fit" parameters as the maximum a-posteriori and return a dictionary with values for the parameters.

        Parameters
        ----------
        return_model : Optional[bool]
            Whether to return the model images but adds a small memory/time overhead, by default True
        rkey : Optional[jax.random.PRNGKey], optional
            rng key, by default jax.random.PRNGKey(3)

        Returns
        -------
        dict
            dictionary with fit parameters and their values.
        """
        model_cur = self.build_model(return_model=return_model)
        autoguide_map = infer.autoguide.AutoDelta(model_cur, init_loc_fn= infer.init_to_median)
        train_kwargs = dict(lr_init = 0.05, num_round = 3,frac_lr_decrease  = 0.1, patience = 250, optimizer = optim.Adam, max_train = int(2e4))
        svi_kernel = SVI(model_cur,autoguide_map, optim.Adam(0.01),loss=Trace_ELBO())

        res = train_numpyro_svi_early_stop(svi_kernel,rkey=rkey, **train_kwargs)

        use_dict = {}
        for key in res.params.keys():
            pref = key.split('_auto_loc')[0]
            use_dict[pref] = res.params[key]
        trace_out = trace(condition(model_cur, use_dict)).get_trace()
        real_out = {}

        if purge_extra:
            for key in trace_out:
                if 'Loss' in key:
                    continue
                elif key == 'model':
                    real_out[key] = np.asarray(trace_out[key]['value'])
                elif not ('base' in key or 'auto' in key or 'unwrapped' in key or 'factor' in key or 'loss' in key):
                    real_out[key] = np.round(trace_out[key]['value'], 5 )
        else:
            for k,v in trace_out.items():
                real_out[k] = v['value']

        return real_out

    def estimate_posterior(self,
                        method : str='laplace',
                        return_model: bool = True,
                        num_sample: Optional[int] = 1_000,
                        rkey: Optional[jax.random.PRNGKey] = jax.random.PRNGKey(6),
                        ) -> pandas.DataFrame:
        """Estimate the posterior using a method other than MCMC sampling. Generally faster than MCMC, but could be less accurate.
        Current Options are:
        - 'laplace'
            - Uses the Laplace approximation, which finds the MAP and then uses a Gaussian approximation to the posterior. The covariance matrix is calculated using the Hessian of the log posterior at the MAP. Generally the fastest method but can lead to numerical problems, especially for fitting several (~> 5) sources since this involves inverting a large matrix.
        - 'svi-mvn'
            - Use variational inference to fit a multivariate normal distribution to the posterior. In practice should give similar results to 'laplace', as both assume the posterior is a multivariate Gaussian but is trained differently and generally less susceptible to the numerical issues discussed above.
        - 'svi-flow'
            - Uses a neural flow (currently a BNAF, https://arxiv.org/abs/1904.04676) to approximate the posterior. This is more flexible than the two methods above as the flow is a more flexible representation of the posterior which can capture non-Gaussian behavior.However it is slower to train. Also optimization can be inconsistent so use and interpret with caution. Best to cross-reference with sample on tests cases.

        Parameters
        ----------
        method : str, optional
            method to use, by default 'laplace'
        return_model : Optional[bool]
            Whether to return the model images but adds a small memory/time overhead, by default True
                num_sample: Optional[int]
        Number of samples to draw from trained SVI posterior
            Number of samples to draw from trained SVI posterior
        rkey : Optional[jax.random.PRNGKey], optional
            rng key, by default jax.random.PRNGKey(6)

        """
        assert method in ['laplace','svi-flow','svi-mvn']
        if method=='laplace':
            train_kwargs = dict(patience = 250, max_train = 20000)
            guide_func = partial(infer.autoguide.AutoLaplaceApproximation, init_loc_fn = infer.init_to_median )
            results = self._train_SVI(guide_func,method=method, lr_init = 0.05, train_kwargs=train_kwargs, return_model = return_model, rkey=rkey, num_sample=num_sample)
        elif method=='svi-flow':
            train_kwargs = dict(patience = 500, max_train = 20000)
            guide_func = partial(infer.autoguide.AutoBNAFNormal, num_flows =4,hidden_factors = [5,], init_loc_fn = infer.init_to_median)
            results = self._train_SVI(guide_func,method='svi-flow',ELBO_loss= infer.Trace_ELBO(8),train_kwargs=train_kwargs,num_round=3,lr_init = 1e-2, rkey=rkey,return_model = return_model,num_sample=num_sample)
        elif method=='svi-mvn':
            train_kwargs = dict(patience = 200, max_train = 5000)
            guide_func = partial(infer.autoguide.AutoLowRankMultivariateNormal, init_scale = 5e-3, init_loc_fn = infer.init_to_median)
            results = self._train_SVI(guide_func,method='svi-mvn',ELBO_loss= infer.TraceMeanField_ELBO(16),train_kwargs=train_kwargs,num_round=3,lr_init = 1e-1, rkey=rkey,return_model = return_model,num_sample=num_sample)
        return results

    @abstractmethod
    def build_model(self,return_model: bool = True):
        raise NotImplementedError

class FitSingle(BaseFitter):
    """
    Class used to fit a single source
    """
    def __init__(self,
        data: ArrayLike,
        rms: ArrayLike,
        psf: ArrayLike,
        prior: PySersicSourcePrior,
        mask: Optional[ArrayLike] = None,
        loss_func: Optional[Callable] = gaussian_loss,
        renderer: Optional[BaseRenderer] =  HybridRenderer,
        renderer_kwargs: Optional[dict] = {}) -> None:
        """Initialze FitSingle class

        Parameters
        ----------
        data : ArrayLike
            Science image to be fit
        weight_map : ArrayLike
            Weight map (one over the variance) corresponding to `data`, must be the same shape
        psf_map : ArrayLike
            Pixelized PSF
        mask : Optional[ArrayLike], optional
            Array specifying the mask, `True` or 1 signifies a pixel should be masked, must be same shape as `data`
        sky_model : Optional[str], optional
            One of None, 'flat' or 'tilted-plane' specifying how to model the sky background
        profile_type : Optional[str], optional
            Must be one of: ['sersic','doublesersic','pointsource','exp','dev'] specifying how to paramaterize the source, default 'sersic'
        renderer : Optional[BaseRenderer], optional
            The renderer to be used to generate model images, by default HybridRenderer
        renderer_kwargs : Optional[dict], optional
            Any additional arguments to pass to the renderer, by default {}
        """

        super().__init__(data,rms,psf,loss_func = loss_func, mask = mask, renderer = renderer, renderer_kwargs = renderer_kwargs)

        if prior.profile_type not in self.renderer.profile_types:
            raise AssertionError('Profile must be one of:', self.renderer.profile_types)
        self.profile_type_number = 0
        self.prior = prior

    def build_model(self, return_model: bool = True) -> Callable:
        """ Generate Numpyro model for the specified image, profile and priors

        Returns
        -------
        model: Callable
            Function specifying the current model in Numpyro, can be passed to inference algorithms
        """
        # Make sure all variables have priors
        check_prior = self.prior.check_vars
        if not check_prior:
            raise AssertionError('Not all variables have priors, please run .autogenerate_priors')

        @numpyro.handlers.reparam(config = self.prior.reparam_dict)
        def model(return_model: bool = return_model):
            params = self.prior()
            out = self.renderer.render_source(params, self.prior.profile_type, suffix = self.prior.suffix)

            sky = self.prior.sample_sky(self.renderer.X, self.renderer.Y)

            obs = out + sky
            if return_model:
                deterministic(f'model{self.prior.suffix}', obs)
            self.loss_func(obs, self.data, self.rms, self.mask, suffix = self.prior.suffix)
        return model

class FitMulti(BaseFitter):
    """
    Class used to fit multiple sources within a single image
    """
    def __init__(self,
        data: ArrayLike,
        rms: ArrayLike,
        psf: ArrayLike,
        prior: PySersicMultiPrior,
        mask: Optional[ArrayLike] = None,
        loss_func: Optional[Callable] = gaussian_loss,
        renderer: Optional[BaseRenderer] =  HybridRenderer,
        renderer_kwargs: Optional[dict] = {}) -> None:
        """Initialze FitMulti class

        Parameters
        ----------
        data : ArrayLike
            Science image to be fit
        weight_map : ArrayLike
            Weight map (one over the variance) corresponding to `data`, must be the same shape
        psf_map : ArrayLike
            Pixelized PSF
        mask : Optional[ArrayLike], optional
            Array specifying the mask, `True` or 1 signifies a pixel should be masked, must be same shape as `data`
        sky_model : Optional[str], optional
            One of None, 'flat' or 'tilted-plane' specifying how to model the sky background
        profile_type : Optional[str], optional
            Must be one of: ['sersic','doublesersic','pointsource','exp','dev'] specifying how to paramaterize the source, default 'sersic'
        renderer : Optional[BaseRenderer], optional
            The renderer to be used to generate model images, by default HybridRenderer
        renderer_kwargs : Optional[dict], optional
            Any additional arguments to pass to the renderer, by default {}
        """
        super().__init__(data,rms,psf,mask = mask,loss_func = loss_func,renderer = renderer, renderer_kwargs = renderer_kwargs)
        self.prior = prior

    def build_model(self, return_model: bool = True) -> Callable:
        """Generate Numpyro model for the specified image, profile and priors

        Returns
        -------
        model: Callable
            Function specifying the current model in Numpyro, can be passed to inference algorithms
        """

        @numpyro.handlers.reparam(config = self.prior.reparam_dict)
        def model(return_model: bool = return_model):
            source_variables = self.prior()

            out = self.renderer.render_for_model(source_variables,self.prior.catalog['type'], suffix = self.prior.suffix)
            sky = self.prior.sample_sky(self.renderer.X, self.renderer.Y)

            obs = out + sky

            if return_model:
                deterministic(f'model{self.prior.suffix}', obs)
            self.loss_func(obs, self.data, self.rms, self.mask, suffix = self.prior.suffix)

        return model

    def find_MAP(self,
                return_model: Optional[bool] = True,
                rkey: Optional[jax.random.PRNGKey] = jax.random.PRNGKey(3),
                purge_extra: Optional[bool] = True):
        """Find the "best-fit" parameters as the maximum a-posteriori and return a dictionary with values for the parameters.

        Parameters
        ----------
        return_model : Optional[bool], optional
            whether to return the model image, adds a small time and memory overhead, by default True
        rkey : Optional[jax.random.PRNGKey], optional
            rng key, by default jax.random.PRNGKey(3)

        Returns
        -------
        dict
            dictionary with fit parameters and their values.
        """
        raw_dict = super().find_MAP(return_model=return_model, rkey=rkey, purge_extra= purge_extra)

        if purge_extra:
            results_dict = {}
            for i in range(self.prior.N_sources):
                results_dict[f'source_{i}'] = {}
                for pname in base_profile_params[self.prior.catalog['type'][i]]:
                    results_dict[f'source_{i}'][pname] = raw_dict.pop(pname + f'_{i:d}')
            results_dict.update(raw_dict)
        else:
            results_dict = raw_dict
        return results_dict

def train_numpyro_svi_early_stop(
        svi_class: SVI,
        num_round: Optional[int] = 3,
        max_train: Optional[int] = 5000,
        lr_init: Optional[float] = 0.01,
        frac_lr_decrease: Optional[float]  = 0.1,
        patience: Optional[int] = 100,
        optimizer: Optional[optim._NumPyroOptim] = optim.Adam,
        rkey: Optional[PRNGKey] = PRNGKey(10),
    )-> SVIRunResult:
    """Optimize a SVI model by training for multiple rounds with a deacreasing learning rate, and early stopping for each round

    Parameters
    ----------
    svi_class : SVI
        Initialized numpyo SVI class, note that the optimizer will be overwritten
    num_round : Optional[int], optional
        Number of training rounds, by default 3
    max_train : Optional[int], optional
        Max number of training epochs per ropund, by default 3000
    lr_init : Optional[float], optional
        Initial learning rate, by default 0.1
    frac_lr_decrease : Optional[float], optional
        Multiplicative factor to change learning rate each round, by default 0.1
    patience : Optional[int], optional
        Number of training epochs to wait for improvement, by default 100
    optimizer : Optional[optim._NumPyroOptim], optional
        Optimizer algorithm tro use, by default optim.Adam
    rkey : Optional[PRNGKey], optional
        Jax PRNG key, by default PRNGKey(10)

    Returns
    -------
    SVIRunResult
        SVI Result class containing trained model
    """
    optim_init = optimizer(lr_init)
    svi_class.__setattr__('optim', optim_init)

    init_state = svi_class.init(rkey)
    all_losses = []

    @partial(jax.jit, static_argnums = 1)
    def update_func(state,svi_class,lr):
        svi_class.__setattr__('optim', optimizer(lr))
        state,loss = svi_class.stable_update(state)
        return state,loss

    best_state, best_loss = update_func(init_state, svi_class, lr_init)

    for r in range(num_round):
        losses = []
        wait_counter = 0
        svi_state = copy.copy(best_state)

        if r>0:
            lr_cur = lr_init*frac_lr_decrease**r
            best_loss = jnp.inf
        else:
            lr_cur = lr_init

        with tqdm.trange(1, max_train + 1) as t:
            for j in t:
                svi_state, loss = update_func(svi_state, svi_class, lr_cur)
                if loss < best_loss:
                    best_loss = loss
                    best_state = copy.copy(svi_state)
                    wait_counter = 0
                elif wait_counter >= patience:
                    break
                else:
                    wait_counter += 1
                t.set_postfix_str(f'Round = {r:d},step_size = {lr_cur:.1e} loss: {best_loss:.3e}',refresh=False)
                losses.append(loss)

        all_losses.append(losses)

    return SVIRunResult(svi_class.get_params(best_state), svi_state,losses)

def parse_mask(mask:ArrayLike=None,data:ArrayLike=None):
    if mask is None:
        return jnp.ones_like(data).astype(jnp.bool_)
    else:
        return jnp.logical_not(jnp.array(mask.astype(int))).astype(jnp.bool_)

def check_input_data(data:ArrayLike,rms:ArrayLike,psf:ArrayLike,mask:ArrayLike=None):
    """Check input data for certain conditions and raise warnings or exceptions if needed

    Parameters
    ----------
    data : ArrayLike
        input image (galaxy/cutout)
    rms : ArrayLike
       rms/error map of the data.
    psf : ArrayLike
        pixelized PSF
    mask :ArrayLike, optional
       mask with True/1 indicating a pixel should be masked, by default None

    Raises
    ------
    RMSWarning
        If the rms map is highly discrepant from the rms of the data
    PSFNormalizationWarning
        if the pst normalization is not ~1
    KernelError
        if the provided PSF is larger than the input image (exception)
    MaskWarning
        If more than 50% of the image is masked.
    """
    rms = jnp.array(rms)
    data = jnp.array(data)
    psf = jnp.array(psf)
    if data.shape != rms.shape:
        raise ShapeMatchError('RMS map ndims must match input data ndims.')
    if jnp.median(rms) > 5*jnp.std(data):
        warnings.warn('Input RMS map appears to be highly offset (>5x) in magnitude from the rms of the pixels in the input image.')
    if not jnp.isclose(jnp.sum(psf),1.0,0.1):
        warnings.warn('PSF does not appear to be appropriately normalized; Sum(psf) is more than 0.1 away from 1.')
    if jnp.all(data.shape<psf.shape):
        raise KernelError('PSF pixel image size must be smaller than science image.')
    if mask is not None:
        mask = parse_mask(mask,data)
        if jnp.sum(mask)/jnp.prod(jnp.array(mask.shape))>0.8:
            warnings.warn('More than 80 percent of input image is masked. Is this correct? (Pysersic treats True/1 as masked; you may need to flip your boolean array.) ')
        if mask.shape !=data.shape:
            raise ShapeMatchError('Mask ndims must match input data ndims.')
    return True