from __future__ import division

import traceback
import logging
import datetime
import paho.mqtt.client as mqtt
import os
from time                  import time, sleep
import sys
import threading
from six.moves import queue
import google
from google.cloud import speech
import numpy as np
import json
import pygame
import pyaudio
from vad_utils import Audio, VADAudio
import deepspeech

WAKE_WORD_KALDI = ['Kate', 'eight kate', 'a cave', 'it came', 'ec aid', 'kate', 'in case',"cain't", 'it cave', 'eight', 'pete', 'and kate', 'a kate', 'nay tate', 'a paint', 'a cake', 'a tape', 'they take', 'kid', 'hey kid', 'it kid', 'kit', 'a tate']
WAKE_WORD_GOOGLE = ['Kate', 'kate', 'Hecate', '8/8', 'placate', 'cake', 'k', 'bait', 'hey', 'AK', 'KK', 'Escape', 'locate', 'dictate', "I can't", 'okay', 'pick 8', 'Tay k']

language_code = 'en-US'  # a BCP-47 language tag

audio_in_index = -1
mode = "N/A"
online_account = "cortic"
VoiceInit = False
startListen = False
wakeWordMode = False
robotSpeak = False
robotSpeakMsg = ""
playedNotification = False
wasListening = False
doneSpeaking = True

logging.getLogger().setLevel(logging.INFO)

def restart_program():
    python = sys.executable
    logging.info("in restart program")
    os.execl(python, python, *sys.argv)
    logging.info("Done restart program")

def connectMQTT(client):
    try:
        client.connect("127.0.0.1",1883,60)
        logging.info("Connected to broker")
        return 0
    except:
        logging.info("Broker not up yet, retrying...")
        return -1

client_stt_response = mqtt.Client()
ret = connectMQTT(client_stt_response)
while ret != 0:
    sleep(1)
    ret = connectMQTT(client_stt_response)
client_stt_response.loop_start()

def on_connect_stt_mode(client, userdata, flags, rc):
    print("STT mode client Connected with result code "+str(rc))
    client.subscribe("cait/voice_control")

def on_message_stt_mode(client, userdata, msg):
    global audio_in_index
    global mode
    global language_code
    global online_account
    global VoiceInit
    global startListen
    data = msg.payload.decode()
    logging.info("Voice Control: " + data)
    #logging.info("STT Control Init: "+data)
    if data.find("Online") != -1 or data.find("Offline") != -1:
        audio_device = json.loads(data[data.find(" ")+1:])
        audio_in_index = int(audio_device["index"])
    # if data == "Online" or data == "Offline":
        if mode == "N/A":
            logging.info("got init mode")
            if data.find("Offline") != -1:
                mode = "Offline"
            else:
                online_account = audio_device["account"]
                if audio_device["language"] == "french":
                    language_code = "fr-FR"
                elif audio_device["language"] == "chinese":
                    language_code = "zh-CN"
                mode = "Online"
        else:
            if VoiceInit:
                restart_program()
    elif data.find("VoiceDown") != -1:
        startListen = False

client_stt_mode = mqtt.Client()
client_stt_mode.on_connect = on_connect_stt_mode
client_stt_mode.on_message = on_message_stt_mode
ret = connectMQTT(client_stt_mode)
while ret != 0:
    sleep(1)
    ret = connectMQTT(client_stt_mode)
client_stt_mode.loop_start()

logging.info("Not recieve mode yet")
while mode == "N/A":
    sleep(0.2)

logging.info("Recieved mode, initializing pygame")
pygame.mixer.init()
effect = pygame.mixer.Sound('/voice_module/siri.wav')
logging.info("pygame done init")

client_stt_response.publish("cait/module_states", "STT Init Done", qos=1)
VoiceInit = True
logging.info("Recieved mode, initializing")

def on_connect_stt_control(client, userdata, flags, rc):
    print("STT control client Connected with result code "+str(rc))
    client.subscribe("cait/voice_control")

def on_message_stt_control(client, userdata, msg):
    global startListen
    global wakeWordMode
    global robotSpeak
    global robotSpeakMsg
    global playedNotification
    global wasListening
    data = msg.payload.decode()
    print("STT Control Online mode:",data)
    if data == "Reset":
        startListen = False
        wakeWordMode = False
        robotSpeak = False
        robotSpeakMsg = ""
        playedNotification = False
        wasListening = False
    elif data == "Start Listen":
        client_stt_response.publish("cait/module_states", "Init recording", qos=1)
        print("received Start Listen")
        startListen = True
        wakeWordMode = False
    elif data == "Start Listen Wakeword":
        print("received Start Listen Wakeword")
        client_stt_response.publish("cait/module_states", "Init recording", qos=1)
        startListen = True
        wakeWordMode = True

client_stt_control = mqtt.Client()
client_stt_control.on_connect = on_connect_stt_control
client_stt_control.on_message = on_message_stt_control
ret = connectMQTT(client_stt_control)
while ret != 0:
    sleep(1)
    ret = connectMQTT(client_stt_control)
client_stt_control.loop_start()

def on_connect_tts(client, userdata, flags, rc):
    print("TTS client Connected with result code "+str(rc))
    client.subscribe("cait/ttsInput")

def on_message_tts(client, userdata, msg):
    global robotSpeak
    global startListen
    global robotSpeakMsg
    global playedNotification
    global wasListening
    global doneSpeaking
    data = msg.payload.decode()
    print("ttsInput:",data)
    if startListen:
        wasListening = True
    startListen = False
    robotSpeak = True
    doneSpeaking = False
    print("Set done speaking to false")
    robotSpeakMsg = data
    playedNotification = False

client_tts = mqtt.Client()
client_tts.on_connect = on_connect_tts
client_tts.on_message = on_message_tts
ret = connectMQTT(client_tts)
while ret != 0:
    sleep(1)
    ret = connectMQTT(client_tts)
client_tts.loop_start()

client_heartbeat = mqtt.Client()
ret = connectMQTT(client_heartbeat)
while ret != 0:
    time.sleep(1)
    ret = connectMQTT(client_heartbeat)
client_heartbeat.loop_start()

class MicrophoneStream(object):
    """Opens a recording stream as a generator yielding the audio chunks."""
    def __init__(self, mic_index, rate, chunk):
        self._rate = rate
        self._chunk = chunk

        # Create a thread-safe buffer of audio data
        self._buff = queue.Queue()
        self.closed = True
        self.mic_index = mic_index

    def open_audio_interface(self):
        try:
            self._audio_stream = self._audio_interface.open(
                format=pyaudio.paInt16,
                # The API currently only supports 1-channel (mono) audio
                # https://goo.gl/z757pE
                channels=1, rate=self._rate,
                input=True, frames_per_buffer=self._chunk,
                input_device_index = self.mic_index,
                # Run the audio stream asynchronously to fill the buffer object.
                # This is necessary so that the input device's buffer doesn't
                # overflow while the calling thread makes network requests, etc.
                stream_callback=self._fill_buffer,
            )
            return True
        except:
            return False

    def __enter__(self):
        global mic_index
        self._audio_interface = pyaudio.PyAudio()
        success = self.open_audio_interface()
        retry_counter = 0
        while not success:
            retry_counter += 1
            success = self.open_audio_interface()
            if retry_counter >= 2:
                break
            sleep(0.5)
        self.closed = False

        return self

    def __exit__(self, type, value, traceback):
        self._audio_stream.stop_stream()
        self._audio_stream.close()
        self.closed = True
        # Signal the generator to terminate so that the client's
        # streaming_recognize method will not block the process termination.
        self._buff.put(None)
        self._audio_interface.terminate()

    def _fill_buffer(self, in_data, frame_count, time_info, status_flags):
        """Continuously collect data from the audio stream, into the buffer."""
        self._buff.put(in_data)
        return None, pyaudio.paContinue

    def generator(self):
        global startListen
        global doneSpeaking
        while not self.closed:
            if not startListen or not doneSpeaking:
                break
            # Use a blocking get() to ensure there's at least one chunk of
            # data, and stop iteration if the chunk is None, indicating the
            # end of the audio stream.
            chunk = self._buff.get()
            if chunk is None:
                return
            data = [chunk]

            # Now consume whatever other data's still buffered.
            while True:
                try:
                    chunk = self._buff.get(block=False)
                    if chunk is None:
                        return
                    data.append(chunk)
                except queue.Empty:
                    break

            yield b''.join(data)

def listen_print_loop(responses):
    """Iterates through server responses and prints them.

    The responses passed is a generator that will block until a response
    is provided by the server.

    Each response may contain multiple results, and each result may contain
    multiple alternatives; for details, see https://goo.gl/tjCPAU.  Here we
    print only the transcription for the top alternative of the top result.

    In this case, responses are provided for interim results as well. If the
    response is an interim one, print a line feed at the end of it, to allow
    the next result to overwrite it, until the response is a final one. For the
    final one, print a newline to preserve the finalized transcription.
    """
    num_chars_printed = 0
    global startListen
    global playedNotification
    global wakeWordMode
    global doneSpeaking
    if responses is None:
        return
    for response in responses:
        if not response.results:
            continue

        # The `results` list is consecutive. For streaming, we only care about
        # the first result being considered, since once it's `is_final`, it
        # moves on to considering the next utterance.
        result = response.results[0]
        if not result.alternatives:
            continue

        # Display the transcription of the top alternative.
        transcript = result.alternatives[0].transcript

        # Display interim results, but with a carriage return at the end of the
        # line, so subsequent lines will overwrite them.
        #
        # If the previous result was longer than this one, we need to print
        # some extra spaces to overwrite the previous result
        overwrite_chars = ' ' * (num_chars_printed - len(transcript))

        if not startListen or not doneSpeaking:
            break
        if not result.is_final:
            logging.info("In print loop for not final")
            sys.stdout.write(transcript + overwrite_chars + '\r')
            sys.stdout.flush()
            logging.info("In print loop for not final, flushed")
            num_chars_printed = len(transcript)

        else:
            sentence = transcript + overwrite_chars
            logging.info("******************** "+sentence)
            logging.info("In print loop for final")
            if not wakeWordMode:
                client_stt_response.publish("cait/sttData", sentence, qos=0)
                logging.info("Publish stt data")
                startListen = False
                playedNotification = False
                logging.info("Breaking")
                break
            else:
                sentence_list = sentence.split()
                if ' '.join(sentence_list[-2:]) in WAKE_WORD_GOOGLE or sentence_list[-1] in WAKE_WORD_GOOGLE:
                    client_stt_response.publish("cait/module_states", "WAKEWORD", qos=0)
                    #print("Published:", sentence)
                    startListen = False
                    break

def heartbeat_func():
    global client_heartbeat
    while True:
        client_heartbeat.publish("cait/module_states", "Voice Up", qos=1)
        sleep(1)

def main():
    global audio_in_index
    global robotSpeak
    global startListen
    global robotSpeakMsg
    global playedNotification
    global wasListening
    global doneSpeaking
    global online_account
    global mode
    global language_code

    DEFAULT_SAMPLE_RATE = 16000
    MODEL_FILE = "/opt/deepspeech-models/deepspeech-0.9.1-models.tflite"
    SCORE_FILE = "/opt/deepspeech-models/deepspeech-0.9.1-models.scorer"
    MODEL_FILE_CN = "/opt/deepspeech-models/deepspeech-0.9.1-models-zh-CN.tflite"
    SCORE_FILE_CN = "/opt/deepspeech-models/deepspeech-0.9.1-models-zh-CN.scorerr"

    json_file = "/voice_module/" + online_account
    # Audio recording parameters
    RATE = 16000
    CHUNK = int(RATE / 10)  # 100ms

    if mode == "Online":
        logging.warning("Using cloud account: " + json_file)
        client = speech.SpeechClient.from_service_account_json(json_file)
        config = speech.RecognitionConfig(
            encoding=speech.RecognitionConfig.AudioEncoding.LINEAR16,
            sample_rate_hertz=RATE,
            language_code=language_code)
        streaming_config = speech.StreamingRecognitionConfig(
            config=config,
            interim_results=True)
    else:
        client = deepspeech.Model(MODEL_FILE)
        client.enableExternalScorer(SCORE_FILE)

    heartbeat_thread = threading.Thread(target=heartbeat_func, daemon=True)
    heartbeat_thread.start()
    while True:
        if startListen and doneSpeaking:
            print("Start listening")
            if mode == "Online":
                with MicrophoneStream(audio_in_index, RATE, CHUNK) as stream:
                    if not playedNotification and not wakeWordMode:
                        try:
                            effect.play()
                        except:
                            effect.play()
                        playedNotification = True
                    client_stt_response.publish("cait/module_states", "Init recording done", qos=1)
                    audio_generator = stream.generator()
                    requests = (speech.StreamingRecognizeRequest(audio_content=content)
                                for content in audio_generator)
                    try:
                        responses = client.streaming_recognize(streaming_config, requests)
                    except google.api_core.exceptions.PermissionDenied:
                        logging.warning("Google account error.")
                        responses = None
                        client_stt_response.publish("cait/module_states", "Voice Exception: Google cloud client denied, check your account to make sure it is working", qos=1)
                        startListen = False
                        playedNotification = False
                    # Now, put the transcription responses to use.
                    try:
                        listen_print_loop(responses)
                    except Exception as e:
                        print("Excption handle : Exceeded maximum allowed stream duration, re-listening")
            else:
                if not playedNotification and not wakeWordMode:
                    try:
                        effect.play()
                    except:
                        effect.play()
                    playedNotification = True
                #sleep(1)
                while pygame.mixer.get_busy() == True:
                    #logging.info("Playing du du")
                    continue
                vad_audio = VADAudio(audio_in_index, aggressiveness=2,
                            input_rate=DEFAULT_SAMPLE_RATE)
                logging.info("Created VAD")
                client_stt_response.publish("cait/module_states", "Init recording done", qos=1)
                frames = vad_audio.vad_collector()
                stream_context = client.createStream()
                wav_data = bytearray()
                for frame in frames:
                    if frame is not None:
                        logging.info("Streaming frame")
                        stream_context.feedAudioContent(np.frombuffer(frame, np.int16))
                    else:
                        logging.info("end utterence")
                        sentence = stream_context.finishStream()
                        logging.info("User said: " + sentence)
                        if sentence == "":
                            stream_context = client.createStream()
                            continue
                        if not wakeWordMode:
                            client_stt_response.publish("cait/sttData", sentence, qos=2)
                            startListen = False
                            playedNotification = False
                            break
                        else:
                            sentence_list = sentence.split()
                            if ' '.join(sentence_list[-2:]) in WAKE_WORD_GOOGLE or sentence_list[-1] in WAKE_WORD_GOOGLE:
                                client_stt_response.publish("cait/module_states", "WAKEWORD", qos=2)
                                #print("Published:", sentence)
                                startListen = False
                                break
                if not startListen:
                    logging.info("stop listen")
                    vad_audio.destroy()
                    logging.info("Destroyed VAD")
        else:
            if robotSpeak:
                client_stt_control.publish("cait/module_states", "Start Speaking", qos=1)
                if mode == "Online":
                    if language_code == "fr-FR":
                        os.system('/voice_module/speech_fr.sh "' + robotSpeakMsg + '"')
                    elif language_code == "zh-CN":
                        os.system('/voice_module/speech_chs.sh "' + robotSpeakMsg + '"')
                    else:
                        os.system('/voice_module/speech_en.sh "' + robotSpeakMsg + '"')
                else:
                    os.system('mimic -t "' + robotSpeakMsg + '" -voice slt')
                robotSpeak = False
                if wasListening:
                    startListen = True
                    wasListening = False
                doneSpeaking = True
                print("Set done speaking to true")
                client_stt_control.publish("cait/module_states", "Done Speaking", qos=1)
            sleep(0.03)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
    Formatter - Demo

    Format column based document on user choice

    Usage:
    Run script to see how it works

    Note: By commenting or deleting everything, including,
    the <comment_or_delete> tags it is ready to be used

"""
from Npp import editor, notepad, console, MESSAGEBOXFLAGS
from itertools import zip_longest

try:
    import tkinter as tk
    from tkinter import ttk
except ImportError as e:
    notepad.messageBox(('Unable to import Tkinter libraries,\n'
                        'these are needed for the UI.\n\n'
                        'Check your installation.\n\n'
                        '{}'.format(e.message)),'Missing Library', MESSAGEBOXFLAGS.ICONERROR)
    console.show()
    raise

def main():

    list_of_separators = [('Comma',0),('Semicolon',1),('Space',2),('Clipboard', 3)]
    separator_dict = {0:',', 1:';', 2:' ', 3:'clipboard'}

    list_of_areas = [('Whole document',0),('Selection only',1),('Visible lines only',2)]

    format_types_dict = {0:'\r\n',1:'\r',2:'\n'}

    def button_reformat():
        reformat(separator_value.get(), reformat_value.get())
        window.destroy()

    def button_quit(): window.destroy()

    def set_reformat_choice(): reformat_value.set(reformat_value.get())

    def set_separator_choice(): separator_value.set(separator_value.get())

    def prepare(separator, rows, eol):
        column_width_list = []
        for row in rows:
            splitted_row = row.split(separator)
            column_width_list = [max(x) for x in zip_longest(column_width_list, [len(x.strip()) for x in splitted_row], fillvalue=(0))]

        new_formatted_list = []
        for row in (x for x in rows):
            splitted_row = row.split(separator)
            new_formatted_list.append('{} '.format(separator).join(['{0:<{1}}'.format(x.strip(),column_width_list[i]) for i,x in enumerate(splitted_row)]))

        return new_formatted_list

    def reformat(separator,area):
        separator = window.clipboard_get() if separator_dict.get(separator) == 'clipboard' else separator_dict.get(separator)

        used_eol = format_types_dict.get(notepad.getFormatType())

        editor.beginUndoAction()

        if area == 0:
            rows = editor.getCharacterPointer().splitlines()
            rows = prepare(separator, rows, used_eol)
            editor.setText(used_eol.join(rows))
        elif area == 1:
            if editor.selectionIsRectangle():

                selections = [(editor.getSelectionNStart(i),editor.getSelectionNEnd(i)) for i in range(editor.getSelections())]
                rows = [editor.getRangePointer(x, y-x) for x,y in selections]
                rows = prepare(separator, rows, used_eol)

                editor.clearSelections()

                for i, position in zip(reversed(range(len(selections))), reversed(selections)):
                    editor.setTarget(*position)
                    editor.replaceTarget(rows[i])
            else:
                first_selected_line, last_selected_line = editor.getUserLineSelection()
                startpos = editor.positionFromLine(first_selected_line)
                endpos = editor.positionFromLine(last_selected_line)

                rows = editor.getRangePointer(startpos, endpos-startpos)
                rows = prepare(separator, rows.split(used_eol), used_eol)

                editor.setTarget(startpos, endpos)
                editor.replaceTarget(used_eol.join(rows))
        else:
            first_visible_line = editor.getFirstVisibleLine()
            startpos = editor.positionFromLine(first_visible_line)
            endpos = editor.positionFromLine(editor.linesOnScreen() + first_visible_line)

            rows = editor.getRangePointer(startpos, endpos-startpos)
            rows = prepare(separator, rows.split(used_eol), used_eol)

            editor.setTarget(startpos, endpos)
            editor.replaceTarget(used_eol.join(rows))
        editor.endUndoAction()

    window = tk.Tk()
    window.title("Formatter")
    window.attributes('-toolwindow', 1)
    window.attributes('-topmost', 1)
    window.configure(background='#282C34')
    window.grid_columnconfigure(0, weight=1)
    window.grid_columnconfigure(1, weight=1)

    control_styles = ttk.Style()
    control_styles.configure('TButton', font=('courier', 16, 'bold'), background='#282C34', relief='flat')
    control_styles.configure('TCheckbutton', font=('courier', 12), background='#282C34', foreground='#B5BCC9')
    control_styles.configure('TRadiobutton', font=('courier', 12), background='#282C34', foreground='#B5BCC9')
    control_styles.configure('TLabelframe.Label', font=('courier', 12, 'italic'), background='#282C34', foreground='#61AFEF')
    control_styles.configure('TLabelframe', background='#282C34', foreground='#B5BCC9')
    control_styles.configure('TLabel', font=('courier', 12), background='#282C34', foreground='#B5BCC9')
    control_styles.configure('TFrame', background='#282C34')

    separator_value = tk.IntVar()
    reformat_value = tk.IntVar()

    separator_frame = ttk.LabelFrame(window)
    separator_frame.configure(text='Separator')
    separator_frame.configure(width=280)

    for label, _value in list_of_separators:
        ttk.Radiobutton(separator_frame,
                        text=label,
                        command = set_separator_choice,
                        variable = separator_value,
                        value = _value).pack(padx=10, anchor=tk.W)
    separator_frame.grid(row=0, column=0, padx=10, pady=15, sticky=tk.W+tk.E+tk.N+tk.S)

    reformat_frame = ttk.LabelFrame(window)
    reformat_frame.configure(text='Area to reformat')
    reformat_frame.configure(width=280)

    for label, _value in list_of_areas:
        ttk.Radiobutton(reformat_frame,
                        text=label,
                        command=set_reformat_choice,
                        variable=reformat_value,
                        value=_value).pack(padx=10, anchor=tk.W)
    reformat_frame.grid(row=0, column=1, padx=10, pady=15, sticky=tk.W+tk.E+tk.N+tk.S)

    btn_frame = ttk.Frame(window)
    btn_frame.place(relx=0.02, rely=0.64, relheight=0.32, relwidth=0.96)
    btn_frame.configure(relief=tk.GROOVE)
    btn_frame.configure(borderwidth="2")
    btn_frame.configure(width=550)

    btn_reformat = ttk.Button(btn_frame)
    btn_reformat.place(relx=0.07, rely=0.24, height=47, width=207)
    btn_reformat.configure(text='Reformat')
    btn_reformat.configure(width=207)
    btn_reformat.configure(command=button_reformat)

    btn_quit = ttk.Button(btn_frame)
    btn_quit.place(relx=0.58, rely=0.26, height=47, width=207)
    btn_quit.configure(text='Quit')
    btn_quit.configure(width=207)
    btn_quit.configure(command=button_quit)

    # center window
    w = window.winfo_screenwidth()
    h = window.winfo_screenheight()
    size = (600,250)
    xpos = w/2 - size[0]/2
    ypos = h/2 - size[1]/2
    window.geometry("%dx%d+%d+%d" % (size + (xpos, ypos)))

    # start app - blocks rest of the script
    window.mainloop()

# <comment_or_delete>
def run_demo():

    demo_message = ('Running in demo mode\n'
                    'By commenting run_demo() line or by commenting/deleting the whole tag it is ready to be used')
    notepad.messageBox(demo_message,'Demo Mode')

    import random
    _number_of_lines = 100000
    _s = 50
    _line_split_separator = ','
    _data = ['{0}{3}{1}{3}{2}\r\n'.format('a'*random.randint(0,_s),'b'*random.randint(0,_s),'c'*random.randint(0,_s),_line_split_separator) for i in range(_number_of_lines)]
    notepad.new()
    editor.setText(''.join(_data))
    del _data

run_demo()
# </comment_or_delete>

main()


# -*- coding: utf-8 -*-
"""
Created on Tue Mar  3 12:32:33 2020

@author: abz1e14
"""
#1
#matplotlib inline
from pandas import read_excel
import matplotlib.pyplot as plt
import statsmodels.api as sm
import warnings
import itertools
plt.style.use('fivethirtyeight')

df = read_excel('BuildingMaterials.xls', sheet_name='Data', header=0,
                index_col=0, parse_dates=True, squeeze=True)

#===================================================
#Identifying the parameters with smallest AIC
#===================================================
#Define the p, d and q parameters to take any value between 0 and 1
p = d = q = range(0, 2)

# Generate all different combinations of p, q and q triplets
pdq = list(itertools.product(p, d, q))

# Generate all different combinations of seasonal p, q and q triplets
seasonal_pdq = [(x[0], x[1], x[2], 12) for x in list(itertools.product(p, d, q))]

#print(len(pdq))
warnings.filterwarnings("ignore") # specify to ignore warning messages

collectAICs=[]
for param in pdq:
    for param_seasonal in seasonal_pdq:
        try:
            mod = sm.tsa.statespace.SARIMAX(df,
                                            order=param,
                                            seasonal_order=param_seasonal,
                                            enforce_invertibility=False)
            results = mod.fit()
            print('ARIMA{}x{} - AIC:{}'.format(param, param_seasonal, results.aic))
        except:
            continue

LearningResource/Forecasting/Chapter 3 Data and Codes 2023/Chapter 3 Workshop Data/Workshop 3 - solution codes/AutomaticParameterComputationBuildingMaterials.py


import base64
import sqlite3
from datetime import date, timedelta, datetime
from io import BytesIO

import matplotlib.pyplot as plt
from more_itertools import batched
from quart import Quart, render_template_string, request

db_con = sqlite3.connect("temperatures.db")
app = Quart(__name__)

TEMPLATE = """
    <!DOCTYPE html>

            <title>TempLogger</title>

            Last reading

            <big>{{last_temperature[1]}}°C</big>

            {{last_temperature[0]}}

                <form method="get">
                    From: <input type="date" name="from-date" value="{{from_date}}">
                    To: <input type="date" name="to-date" value={{to_date}}>
                    <input type="submit" value="Select">
                </form>

"""

def get_temperatures(cursor, from_date: date, to_date: date) -> list[tuple]:
    from_timestamp = datetime(year=from_date.year, month=from_date.month, day=from_date.day).timestamp()
    to_timestamp = datetime(year=to_date.year, month=to_date.month, day=to_date.day, hour=23, minute=59).timestamp()
    cursor.execute(
        "SELECT * FROM temperatures WHERE timestamp >= ? AND timestamp <= ? ORDER BY timestamp ASC",
        (from_timestamp, to_timestamp),
    )
    temperature_records = [(datetime.fromtimestamp(r[0]), r[1]) for r in cursor]
    return temperature_records

def get_last_temperature(cursor) -> tuple:
    cursor.execute("SELECT * FROM temperatures ORDER BY timestamp DESC LIMIT 1")
    last_temperature = cursor.fetchone()
    app.logger.info(last_temperature)
    return datetime.fromtimestamp(last_temperature[0]), last_temperature[1]

@app.get("/")
async def chart():
    db_cursor = db_con.cursor()
    from_date = request.args.get("from-date", date.today() - timedelta(days=1))
    from_date = datetime.strptime(from_date, "%Y-%m-%d").date() if type(from_date) == str else from_date
    to_date = request.args.get("to-date", date.today())
    to_date = datetime.strptime(to_date, "%Y-%m-%d").date() if type(to_date) == str else to_date
    temperature_records = get_temperatures(db_cursor, from_date, to_date)
    last_temperature = get_last_temperature(db_cursor)
    plt.figure(figsize=(15, 5))
    plt.title("Temperature history")
    plt.grid()
    plt.xlabel("Time")
    plt.ylabel("Temperature")
    plt.plot([r[0] for r in temperature_records], [r[1] for r in temperature_records], color="tab:blue")
    batches = batched(temperature_records, 12)
    temperature_averages = [(b[-1][0], sum([r[1] for r in b]) / len(b)) for b in batches]
    plt.plot([r[0] for r in temperature_averages], [r[1] for r in temperature_averages], color="tab:green")
    figfile = BytesIO()
    plt.savefig(figfile, format="png")
    figfile.seek(0)
    figdata = base64.b64encode(figfile.getvalue())
    return await render_template_string(
        TEMPLATE, plot=figdata.decode(), last_temperature=last_temperature, from_date=from_date, to_date=to_date
    )

def main():
    app.run(debug=True, port=5000, host="0.0.0.0")

if __name__ == "__main__":
    main()


def plot_box_per_column(df, columns, ratio_IQR = 1.5):
    import matplotlib.pyplot as plt
    plt.figure()
    for idx, col in enumerate(columns):
        plt.subplot(2,len(columns), idx+1)
        df.boxplot(column=col)

    print(f"before: length is {len(df)}")
    ##################################################################################################
    ## IQR 기준으로 outlier 지우기: df.loc[df[col].between(df[col].quantile(q=0.25)-1.5*col_IQR, df[col].quantile(q=0.75)+1.5*col_IQR)]
    ##################################################################################################

    for col in columns:
        col_IQR = df[col].quantile(0.75) - df[col].quantile(0.25)
        df=df.loc[df[col].between(df[col].quantile(q=0.25)-ratio_IQR*col_IQR, df[col].quantile(q=0.75)+ratio_IQR*col_IQR)]

    print(f"after: length is {len(df)}")

    for idx, col in enumerate(columns):
        plt.subplot(2,len(columns), len(columns)+idx+1)
        df.boxplot(column=col)

    plt.show()


from kafka import KafkaProducer
import json
import argparse
import os
from common import Common

common = Common("client")
common.load()

parser = argparse.ArgumentParser()
parser.add_argument("topic", help="Name for the topic to submit to.")
parser.add_argument("name", help="Name for the task to submit.")
parser.add_argument("style", help="File name of style.")
parser.add_argument("--args", default='-backend cudnn -batch_size 1 -model starling', help="Arguments to use for training.")
parser.add_argument("--iters", type=int, default=250, help="Iterations per cycle.")
parser.add_argument("--cycles", type=int, default=200, help="Number of cycles to train for.")
parser.add_argument("--learning_rate", type=float, default=4e-2, help="Initial learning rate.")
parser.add_argument("--learning_decay", type=float, default=0.8, help="Discount factor for learning rate by period.")
parser.add_argument("--learning_period", type=int, default=25, help="Number of cycles between learning rate discounts.")

args = parser.parse_args()

producer = KafkaProducer(
        value_serializer=lambda m: json.dumps(m).encode("utf-8"))

print ("Ready.")

task =  {
            "args": args.args,
            "style": args.style,
            "starting_checkpoint": "",
            "iterations_per_cycle": args.iters,
            "cycles_remaining": args.cycles,
            "cycles_completed": 0,
            "base_learning_rate": args.learning_rate,
            "learning_decay": args.learning_decay,
            "learning_period": args.learning_period
        }

common.log("Uploading %s" % args.style)

common.upload("style", args.style)
common.log("Upload complete; submitting job named %s" % args.name)

producer.send(args.topic, key=args.name.encode("utf-8"), value=task)
common.log("Job submitted")


'''
- We’ll begin building the game by creating an empty Pygame window.

- Later, we’ll draw the game elements, such as the ship & the aliens, on this window.

- We’ll also make our game respond to user input, set the background color & load a ship image

- We’ll make an empty Pygame window by creating a class to represent the game
'''
import sys
import pygame
from settings import Settings
from ship import Ship
from bullet import Bullet
from alien import Alien

class AlienInvasion:
    """ the overall class to manage game assets & behavior """
    def __init__(self):
        """ initialize the game & create game resources """
        '''
        - In the __init__() method, the pygame.init() function initializes the background settings that Pygame needs to work properly

        - We call pygame.display.set_mode() to create a display window, on which we’ll draw all the game’s graphical elements

        - The argument (self.settings.screen_width, self.settings.screen_height) is a tuple that defines the dimensions of the game window,
          which will be 1200 pixels wide by 800 pixels high

        - We assign this display window to the attribute self.screen, so it will be available in all methods in the class

        - The object we assigned to self.screen is called a surface.

        - A surface in Pygame is a part of the screen where a game element can be displayed.

        - Each element in the game, like an alien or a ship, is its own surface.

        - The surface returned by display.set_mode() represents the entire game window.

        - When we activate the game’s animation loop, this surface will be redrawn on every pass through the loop,
          so it can be updated with any changes triggered by user input
        '''
        pygame.init()

        self.settings = Settings() # create a Settings instance

        #self.screen = pygame.display.set_mode((1200, 800))

        self.screen = pygame.display.set_mode((self.settings.screen_width, self.settings.screen_height))

        pygame.display.set_caption('Alien Invasion')

        #self.background_color = (230, 230, 230)

        '''
        - We import Ship and then make an instance of Ship after the screen has been created.

        - The call to Ship() requires 1 argument: an instance of AlienInvasion.

        - The self argument here refers to the current instance of AlienInvasion.

        - This is the parameter that gives Ship access to the game’s resources, such as the screen object.

        - We assign this Ship instance to self.ship

        - After filling the background, we draw the ship on the screen by calling ship.blitme() in run_game()
        '''
        self.ship = Ship(self)

        '''
        - Pygame has a fullscreen mode that you might like better than running the game in a regular window (optional)

        - When creating the screen surface, we pass a size of (0, 0) and the parameter pygame.FULLSCREEN

        - This tells Pygame to figure out a window size that will fill the screen.

        - Because we don’t know the width & height of the screen ahead of time, we update these settings after the screen is created

        - We use the width and height attributes of the screen’s rect to update the settings object

        - Make sure you can quit by pressing Q before running the game in fullscreen mode;
          Pygame offers no default way to quit a game while in fullscreen mode
        '''
        #self.screen = pygame.display.set_mode((0, 0), pygame.FULLSCREEN)

        #self.settings.screen_width = self.screen.get_rect().width

        #self.settings.screen_height = self.screen.get_rect().height

        '''
        - Now that we have a Bullet class and the necessary settings defined,
          we can write code to fire a bullet each time the player presses the spacebar.

        - We’ll create a group in AlienInvasion to store all the live bullets
          so we can manage the bullets that have already been fired.

        - This group will be an instance of the pygame.sprite.Group class,
          which behaves like a list with some extra functionality that’s helpful when building games.

        - We’ll use this group to draw bullets to the screen on each pass through the main loop and to update each bullet’s position

        - Then we need to update the position of the bullets on each pass through the while loop in run_game()
        '''
        self.bullets = pygame.sprite.Group()

        '''
        - We want to create an instance of Alien so we can see the 1st alien on the screen.

        - As part of our setup work, the code for this instance is at the end of __init__() in AlienInvasion.

        - Eventually, we’ll create an entire fleet of aliens, which will be quite a bit of work,
          so we’ll make a new helper method called _create_fleet()

        - The order of methods in a class doesn’t matter as long as there’s some consistency to how they’re placed.

        - I’ll place _create_fleet() just before _update_screen() but anywhere in AlienInvasion will work.

        - 1st, we’ll import the Alien class
        '''
        self.aliens = pygame.sprite.Group()
        self._create_fleet()

    def run_game(self):
        """ start the game's main loop """
        '''
        - This method contains a while loop that runs continually.

        - The while loop contains an event loop and code that manages screen updates.

        - An event is an action that the user performs while playing the game, such as pressing a key or moving the mouse.

        - To make our program respond to events, we write this event loop to listen for events and
          perform appropriate tasks depending on the kinds of events that occur.

        - The for loop is an event loop.

        - To access the events that Pygame detects, we’ll use the pygame.event.get() function.

        - This function returns a list of events that have taken place since the last time this function was called.

        - Any keyboard or mouse event will cause this for loop to run.

        - Inside the loop, we’ll write a series of if statements to detect and respond to specific events.

        - For example, when the player clicks the game window’s close button, a pygame.QUIT event is detected and we call sys.exit() to exit the game.

        - The call to pygame.display.flip() tells Pygame to make the most recently drawn screen visible.

        - In this case, it simply draws an empty screen on each pass through the while loop, erasing the old screen so only the new screen is visible.

        - When we move the game elements around, pygame.display.flip() continually updates the display to show the new positions of game elements and
          hides the old ones, creating the illusion of smooth movement

        - We fill the screen with the background color using the fill() method, which acts on a surface and takes only 1 argument: a color
        '''
        while True:
            # for mouse & keyboard events
            '''
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    sys.exit()
            '''
            self._check_events() # a helper method call using dot notation with self & the method name

            '''
            - The ship’s position will be updated after we’ve checked for keyboard events and before we update the screen.

            - This allows the ship’s position to be updated in response to player input and
              ensures the updated position will be used when drawing the ship to the screen
            '''
            self.ship.update()

            # When you call update() on a group, the group automatically calls update() for each sprite in the group
            # -> this line of code calls bullet.update() for each bullet we place in the bullets group
            #self.bullets.update()

            '''
            - At the moment, the bullets disappear when they reach the top but only
              because Pygame can’t draw them above the top of the screen.

            - The bullets actually continue to exist; their y-coordinate values just grow increasingly negative.

            - This is a problem because they continue to consume memory and processing power.

            - We need to get rid of these old bullets or the game will slow down from doing so much unnecessary work.

            - To do this, we need to detect when the bottom value of a bullet’s rect has a value of 0,
              which indicates the bullet has passed off the top of the screen

            - When you use a for loop with a list (or a group in Pygame),
              Python expects that the list will stay the same length as long as the loop is running.

            - As we can’t remove items from a list or group within a for loop, we have to loop over a copy of the group.

            - We use the copy() method to set up the for loop, which enables us to modify bullets inside the loop.

            - We check each bullet to see whether it has disappeared off the top of the screen.

            - If it has, we remove it from bullets.
            '''
            '''
            for bullet in self.bullets.copy():
              if bullet.rect.bottom <= 0:
                self.bullets.remove(bullet)
            '''
            '''
            - Finally we insert a print() call to show how many bullets currently exist in the game and
              verify that they’re being deleted when they reach the top of the screen

            - If this code works correctly, we can watch the terminal output while firing bullets and
              see that the number of bullets decreases to zero after each series of bullets has cleared the top of the screen.

            - After you run the game and verify that bullets are being deleted properly, remove the print() call

            - If you leave it in, the game will slow down significantly because it takes more time to
              write output to the terminal than it does to draw graphics to the game window
            '''
            #print(len(self.bullets))

            # to keep run_game() well-organized, move the code that updates the bullets & removes the old ones to _update_bullets()
            self._update_bullets()

            # update the position of each alien in the aliens group
            # We update the aliens’ positions after the bullets have been updated
            # as we’ll soon be checking to see whether any bullets hit any aliens
            self._update_aliens()

            '''
            # redraw the screen during each pass thru the loop
            #self.screen.fill(self.background_color)
            self.screen.fill(self.settings.background_color)

            self.ship.blitme()

            # make the most recently drawn screen visible
            pygame.display.flip()
            '''
            self._update_screen() # a helper method call

    '''
    - In large projects, you’ll often refactor code you’ve written before adding more code.

    - Refactoring simplifies the structure of the code you’ve already written, making it easier to build on.

    - In this section, we’ll break the run_game() method, which is getting lengthy, into 2 helper methods: _check_events() & _update_screen()

    - A helper method does work inside a class but isn’t meant to be called through an instance.

    - In Python, a single leading underscore indicates a helper method

    - If you’ve already built a number of games, you’ll probably start out by breaking your code into methods like these.

    - But if you’ve never tackled a project like this, you probably won’t know how to structure your code.

    - This approach of writing code that works & then restructuring it as it grows more complex gives you an idea of a realistic development process:
      you start out writing your code as simply as possible and then refactor it as your project becomes more complex
    '''
    def _check_events(self):
        '''
        - Moving the code that manages events to _check_events() simplifies run_game() & isolates the event management loop.

        - Isolating the event loop allows you to manage events separately from other aspects of the game, such as updating the screen
        '''

        ''' elif event.type == pygame.KEYDOWN ->

        - we’ll give the player the ability to move the ship right and left.

        - We’ll write code that responds when the player presses the right or left arrow key.

        - We’ll focus on movement to the right 1st and then we’ll apply the same principles to control movement to the left

        - Whenever the player presses a key, that keypress is registered in Pygame as an event.

        - Each event is picked up by the pygame.event.get() method.

        - We need to specify in our _check_events() method what kind of events we want the game to check for.

        - Each keypress is registered as a KEYDOWN event.

        - When Pygame detects a KEYDOWN event, we need to check whether the key that was pressed is one that triggers a certain action.

        - For example, if the player presses the right arrow key, we want to increase the ship’s rect.x value to move the ship to the right
        '''

        ''' elif event.type == pygame.KEYUP ->

        - When the player holds down the right arrow key, we want the ship to continue moving right until the player releases the key.

        - We’ll have the game detect a pygame.KEYUP event so we’ll know when the right arrow key is released;
          then we’ll use the KEYDOWN and KEYUP events together with a flag called moving_right to implement continuous motion.

        - When the moving_right flag is False, the ship will be motionless.

        - When the player presses the right arrow key, we’ll set the flag to True and
          when the player releases the key, we’ll set the flag to False again.

        - The Ship class in ship.py controls all attributes of the ship, so we’ll give it an attribute called moving_right and
          an update() method to check the status of the moving_right flag.

        - The update() method will change the position of the ship if the flag is set to True.

        - We’ll call this method once on each pass through the while loop to update the position of the ship

        - The new elif block responds to KEYUP events.

        - When the player releases the right arrow key (K_RIGHT), we set moving_right to False
        '''

        ''' elif event.key == pygame.K_LEFT ->

        - Now that the ship can move continuously to the right, adding movement to the left is straightforward.

        - Again, we’ll modify the Ship class and the _check_events() method

        - If a KEYDOWN event occurs for the K_LEFT key, we set moving_left to True.

        - If a KEYUP event occurs for the K_LEFT key, we set moving_left to False.

        - We can use elif blocks here because each event is connected to only 1 key.

        - If the player presses both keys at once, 2 separate events will be detected.

        - When you run alien_invasion.py now, you should be able to move the ship continuously to the right and left.

        - If you hold down both keys, the ship should stop moving
        '''
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
              sys.exit()
            elif event.type == pygame.KEYDOWN:
              '''
              if event.key == pygame.K_RIGHT:
                #self.ship.rect.x += 1

                # how the game responds when the right arrow key's pressed:
                # instead of changing the ship’s position directly, moving_right's set to True
                self.ship.moving_right = True
              elif event.key == pygame.K_LEFT:
                self.ship.moving_left = True
              '''
              self._check_keydown_events(event)
            elif event.type == pygame.KEYUP:
              '''
              if event.key == pygame.K_RIGHT:
                self.ship.moving_right = False
              elif event.key == pygame.K_LEFT:
                self.ship.moving_left = False
              '''
              self._check_keyup_events(event)

    '''
    - The _check_events() method will increase in length as we continue to develop the game,
      so let’s break _check_events() into 2 new helper methods: one that handles KEYDOWN events & another that handles KEYUP events

    - Each needs a self parameter and an event parameter.

    - The bodies of these 2 methods are copied from _check_events() & we’ve replaced the old code with calls to the new methods
    '''
    def _check_keydown_events(self, event):
      if event.key == pygame.K_RIGHT:
        self.ship.moving_right = True
      elif event.key == pygame.K_LEFT:
        self.ship.moving_left = True
      elif event.key == pygame.K_q:
        # a keyboard shortcut to end the game when the player presses Q, instead of using cursor to close the game window
        sys.exit()
      elif event.key == pygame.K_SPACE:
        # modify _check_keydown_events() to fire a bullet when the player presses the spacebar
        # leave _check_keyup_events() unchanged as nothing happens when the spacebar is released
        # modify _update_screen() to make sure each bullet is drawn to the screen before calling flip()
        # write a new method called _fire_bullet() to handle this work
        self._fire_bullet()

    def _check_keyup_events(self, event):
      if event.key == pygame.K_RIGHT:
        self.ship.moving_right = False
      elif event.key == pygame.K_LEFT:
        self.ship.moving_left = False

    def _fire_bullet(self):
      """ Create a new bullet and add it to the bullets group """
      '''
      - In this method, we make an instance of Bullet and call it new_bullet

      - We then add it to the group bullets using the add() method

      - The add() method is similar to append() but it’s a method that’s written specifically for Pygame groups

      - When the player presses the spacebar, we check the length of bullets.

      - If len(self.bullets) is less than 3, we create a new bullet.

      - But if 3 bullets are already active, nothing happens when the spacebar is pressed.

      - When you run the game now, you should be able to fire bullets only in groups of 3
      '''
      if len(self.bullets) < self.settings.bullets_allowed:
        new_bullet = Bullet(self)

        self.bullets.add(new_bullet)

    def _update_bullets(self):
      """ Update position of bullets and get rid of old bullets """
      self.bullets.update()

      for bullet in self.bullets.copy():
        if bullet.rect.bottom <= 0:
          self.bullets.remove(bullet)

    def _create_fleet(self):
      """ Create the fleet of aliens """
      '''
      - To draw a fleet, we need to figure out how many aliens can fit across the screen and
        how many rows of aliens can fit down the screen.

      - We’ll 1st figure out the horizontal spacing between aliens and create a row;
        then we’ll determine the vertical spacing and create an entire fleet

      - To figure out how many aliens fit in a row, let’s look at how much horizontal space we have.

      - The screen width is stored in settings.screen_width but we need an empty margin on either side of the screen.

      - We’ll make this margin the width of 1 alien.

      - Because we have 2 margins, the available space for aliens is the screen width minus 2 alien widths:
        available_space_x = settings.screen_width – (2 * alien_width)

      - We also need to make the spacing between aliens 1 alien width.

      - The space needed to display 1 alien is twice its width:
        1 width for the alien & 1 width for the empty space to its right (alien_width & spacing_bw_aliens)

      - To find the number of aliens that fit across the screen, we divide the available space by 2 times the width of an alien.

      - We use floor division (//), which divides 2 numbers & drops any remainder, so we’ll get an integer number of aliens:
        number_aliens_x = available_space_x // (alien_width + spacing_bw_aliens)

      - We’ll use these calculations when we create the fleet
      '''

      '''
      - To finish the fleet, we’ll determine the number of rows that fit on the screen and then
        repeat the loop for creating the aliens in 1 row until we have the correct number of rows.

      - To determine the number of rows, we find the available vertical space by subtracting the alien height from the top,
        the ship height from the bottom & 2 alien heights from the bottom of the screen:

        =>  available_space_y = settings.screen_height – (3 * alien_height) – ship_height

      - The result will create some empty space above the ship,
        so the player has some time to start shooting aliens at the beginning of each level.

      - Each row needs some empty space below it, which is equal to the height of 1 alien.

      - To find the number of rows, we divide the available vertical space by 2 times the height of an alien.

      - We use floor division because we can only make an integer number of rows
        (Again, if these calculations are off, we’ll see it right away & adjust our approach until we have reasonable spacing)

        =>  number_rows = available_height_y // (2 * alien_height)

      - Now that we know how many rows fit in a fleet, we can repeat the code for creating a row
      '''
      # an Alien instance is created & placed in the default upper-left area of the screen
      # but it isn't part of the fleet at this point, so it's not added to the aliens group yet
      alien = Alien(self)

      #self.aliens.add(alien)

      #alien_width = alien.rect.width # 1 horizontal space

      # the size attribute contains a tuple with the width & height (1 vertical space) of a rect object
      alien_width, alien_height = alien.rect.size

      empty_space = alien_width

      available_space_x = self.settings.screen_width - (alien_width * 2)

      alien_space = alien_width + empty_space

      alien_quantity_x = available_space_x // alien_space

      # determine the number of rows of aliens that fit on the screen
      ship_height = self.ship.rect.height

      available_space_y = self.settings.screen_height - (alien_height * 3) - ship_height

      rows_of_aliens = available_space_y // (alien_height * 2)

      # checking the attributes' values
      print(f'\n\tscreen_width: {self.settings.screen_width} | screen_height: {self.settings.screen_height}')

      print(f'\n\talien.rect.size: {alien.rect.size} | alien_width: {alien_width} | alien_height: {alien_height} | ship_height: {ship_height}')

      print(f'\n\tempty_space: {empty_space} | alien_space: {alien_space} | alien_quantity_x: {alien_quantity_x} | rows_of_aliens: {rows_of_aliens}')

      print(f'\n\tavailable_space_x: {available_space_x} | available_space_y: {available_space_y}\n')

      '''
      - we set up a loop that counts from 0 to the number of aliens we need to make.

      - In the loop's main body, we create a new alien & then set its x-coordinate value to place it in the row.

      - Each alien is pushed to the right 1 alien width from the left margin.

      - Next, we multiply the alien width by 2 to account for the space each alien takes up,
        including the empty space to its right & we multiply this amount by the alien’s position in the row.

      - We use the alien’s x attribute to set the position of its rect.

      - Then we add each new alien to the group aliens.

      - When you run Alien Invasion now, you should see the 1st row of aliens appear

      - The 1st row is offset to the left, which is actually good for gameplay.

      - The reason is that we want the fleet to move right until it hits the edge of the screen,
        then drop down a bit, then move left & so forth.

      - Like the classic game Space Invaders, this movement is more interesting than having the fleet drop straight down.

      - We’ll continue this motion until all aliens are shot down or until an alien hits the ship or the bottom of the screen
      '''

      '''
      - To create multiple rows, we use 2 nested loops: 1 outer and 1 inner loop.

      - The inner loop creates the aliens in 1 row.

      - The outer loop counts from 0 to the number of rows we want;
        Python uses the code for making a single row and repeats it specific number of times

      - Now when we call _create_alien(), we include an argument for the row position (1st, 2nd, 3rd, etc)
        so each row can be placed farther down the screen.

      - The definition of _create_alien() needs a parameter to hold the row position.
      '''
      # the outer loop creates a specific number of rows of aliens
      for row_position in range(rows_of_aliens):
        # the inner loop creates a specific number of aliens per row
        for alien_position in range(alien_quantity_x):
          '''
          et = Alien(self)

          # 1 horizontal space + (2 horizontal spaces * 0....8) => 1 | 3 | 5 | 7 | 9 | 11 | 13 | 15 | 17
          # 60 + (120 * 0....8) => 60 | 180 | 300 | 420 | 540 | 660 | 780 | 900 |1020
          et.x = alien_width + (alien_space * alien_position)

          et.rect.x = et.x

          self.aliens.add(et)
          '''
          self._create_alien(alien_position, row_position) # create the rows & aliens

    def _create_alien(self, alien_position, row_number):
      """ Create an alien and place it in the row """
      alien = Alien(self)

      #alien_width = alien.rect.width # get an alien's width inside the method instead of passing it as an argument

      alien_width, alien_height = alien.rect.size

      # 1 horizontal space + 2 horizontal spaces * (0....8) => 1 | 3 | 5 | 7 | 9 | 11 | 13 | 15 | 17
      # 60 + 120 * (0....8) => 60 | 180 | 300 | 420 | 540 | 660 | 780 | 900 |1020
      alien.x = alien_width + 2 * alien_width * alien_position

      alien.rect.x = alien.x

      '''
      - Here we change an alien’s y-coordinate value when it’s not in the 1st row
        by starting with 1 alien’s height to create empty space at the top of the screen.

      - Each row (except the 1st row) starts 2 alien heights below the previous row,
        so we multiply the alien height by 2 & then by the row position.

      - The 1st row number is 0, so the vertical placement of the 1st row is unchanged.

      - All subsequent rows are placed farther down the screen
      '''
      # 1 vertical space + 2 vertical spaces * (0...3) => 1 | 3 | 5 | 7
      # 58 + 2 * 58 * (0....3) => 58 | 174 | 290 | 406
      alien.rect.y = alien.rect.height + 2 * alien.rect.height * row_number

      self.aliens.add(alien) # add the alien instance to the aliens group/fleet

      # checking the attributes' values
      print('\t----------------------------------------------------------------')

      print(f'\n\talien.rect.x: {alien.rect.x} | alien.rect.y: {alien.rect.y} | alien.rect.height: {alien.rect.height}\n')

    def _check_fleet_edges(self):
      " Respond appropriately if any aliens have reached an edge "
      '''
      - When an alien reaches the edge, the entire fleet needs to drop down and change direction.

      - So AlienInvasion is where we’ll add code to check if any aliens are at the left or right edge.

      - This's done by writing _check_fleet_edges() & _change_fleet_direction() & then modifying _update_aliens()

      - In this method, we loop through the fleet and call check_edges() on each alien.

      - If check_edges() returns True, an alien is at an edge and the whole fleet needs to drop down & change direction;
        so we call _change_fleet_direction() and break out of the if statement.
      '''
      for alien in self.aliens.sprites():
        if alien.check_edges():
          self._change_fleet_direction()

          break

    def _change_fleet_direction(self):
      " Drop the entire fleet and change the fleet's direction "
      '''
      - In this method, we loop thru all the aliens and drop each one using the fleet_drop_speed setting,
        then we change the value of fleet_direction by multiplying its current value by –1.

      - The line that changes the fleet’s direction isn’t part of the for loop.

      - We want to change every alien’s vertical position but the fleet's direction is changed just once
      '''
      for alien in self.aliens.sprites():
        alien.rect.y += self.settings.fleet_drop_speed

      # fleet_direction's initial value in settings.py is 1 => the fleet at 1st moves towards the right edge
      # when it reaches the right edge & drops down, fleet_direction's value * -1 = -1 => now the fleet moves towards the left edge
      # when it reaches the left edge & drops down, fleet_direction's value * -1 => -1 * -1 = 1 => the fleet once again moves towards the right edge
      # and so on
      self.settings.fleet_direction *= -1

      # checking the attribute' values
      print(f'\n\tself.settings.fleet_direction: {self.settings.fleet_direction}')

    def _update_aliens(self):
      " Update the positions of all aliens in the fleet, if the fleet is at an edge "
      '''
      - Using update() on the aliens group calls each alien’s update(), which is defined in alien.py

      - When you run alien_invasion.py now, you should see the fleet move right and disappear off the side of the screen
      '''

      '''
      - We’ve modified the method by calling _check_fleet_edges() before updating each alien’s position.

      - When you run the game now, the fleet should move back and forth between the screen's edges and
        drop down every time it hits an edge.

      - Now we can start shooting down aliens and watch for any aliens that hit the ship or reach the bottom of the screen
      '''
      self._check_fleet_edges()

      self.aliens.update()

    def _update_screen(self):
        '''
        - We moved the code that draws the background, the ship & flips the screen to _update_screen().

        - Now the body of the main loop in run_game() is much simpler.

        - It’s easy to see that we’re looking for new events and updating the screen on each pass through the loop
        '''
        self.screen.fill(self.settings.background_color)

        self.ship.blitme()

        # The bullets.sprites() method returns a list of all sprites in the bullets group
        # To draw all fired bullets to the screen, loop thru the sprites in bullets & call draw_bullet() on each one
        for bullet in self.bullets.sprites():
          bullet.draw_bullet()

        '''
        - To make the alien appear, the aliens group’s draw() method is called

        - When you call draw() on a group, Pygame draws each element in the group at the position defined by its rect attribute.

        - draw() requires 1 argument: a surface on which the elements from the group are drawn
        '''
        self.aliens.draw(self.screen)

        pygame.display.flip()

'''
- At the end of the file, we create an instance of the game and then call run_game().

- We place run_game() in an if block that only runs if the file is called directly.

- When you run this alien_invasion.py file, you should see an empty Pygame window
'''
if __name__ == '__main__':
    # make a game instance & run the game
    ai = AlienInvasion()

    ai.run_game()


# pyright: reportMissingTypeStubs=false
# pyright: reportUnknownVariableType=false

from datasets.load import load_dataset
from datasets.utils.logging import set_verbosity_error
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.tag import pos_tag
from nltk.tokenize import word_tokenize
import random
from typing import cast, Optional, TypedDict

from corpora.abstract import AbstractCorpus
from tagsets.penn_treebank import PennTreebankTagset

# Suppress `datasets` log messages.
set_verbosity_error()

class Article(TypedDict):
    id: str
    url: str
    title: str
    text: str

class WikiCorpus(AbstractCorpus):
    def __init__(self, seed: Optional[int] = None, sample_size: float = 1.0) -> None:
        super().__init__(seed, sample_size)

        self._tagset = PennTreebankTagset()

        # Load all of the articles
        dataset = load_dataset(
            "wikipedia", "20220301.simple", split="train")

        articles: list[Article] = []
        article_count = 0

        for article in dataset:
            articles.append(cast(Article, article))
            article_count += 1

        # Find the number of articles to randomly sample.
        article_count = int(article_count * self._sample_size)

        # Initialise the random number generator.
        random.seed(seed)

        # Randomly sample the articles.
        self._articles = random.sample(articles, article_count)

    _article_index = 0

    def documents(self) -> list[list[tuple[str, bool]]]:
        lemmatizer = WordNetLemmatizer()

        documents: list[list[tuple[str, bool]]] = []

        for article in self._articles:
            document: list[tuple[str, bool]] = []

            for (word, tag) in pos_tag(word_tokenize(article["text"]), tagset=None):
                # Appease the type checker.
                assert isinstance(word, str)

                if not self._tagset.is_removed_tag(tag):
                    lemma = lemmatizer.lemmatize(word.lower())
                    document.append(
                        (lemma, self._tagset.is_function_word_tag(tag)))

            documents.append(document)

        return documents


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import os

from buildconfig import topsrcdir
from mozpack import path as mozpath
from mozunit import MockedOpen, main

from common import BaseConfigureTest
from mozbuild.configure.options import InvalidOptionError

class TestToolkitMozConfigure(BaseConfigureTest):
    def test_moz_configure_options(self):
        def get_value_for(args=[], environ={}, mozconfig=""):
            sandbox = self.get_sandbox({}, {}, args, environ, mozconfig)

            # Add a fake old-configure option
            sandbox.option_impl(
                "--with-foo", nargs="*", help="Help missing for old configure options"
            )

            # Remove all implied options, otherwise, getting
            # all_configure_options below triggers them, and that triggers
            # configure parts that aren't expected to run during this test.
            del sandbox._implied_options[:]
            result = sandbox._value_for(sandbox["all_configure_options"])
            shell = mozpath.abspath("/bin/sh")
            return result.replace("CONFIG_SHELL=%s " % shell, "")

        self.assertEqual(
            "--enable-application=browser",
            get_value_for(["--enable-application=browser"]),
        )

        self.assertEqual(
            "--enable-application=browser " "MOZ_VTUNE=1",
            get_value_for(["--enable-application=browser", "MOZ_VTUNE=1"]),
        )

        value = get_value_for(
            environ={"MOZ_VTUNE": "1"},
            mozconfig="ac_add_options --enable-application=browser",
        )

        self.assertEqual("--enable-application=browser MOZ_VTUNE=1", value)

        # --disable-js-shell is the default, so it's filtered out.
        self.assertEqual(
            "--enable-application=browser",
            get_value_for(["--enable-application=browser", "--disable-js-shell"]),
        )

        # Normally, --without-foo would be filtered out because that's the
        # default, but since it is a (fake) old-configure option, it always
        # appears.
        self.assertEqual(
            "--enable-application=browser --without-foo",
            get_value_for(["--enable-application=browser", "--without-foo"]),
        )
        self.assertEqual(
            "--enable-application=browser --with-foo",
            get_value_for(["--enable-application=browser", "--with-foo"]),
        )

        self.assertEqual(
            "--enable-application=browser '--with-foo=foo bar'",
            get_value_for(["--enable-application=browser", "--with-foo=foo bar"]),
        )

    def test_developer_options(self, milestone="42.0a1"):
        def get_value(args=[], environ={}):
            sandbox = self.get_sandbox({}, {}, args, environ)
            return sandbox._value_for(sandbox["developer_options"])

        milestone_path = os.path.join(topsrcdir, "config", "milestone.txt")
        with MockedOpen({milestone_path: milestone}):
            # developer options are enabled by default on "nightly" milestone
            # only
            self.assertEqual(get_value(), "a" in milestone or None)

            self.assertEqual(get_value(["--enable-release"]), None)

            self.assertEqual(get_value(environ={"MOZILLA_OFFICIAL": 1}), None)

            self.assertEqual(
                get_value(["--enable-release"], environ={"MOZILLA_OFFICIAL": 1}), None
            )

            with self.assertRaises(InvalidOptionError):
                get_value(["--disable-release"], environ={"MOZILLA_OFFICIAL": 1})

            self.assertEqual(get_value(environ={"MOZ_AUTOMATION": 1}), None)

    def test_developer_options_release(self):
        self.test_developer_options("42.0")

if __name__ == "__main__":
    main()


import frida, sys

# 接受下面js代码中send出来的内容, 也可以在js中直接用consule.log
def on_message(message, data):
    if message['type'] == 'send':
        print("[*] {0}".format(message['payload']))
    else:
        print(message)

with open('hook_native.js') as f:
    jscode = f.read()

# app也必须先启动
device = frida.get_usb_device()
# 找到进程号
pid = device.spawn(['com.yaotong.crackme'])

print(pid)

# attach到进程号
process = device.attach(pid)
script = process.create_script(jscode)
script.on('message', on_message)
print('[*] Running CTF')
script.load()

# 直接重启程序, 重启阶段就能hook到onCreate方法
device.resume(pid)

# 输出日志
sys.stdin.read()

"""
2117
[*] Running CTF
base_address: 0xb72ae000
mod_address: 0xb7778221
lib_module: libc.so
open(/data/app/com.yaotong.crackme-1/lib/arm/libcrackme.so, 0x0)
retval: 0x0
open(libc.so, 0x0)
retval: 0xb774fe5c
open(libc.so, 0x0)
retval: 0xb774fe5c
open(libz.so, 0x0)
retval: 0xb6110004
open(libc.so, 0x0)
retval: 0xb774fe5c
open(libc.so, 0x0)
retval: 0xb774fe5c
open(/system/lib/hw/gralloc.x86.so, 0x0)
retval: 0xb2fa7664
open(/system/lib/egl/libEGL_emulation.so, 0x0)
retval: 0xb2fa7e5c
open(/system/lib/egl/libGLESv1_CM_emulation.so, 0x0)
retval: 0xb26d6004
open(/system/lib/egl/libGLESv2_emulation.so, 0x0)
retval: 0xb26d619c
open(/system/lib/egl/libGLESv1_CM_emulation.so, 0x0)
retval: 0xb26d6004
open(/system/lib/egl/libGLESv2_emulation.so, 0x0)
retval: 0xb26d619c
open(/system/lib/libEGL.so, 0x0)
retval: 0xb76b17fc
open(/system/lib/libGLESv2.so, 0x0)
retval: 0xb76b1b2c
open(/system/lib/libGLESv1_CM.so, 0x0)
retval: 0xb76b1994
"""


from django.conf.urls import url
from src import views

app_name='src'

urlpatterns = [

	url(r'^items/$', views.items, name='items'),
	url(r'^share-menu$', views.ShareMenu.as_view(), name='share-menu'),
	url(r'^$', views.MenuView.as_view(), name='menu'),
]


"""
This code borrows trains MALLET topic model(s) on combined article text data, saving all the outputs of the model, and also adding the topic probabilities to the combined data file. And saving the url to topic distribution mapping itself. Topics themselves will be interpreted and displayed (in terms of top words and top documents) in interpret_topic_modeling_output.ipynb.

Attributes:
    INPUT_DEDUP_FILES_BASE_PATH: Directory path to the deduplicated data files of all media groups.
    OUTPUT_COMBINED_DATAFILE_PATH: File path to save csv file containing all the articles across all media groups.
    MALLET_PATH: Path to mallet exe script needed for topic modeling functionality (clone: https://github.com/mimno/Mallet and built using ant)
    NUM_TOPIC_VALUES: List of values of K or number of topics to use for running mallet.
    MALLET_OUTPUT_PATH: Path to contain all the various outputs created by mallet during topic modeling training.
    OUTPUT_DATA_WITH_TOPIC_DISTRIBUTION_BASEPATH: Path to save the various files that include doc-topic distributions for articles.

Functions:
    create_and_save_combined_dataframe: Loads all the dedup files representing various media groups (the 6 states + nytimes+foxnews) and concatenates and saves them as one dataframe, while adding a column for the media group.
    topic_model_training: Using the full dataframe, obtains the texts (article headlines and stories), preprocesses them to create training data, and trains various topic model for different values of K or num_topics.
    add_topic_distribution_to_combined_data_csv: For a specified value of K (num_topics), loads the doc-topic distributions, and saves that as added columns to the csv data.
    save_url_to_topic_distributions_mapping: For a specified value of K (num_topics), loads the doc-topic distributions, and saves that as URL -> list of topic distribution values.

@author: Pranav Goel
"""

import os
import pickle

import little_mallet_wrapper as lmw  # https://github.com/maria-antoniak/little-mallet-wrapper/tree/master
import numpy as np
import pandas as pd
from tqdm.auto import tqdm

INPUT_DEDUP_FILES_BASE_PATH = (
    "/home/pranavgoel/trans-fer-entropy/obtaining_news_collections/data/"
)
OUTPUT_COMBINED_DATAFILE_PATH = (
    "/home/pranavgoel/trans-fer-entropy/topic_modeling/all_texts_combined.csv"
)

MALLET_PATH = "/home/pranavgoel/Mallet/bin/mallet"

NUM_TOPIC_VALUES = [10, 20]

MALLET_OUTPUT_PATH = (
    "/home/pranavgoel/mallet_output/trans_fer_entropy_first_full_data_tm"
)
OUTPUT_DATA_WITH_TOPIC_DISTRIBUTION_BASEPATH = (
    "/home/pranavgoel/trans-fer-entropy/topic_modeling/"
)

def create_and_save_combined_dataframe(input_file_names):
    """
    Loads all the dedup files representing various media groups (the 6 states + nytimes+foxnews) and concatenates and saves them as one dataframe, while adding a column for the media group.
    """
    all_dfs = []
    for f in input_file_names:
        df = pd.read_csv(INPUT_DEDUP_FILES_BASE_PATH + f)
        media_group = f.split("_article_")[0]
        df["media_group"] = [media_group] * len(df)
        all_dfs.append(df)
    combined_df = pd.concat(all_dfs)
    combined_df.to_csv(OUTPUT_COMBINED_DATAFILE_PATH, index=None)
    return combined_df

def topic_model_training(df):
    """
    Using the full dataframe, obtains the texts (article headlines and stories), preprocesses them to create training data, and trains various topic model for different values of K or num_topics.
    """
    titles_and_texts = list(zip(df["title"], df["text"]))
    titles_and_texts = list(map(lambda x: x[0] + "\n\n\n" + x[1], titles_and_texts))
    training_data = [
        lmw.process_string(t, numbers="remove") for t in tqdm(titles_and_texts)
    ]
    training_data = [d for d in training_data if d.strip()]
    path_to_training_data = MALLET_OUTPUT_PATH + "/training.txt"
    path_to_formatted_training_data = MALLET_OUTPUT_PATH + "/mallet.training"

    lmw.import_data(
        MALLET_PATH,
        path_to_training_data,
        path_to_formatted_training_data,
        training_data,
    )

    for num_topics in NUM_TOPIC_VALUES:
        print("\n -- -- -- number of topics = " + str(num_topics) + " -- -- --\n")
        path_to_model = MALLET_OUTPUT_PATH + "/mallet.model." + str(num_topics)
        path_to_topic_keys = (
            MALLET_OUTPUT_PATH + "/mallet.topic_keys." + str(num_topics)
        )
        path_to_topic_distributions = (
            MALLET_OUTPUT_PATH + "/mallet.topic_distributions." + str(num_topics)
        )
        path_to_word_weights = (
            MALLET_OUTPUT_PATH + "/mallet.word_weights." + str(num_topics)
        )
        path_to_diagnostics = (
            MALLET_OUTPUT_PATH + "/mallet.diagnostics." + str(num_topics) + ".xml"
        )

        lmw.train_topic_model(
            MALLET_PATH,
            path_to_formatted_training_data,
            path_to_model,
            path_to_topic_keys,
            path_to_topic_distributions,
            path_to_word_weights,
            path_to_diagnostics,
            num_topics,
        )

def add_topic_distribution_to_combined_data_csv(df, num_topics):
    """
    For a specified value of K (num_topics), loads the doc-topic distributions, and saves that as added columns to the csv data.
    """
    topic_distributions = lmw.load_topic_distributions(
        MALLET_OUTPUT_PATH + "/mallet.topic_distributions." + str(num_topics)
    )
    topic_distributions = np.array(topic_distributions)
    for topic_ind in range(num_topics):
        df["Topic " + str(topic_ind)] = list(topic_distributions[:, topic_ind])
    df.to_csv(
        OUTPUT_DATA_WITH_TOPIC_DISTRIBUTION_BASEPATH
        + "all_texts_combined_with_topic_distribution_values_for_num_topics_"
        + str(num_topics)
        + ".csv"
    )

def save_url_to_topic_distributions_mapping(df, num_topics):
    """
    For a specified value of K (num_topics), loads the doc-topic distributions, and saves that as URL -> list of topic distribution values.
    """
    topic_distributions = lmw.load_topic_distributions(
        MALLET_OUTPUT_PATH + "/mallet.topic_distributions." + str(num_topics)
    )
    topic_distributions = np.array(topic_distributions)
    out = {}
    urls = list(df["url"])
    for i, url in tqdm(enumerate(urls)):
        out[url] = list(topic_distributions[i, :])
    pickle.dump(
        out,
        open(
            OUTPUT_DATA_WITH_TOPIC_DISTRIBUTION_BASEPATH
            + "url_to_topic_distribution_for_num_topics_"
            + str(num_topics)
            + ".pkl",
            "wb",
        ),
    )

if __name__ == "__main__":
    input_file_names = [
        x for x in os.listdir(INPUT_DEDUP_FILES_BASE_PATH) if "dedup.csv" in x
    ]
    combined_df = create_and_save_combined_dataframe(input_file_names)
    topic_model_training(combined_df)
    for num_topics in NUM_TOPIC_VALUES:
        add_topic_distribution_to_combined_data_csv(combined_df, num_topics)
        save_url_to_topic_distributions_mapping(combined_df, num_topics)


import sys
import os

from src.wix_api import Products, Orders, Inventory
from src.utils import sku_replace, sku_frames_to_clear, setup_logger, send_email
import logging
from pprint import pprint

class StockManager:
    def __init__(self):
        self.sku_replace = sku_replace
        self.ord = Orders()
        self.sg_api_key = os.getenv("SG_API_KEY")
        assert self.sg_api_key, "Sendgrid API key not found"
        self.logger = setup_logger("stck_mngr", "logs/stck_mngr.log")

    def get_order_details(self, orderId):
        try:
            order = self.ord.get_order(orderId)
            name_details = (
                order["order"]["buyerInfo"]["firstName"],
                order["order"]["buyerInfo"]["lastName"],
            )
            order_stock_details = []
            for i in order["order"]["lineItems"]:
                if i["lineItemType"] != "PHYSICAL":
                    self.logger.info(
                        i["name"]
                        + " is not a physical product, so this will not be processed"
                    )
                else:
                    order_stock_details.append((i["sku"], i["quantity"]))
            return name_details, order_stock_details
        except Exception as e:
            self.logger.error(
                f"Error getting SKU and quantity for order {orderId}: {str(e)}"
            )

    def prep_inventory_email(
        self, orderId, product_details, name_details, status, update_status_idx=None
    ):
        """
        Prepares the email to be sent for the inventory update status.
        """
        message = f"
Order {orderId} was processed with status {status}.
"
        message += f"
Order details:
"
        message += f"
Name: {name_details[0]} {name_details[1]}
"
        message += f"
Products:
"
        for product, quantity in product_details:
            message += f"
{product[0]} | {product[-1]}: -{quantity}
"
        if update_status_idx is not None:
            message += f"
Failed to update inventory for the following products:
"
            for idx in update_status_idx:
                product_name = product_details[idx][0][0]
                choices = '-'.join(product_details[idx][0][-1].values()) if product_details[idx][0][-1] is not None else None
                quantity = product_details[idx][1]
                if choices is None:
                    message += f"
{product_name}: {quantity}
"
                else:
                    message += f"
{product_name} | {choices}: {quantity}
"
            message += "
Signed by The Naos bot
"
        subject = (
            f"[{status}] Inventory Update Status - {name_details[0]} {name_details[1]}"
        )
        return message, subject

class SKUHandler(StockManager):
    def __init__(self):
        super().__init__()
        self.prod = Products()
        self.sku_mapping = {}
        self._build_sku_mapping()

    def _build_sku_mapping(self):
        try:
            # get all products
            products = self.prod.query_products(
                includeVariants=True, includeHiddenProducts=True
            )
            i = 0
            for product in products["products"]:
                if product["manageVariants"] == True:
                    # as soon as there is one composite key, we know it's a composite product and don't need to store the individual sku
                    composite = any(["-" in variant["variant"]["sku"] for variant in product["variants"]])
                    if not composite:
                        for variant in product["variants"]:
                            self.sku_mapping[variant["variant"]["sku"]] = (
                                product["name"],
                                product["id"],
                                variant["id"],
                                variant["choices"],
                            )
                elif ((product["sku"] != "")
                    & ("-" not in product["sku"])
                ):
                    self.sku_mapping[product["sku"]] = (
                        product["name"],
                        product["id"],
                        None,
                        None,
                    )
                else:
                    self.logger.warning(f"product {product['name']} has no sku")
        except Exception as e:
            self.logger.error(f"Error getting SKU mapping: {str(e)}")

    def process_sku(self, sku):
        if sku in self.sku_mapping:
            return self.sku_mapping[sku]
        elif sku in self.sku_replace.keys():
            return self.sku_mapping[self.sku_replace[sku]]
        else:
            self.logger.error(f"Sku {sku} not found in  sku mapping")

def main():
    sku_handler = SKUHandler()
    sm = StockManager()
    inv = Inventory()
    order_id = input("Enter order id: ")
    name_details, order_stock_details = sm.get_order_details(order_id)
    if order_stock_details is None:
        sm.logger.error(f"failed to get details for order {order_id}")
    product_details = []
    for concat_sku, quantity in order_stock_details:
        sku_parts = concat_sku.split("-")
        for sku in sku_parts:
            product = sku_handler.process_sku(sku)
            if product is None:
                sm.logger.error(f"failed to get product details for sku {sku}")
                continue
            if sku in sku_frames_to_clear.keys(): # adds spare lens if needed
                clear_lens = sku_handler.process_sku(sku_frames_to_clear[sku])
                product_details.append((clear_lens, quantity))
            product_details.append((product, quantity))

    update_status_idx = []
    i = 0
    for product, quantity in product_details:
        product_name, productid, variantid, choices = product[0], product[1], product[2], "-".join(product[3].values()) if product[3] is not None else None
        _, response = inv.decrease_inventory(productid, variantid, quantity)
        if response.status_code == 200:
            sm.logger.info(
                f"Successfully updated inventory for product {product_name} | {choices}"
            )
        else:
            update_status_idx.append(i)
            i += 1
            logging.error(
                f"Failed to update inventory for product {product_name} | {choices} with status code {response.status_code}"
            )
    if len(update_status_idx) == 0:
        msg, sub = sm.prep_inventory_email(
            order_id, product_details, name_details, status="SUCCESS"
        )
        send_email(
            msg,
            sub,
            "info@naos-optics.com",  # public facing email
            "info@naos-optics.com",  # public facing email
            sm.sg_api_key,
            sm.logger,
        )
    else:
        msg, sub = sm.prep_inventory_email(
            order_id,
            product_details,
            name_details,
            status="FAIL",
            update_status_idx=update_status_idx,
        )
        send_email(
            msg,
            sub,
            "info@naos-optics.com",  # public facing email
            "info@naos-optics.com",  # public facing email
            sm.sg_api_key,
            sm.logger,
        )

if __name__ == "__main__":
    main()


from flask import (
    Blueprint, flash, g, redirect, render_template, request, url_for
)

from werkzeug.exceptions import abort

from flaskr.auth import login_required
from flaskr.db import get_db

bp = Blueprint('blog', __name__) # Unlike the auth blueprint, the blog blueprint does not have a url_prefix. So the index view will be at /, the create view at /create, and so on. The blog is the main feature of Flaskr, so it makes sense that the blog index will be the main index.

@bp.route('/')
def index():
    db = get_db()
    posts = db.execute(
        'SELECT p.id, title, body, created, author_id, username'
        ' FROM post p JOIN user u ON p.author_id = u.id'
        ' ORDER BY created DESC'
    ).fetchall()
    return render_template('blog/index.html', posts=posts)

@bp.route('/create', methods=('GET', 'POST'))
@login_required
def create():
    if request.method == 'POST':
        title = request.form['title']
        body = request.form['body']
        error = None

        if not title:
            error = 'Title is required.'

        if error is not None:
            flash(error)
        else:
            db = get_db()
            db.execute(
                'INSERT INTO post (title, body, author_id)'
                ' VALUES (?, ?, ?)',
                (title, body, g.user['id'])
            )
            db.commit()
            return redirect(url_for('blog.index'))

    return render_template('blog/create.html')

def get_post(id, check_author=True):
    post = get_db().execute(
        'SELECT p.id, title, body, created, author_id, username'
        ' FROM post p JOIN user u ON p.author_id = u.id'
        ' WHERE p.id = ?',
        (id,)
    ).fetchone()

    if post is None:
        abort(404, "Post id {0} does not exist.".format(id))

    if check_author and post['author_id'] != g.user['id']:
        abort(403)

    return post

@bp.route('/<int:id>/update', methods=('GET', 'POST')) # Unlike the views you’ve written so far, the update function takes an argument, id. That corresponds to the <int:id> in the route. A real URL will look like /1/update. Flask will capture the 1, ensure it’s an int, and pass it as the id argument. If you don’t specify int: and instead do <id>, it will be a string. To generate a URL to the update page, url_for() needs to be passed the id so it knows what to fill in: url_for('blog.update', id=post['id']).
@login_required
def update(id):
    post = get_post(id)

    if request.method == 'POST':
        title = request.form['title']
        body =  request.form['body']
        error = None

        if not title:
            error = 'Title is required.'

        if error is not None:
            flash(error)
        else:
            db = get_db()
            db.execute(
                'UPDATE post SET title = ?, body = ?'
                ' WHERE id = ?',
                (title, body, id)
            )
            db.commit()
            return redirect(url_for('blog.index'))

    return render_template('blog/update.html', post=post) # id from post used here too

@bp.route('/<int:id>/delete', methods=('POST',))
@login_required
def delete(id):
    get_post(id)
    db = get_db()
    db.execute('DELETE FROM post WHERE id = ?', (id,))
    db.commit()
    return redirect(url_for('blog.index'))


from datasets import DatasetDict, load_dataset
from tqdm.auto import tqdm

ds = load_dataset("bigbio/ebm_pico")
print(ds)

ds = ds.map(
    lambda x: {
        "entity": [s["text"] for s in x["entities"]],
        "annotation_type": [s["annotation_type"] for s in x["entities"]],
    }
)
print(ds)

def prepare_labels(ds):
    all_labels = []
    for sample in tqdm(ds):
        entities = sample["entity"]
        e2a = {e: a for e, a in zip(entities, sample["annotation_type"])}
        hased_entities = set(entities)
        labels = []
        for word in sample["text"].split():
            if word in hased_entities:
                labels.append(e2a[word])
            else:
                labels.append("<ignore>")
        all_labels.append(labels)
    ds = ds.map(lambda x, i: {"labels": all_labels[i]}, with_indices=True)
    ds = ds.map(lambda x: {"tokens": x["text"].split()})

    # for sample in tqdm(ds):
    #     assert len(ds["tokens"]) == len(ds["labels"])
    return ds

ds = DatasetDict(
    {
        "train": prepare_labels(ds["train"]),
        "test": prepare_labels(ds["test"]),
    }
)
print(ds)

push_to_hub = True

if push_to_hub:
    ds.push_to_hub("ddp-iitm/ebm_pico", private=True)

# from datasets import load_dataset
# ds = load_dataset("ddp-iitm/ebm_pico")