"""Functions for reviewing HITs"""

from hit_info import *
from hit_results import *
import log
import create
import save
import email_worker
import logging
import pandas as pd
import boto3
import xmltodict
import random

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

def get_pending_hit_ids(annotation_log, dataset_filter=None):
    """Returns a list of all HIT ids whose results have not been logged

    Parameters
    ----------
    annotation_log : str
        Path to annotations log file
    dataset_filter : str
        String of dataset that only its pending HITs will be fetched

    Returns
    -------
    list
        The list of pairs of pending HIT ids, and their type
    """
    # return HIT id of HITs without results yet
    pending_hits = []
    df = pd.read_csv(annotation_log)
    df = df.set_index('annotation_id')
    pending_gen_df = df[~df['hitid_gen'].isnull() & df['decomposition'].isnull() & df['decomposition_accepted'].isnull()]
    pending_val_df = df[(~df['hitid_validation'].isnull()) & (~df['hitid_validation'].isin([log.MANUAL_VALIDATION, log.PENDING_MANUAL_VALIDATION])) & \
                        (df['validation_annotation_1'].isnull() | df['validation_annotation_2'].isnull())]

    if dataset_filter != None:
        # filter only pending HITs of specified dataset
        pending_gen_df = pending_gen_df[pending_gen_df['question_id'].str.contains(dataset_filter)]
        pending_val_df = pending_val_df[pending_val_df['question_id'].str.contains(dataset_filter)]
    gen_hit_ids = pending_gen_df['hitid_gen'].values.tolist()
    val_hit_ids = pending_val_df['hitid_validation'].values.tolist()
    for hit_id in gen_hit_ids:
        pending_hits += [(hit_id, 'gen')]
    for hit_id in val_hit_ids:
        pending_hits += [(hit_id, 'val')]
    # randomly shuffle HITs for manual review process
    random.shuffle(pending_hits)
    return pending_hits

def review_pending_hits(manual_review, client, annotation_log, bank_annotations, save_results_to_json=False, json_dir=None):
    """Choose to automatically or manually review all pending HIT results.
        Either automatically return all pending HIT results and update the annotation log files,
        or manually review HIT assignment results and accept/reject and save them.

    Parameters
    ----------
    manual_review : bool
        Flag whether the HIT results will be manually reviewed, or results saved automatically
    client : MTurk.Client
        A boto3 client for MTurk
    annotation_log : str
        Path to annotations log file
    bank_annotations : str
        Path to bank annotations log files for validation HIT tests
    save_results_to_json : bool
        Flag indicating whether to save HITResults of pending HITs to json files
    json_dir : str
        Path to directory where published HIT json files are to be saved

    Returns
    -------
    bool
        returns True if successfully terminated
    """
    # get all pending HIT ids
    dataset = None ####'HOTPOT_' ####None # review only HOTPOT HITs
    pending_hits = get_pending_hit_ids(annotation_log, dataset)
    hits_results_saved = 0
    # automatic review - save all HIT results and finish
    if not manual_review:
        # go over all HITs
        for hit in pending_hits:
            # get HIT information
            hit_id, hit_type = hit
            hit_info = create.get_hit_info(annotation_log=annotation_log, bank_annotations=bank_annotations,\
                                    annotation_id=None, hit_type=hit_type, hit_id=hit_id)
            # get HIT results, if any
            hit_results_list = get_hit_results(client, hit_info.type, hit_info.hit_id)
            if len(hit_results_list) > 0:
                for hit_result in hit_results_list:
                    res = save.save_hit_results(hit_info, hit_result, annotation_log, save_results_to_json, json_dir)
                    assert(res == True)
                    hits_results_saved += 1
        logger.info(f'* Reviewed (automatically) and saved {hits_results_saved} HIT results')
    # manual review - review each pending HIT
    else:
        continue_review = None
        review_batch_size = 50
        batch_iteration = 0
        while continue_review != False:
            pending_hit_info_list = []
            # create a btach of HITs, watch for the end of the list
            limit = min((batch_iteration+1)*review_batch_size, len(pending_hits)-1)
            if limit == len(pending_hits)-1:
                # final batch, quit review afterwards
                continue_review = False
            for hit in pending_hits[batch_iteration*review_batch_size:limit]:
                # get HIT information
                hit_id, hit_type = hit
                hit_info = create.get_hit_info(annotation_log=annotation_log, bank_annotations=bank_annotations,\
                                        annotation_id=None, hit_type=hit_type, hit_id=hit_id)
                pending_hit_info_list += [hit_info]
            # increase batch iteration
            batch_iteration += 1
            # review a batch of HITs
            for hit_info in pending_hit_info_list:
                hit_results_list = get_hit_results(client, hit_info.type, hit_info.hit_id)
                if len(hit_results_list) > 0:
                    # retreive HIT results
                    for hit_result in hit_results_list:
                        result = review_hit_results(client, annotation_log, hit_info, hit_result, json_dir)
                        assert(result != None)
                        hits_results_saved += 1
                    # ask whether to continue
                    user_input = input('Continue manual review of pending HITs? [y/n]').strip().lower()
                    if user_input in ['y', 'n']:
                        continue_review = user_input == 'y'
                    else:
                        print('Please type either "y" or "n".')
                    if not continue_review:
                        break
        logger.info(f'* Reviewed (manually) and saved {hits_results_saved} HIT results')
    return True

def get_hit_results(client, hit_type, hit_id):
    """Returns the results of the specified HIT

    Parameters
    ----------
    client : MTurk.Client
        A boto3 client for MTurk
    hit_type : str
        String indicating if HIT is of type generation or validation
    hit_id : str
        Id of the relevant HIT

    Returns
    -------
    list
        list of GenerationResults/ValidationResults objects representing the relevant HIT results
    """
    assert(hit_type in ['gen', 'val'])
    assignments_results = []
    worker_results = client.list_assignments_for_hit(HITId=hit_id, AssignmentStatuses=['Submitted'])
    num_results = worker_results['NumResults']
    if num_results == 0:
        # If no results returned for 'Submitted' status, check if there are any for 'Approved' status
        worker_results = client.list_assignments_for_hit(HITId=hit_id, AssignmentStatuses=['Approved'])
        num_results = worker_results['NumResults']
    if num_results > 0:
        for assignment in worker_results['Assignments']:
            result = None
            # general assignment info
            assignment_id = assignment['AssignmentId']
            worker_id = assignment['WorkerId']
            submit_time = assignment['SubmitTime']
            # assignment answer
            decomposition = annotation_validation = bank_validation = None
            xml_doc = xmltodict.parse(assignment['Answer'])
            logger.info(f'* Returning results for HIT {hit_id}: found {num_results} results.')
            # Multiple fields in HIT layout
            if type(xml_doc['QuestionFormAnswers']['Answer']) is list:
                for answer_field in xml_doc['QuestionFormAnswers']['Answer']:
                    if hit_type == 'gen':
                        # generation HIT results
                        if answer_field['QuestionIdentifier'] == 'surveycode':
                            decomposition = answer_field['FreeText']
                    else:
                        # validation HIT results
                        if answer_field['QuestionIdentifier'] == 'ant':
                            annotation_validation_str = answer_field['FreeText'].lower().strip()
                            annotation_validation = 1 if (annotation_validation_str == 'yes') else 0
                        elif answer_field['QuestionIdentifier'] == 'bnk':
                            bank_validation_str = answer_field['FreeText'].lower().strip()
                            bank_validation = 1 if (bank_validation_str == 'yes') else 0
            # One field found in HIT layout
            else:
                raise Exception('HIT results should have more than one result field!')
            # Build assignment result
            if hit_type == 'gen':
                result = GenerationResults(hit_id, assignment_id, worker_id, submit_time, decomposition)
                logger.info(f'* Returning generation results for HIT: {hit_id}, decomposition: {decomposition}.')
            else:
                result = ValidationResults(hit_id, assignment_id, worker_id, submit_time, annotation_validation, bank_validation)
                logger.info(f'* Returning validation results for HIT: {hit_id}, annotation val: {annotation_validation}, bank val: {bank_validation}.')
            assignments_results += [result]
    else:
        logger.info(f'* Returning results for HIT {hit_id}: no results were found.')
    return assignments_results

def review_hit_results(client, annotation_log, hit_info, hit_results, json_dir):
    """Manually review HIT assignment results, accept/reject and save them.

    Parameters
    ----------
    client : MTurk.Client
        A boto3 client for MTurk
    annotation_log : str
        Path to annotations log file
    hit_info : GenerationHITInfo / ValidationHITInfo
        HITInfo object storing information regarding the HIT
    hit_results : GenerationResults / ValidationResults
        HIT results object storing the results of the relevant HIT
    json_dir : str
        Path to directory where published HIT json files are to be saved

    Returns
    -------
    GenerationResults / ValidationResults
        returns the HIT results possibly manually validated (if generation HIT)
    """
    assignment_id = hit_results.assignment_id
    hit_type = hit_results.type
    hit_id = hit_results.hit_id
    annotation_id = hit_info.annotation_id
    assert(hit_type in ['gen', 'val'])
    # generation HIT
    if hit_type == 'gen':
        display_generation_hit_results(hit_info, hit_results)
        # would you like to manaully validate the HIT?
        validate = None
        while validate is None:
            user_input = input('Manually validate? [y/n]').strip().lower()
            if user_input in ['y', 'n']:
                validate = user_input == 'y'
            else:
                print('Please type either "y" or "n".')
        if validate:
            logger.info(f'Manually validating (HIT id: {hit_id}, Assignment id: {assignment_id}).')
            hit_results = validate_annotation(hit_results)
        else:
            logger.info(f'Did not manually validate (HIT id: {hit_id}, Assignment id: {assignment_id}).'
                        f'Please make sure to publish a corresponding validation HIT in the future.')
    # validation HIT
    else:
        display_validation_hit_results(hit_info, hit_results)
    # Would you like to accept/reject the HIT?
    accept = None
    while accept is None:
        user_input = input('Manually accept HIT assignment? [accept/reject]').strip().lower()
        if user_input in ['accept', 'reject']:
            accept = user_input == 'accept'
        else:
            print('Please type either "accept" or "reject".')
    if accept:
        hit_results.accept()
        accept_hit(client, assignment_id, "Accepted HIT.")
        logger.info(f'Manually accepted (HIT id: {hit_id}, Assignment id: {assignment_id}).')
    else:
        hit_results.reject()
        reject_hit(client, assignment_id, "Worker results did not comply with the assignment instructions.")
        logger.info(f'Manually rejected (HIT id: {hit_id}, Assignment id: {assignment_id}).')
    # Would you like to save HIT results in a separate json file?
    save_to_json = True ##### None
    while save_to_json is None:
        user_input = input('Save HIT results to JSON file? [y/n]').strip().lower()
        if user_input in ['y', 'n']:
            save_to_json = user_input == 'y'
        else:
            print('Please type either "y" or "n".')
    if save_to_json:
        logger.info(f'Saving HIT results to: {json_dir}/{hit_id}_{assignment_id}.json (HIT id: {hit_id}, Assignment id: {assignment_id}).')
    # Update logs files
    logger.info(f'Updating annotaion & hit log files (HIT id: {hit_id}, Assignment id: {assignment_id}, Annotation id: {annotation_id}).')
    save.save_hit_results(hit_info, hit_results, annotation_log, create_json=save_to_json, json_dir=json_dir)
    # Would you like to email worker regarding the results?
    send_email_worker = None
    while send_email_worker is None:
        user_input = input('Email worker? [y/n]').strip().lower()
        if user_input in ['y', 'n']:
            send_email_worker = user_input == 'y'
        else:
            print('Please type either "y" or "n".')
    if send_email_worker:
        message = format_hit_review_email(hit_info, hit_results)
        print(message)
        user_input = input('Enter additional email review:').strip()
        message += "\n"+user_input
        subject = "Review feedback on HIT id: " + hit_id
        worker_id = hit_results.worker_id
        workers = [worker_id]
        email_worker.email_mturk_workers(client, subject, message, workers)
        logger.info(f'Emailed worker {worker_id} on HIT results (HIT id: {hit_id}, Assignment id: {assignment_id}).')
    return hit_results

def format_hit_review_email(hit_info, hit_results):
    """Returns formatted string to be sent as a review to the worker

    Parameters
    ----------
    hit_info : GenerationHITInfo / ValidationHITInfo
        HITInfo object storing information regarding the HIT
    hit_results : GenerationResults / ValidationResults
        HIT results object storing the results of the relevant HIT

    Returns
    -------
    str
        returns the formatted review string
    """
    review = "Dear worker, \nPlease read the following review of your results on HIT: " + hit_results.hit_id
    if hit_results.type == 'gen':
        review += "\n\nOriginal question: " + "[" + hit_info.question_text + "]"
        review += "\n\nYour decomposition: "
        decomposition = hit_results.decomposition
        formatted_dec = format_decomposition_string(decomposition)
        review += formatted_dec
    else:
        # for the bank decomposition
        review += "\n\nYour answers: "
        bank_dec = hit_info.bank_annotation.decomposition
        formatted_bank_dec = format_decomposition_string(bank_dec)
        bank_valid = hit_results.bank_validation
        bank_verdict = "correct" if bank_valid else "incorrect"
        first_val = "\n 1. Marked the following decomposition as :" + bank_verdict + formatted_bank_dec
        review += first_val
        # for the annotated decomposition
        annotated_dec = hit_info.decomposition
        formatted_annot_dec = format_decomposition_string(annotated_dec)
        annotated_dec_valid = hit_results.annotation_validation
        annot_verdict = "correct" if annotated_dec_valid else "incorrect"
        second_val = "\n 2. Marked the following decomposition as :" + annot_verdict + formatted_annot_dec
        review += second_val
    return review

def display_generation_hit_results(hit_info, hit_results):
    """Displays the results of a generation HIT

    Parameters
    ----------
    hit_info : GenerationHITInfo
        HITInfo object storing information regarding the HIT
    hit_results : GenerationResults
        HIT results object storing the results of the relevant HIT

    Returns
    -------
    bool
        returns True
    """
    dec_string = format_decomposition_string(hit_results.decomposition)
    print(
        'HIT ID: {hit_id}'
        '\nAssignment ID: {assignment_id}'
        '\nHIT Type: Generation'
        '\n'
        '\nResults'
        '\n======='
        '\nAnnotation ID: {annotation_id}'
        '\nQuestion ID: {question_id}'
        '\nQuestion Text: {question_text}'
        '\nDecomposition: {decomposition}'.format(
                            hit_id=hit_results.hit_id,
                            assignment_id=hit_results.assignment_id,
                            annotation_id=hit_info.annotation_id,
                            question_id=hit_info.question_id,
                            question_text=hit_info.question_text,
                            decomposition=dec_string))
    return True

def display_validation_hit_results(hit_info, hit_results):
    """Displays the results of a validation HIT

    Parameters
    ----------
    hit_info : ValidationHITInfo
        HITInfo object storing information regarding the HIT
    hit_results : ValidationResults
        HIT results object storing the results of the relevant HIT

    Returns
    -------
    bool
        returns True
    """
    bank_annotation = hit_info.bank_annotation
    dec_string_annotation = format_decomposition_string(hit_info.decomposition)
    dec_string_bank = format_decomposition_string(bank_annotation.decomposition)
    print(
        'HIT ID: {hit_id}'
        '\nAssignment ID: {assignment_id}'
        '\nHIT Type: Validation'
        '\n'
        '\nResults'
        '\n======='
        '\nAnnotation ID: {annotation_id}'
        '\nQuestion ID: {question_id}'
        '\nQuestion Text: {question_text}'
        '\nDecomposition: {decomposition}'
        '\nValidator marked as correct: {annotation_validation}'
        '\n======='
        '\nBank Annotation ID: {bank_annotation_id}'
        '\nBank ID: {bank_question_id}'
        '\nBank Question Text: {bank_question_text}'
        '\nBank Decomposition: {bank_decomposition}'
        '\nIs correct: {decomposition_correct}'
        '\nValidator marked as correct: {bank_validation}'.format(
                            hit_id=hit_results.hit_id,
                            assignment_id=hit_results.assignment_id,
                            annotation_id=hit_info.annotation_id,
                            question_id=hit_info.question_id,
                            question_text=hit_info.question_text,
                            decomposition=dec_string_annotation,
                            annotation_validation=hit_results.annotation_validation,
                            bank_annotation_id=bank_annotation.annotation_id,
                            bank_question_id=bank_annotation.question_id,
                            bank_question_text=bank_annotation.question_text,
                            bank_decomposition=dec_string_bank,
                            decomposition_correct=bank_annotation.decomposition_correct,
                            bank_validation=hit_results.bank_validation))
    return True

def accept_hit(client, assignment_id, feedback):
    """Accepts a HIT assignment on MTurk

    Parameters
    ----------
    client : MTurk.Client
        A boto3 client for MTurk
    assignment_id : string
        The ID of the assignment. The assignment must correspond to a HIT created by the Requester.
    feedback: string
        A message for the Worker, which the Worker can see in the Status section of the web site.

    Returns
    -------
    bool
        returns whether the operation was successful
    """
    response = client.approve_assignment(
        AssignmentId=assignment_id,
        RequesterFeedback=feedback)
    return response=={}

def reject_hit(client, assignment_id, feedback):
    """Rejects a HIT assignment on MTurk

    Parameters
    ----------
    client : MTurk.Client
        A boto3 client for MTurk
    assignment_id : string
        The ID of the assignment. The assignment must correspond to a HIT created by the Requester.
    feedback: string
        A message for the Worker, which the Worker can see in the Status section of the web site.

    Returns
    -------
    bool
        returns whether the operation was successful
    """
    response = client.reject_assignment(
        AssignmentId=assignment_id,
        RequesterFeedback=feedback)
    return response=={}

def validate_annotation(generation_hit_results):
    """Manually validate a generation HIT annotation

    Parameters
    ----------
    generation_hit_results : GenerationResults
        results of the generation HIT

    Returns
    -------
    GenerationResults
        Returns manually validated results
    """
    valid_annotation = None
    while valid_annotation is None:
        user_input = input('Annotation valid? [y/n]').strip().lower()
        if user_input in ['y', 'n']:
            valid_annotation = user_input == 'y'
        else:
            print('Please type either "y" or "n".')
    generation_hit_results.validate(valid_annotation)
    logger.info(f'* Manually validated generation HIT to {valid_annotation} (HIT id: {generation_hit_results.hit_id}, Assignment id: {generation_hit_results.assignment_id})')
    return generation_hit_results

def format_decomposition_string(decomposition_string, delimiter=";"):
    """Returns a formatted text string representation of a query decomposition
        for manually reviewing HIT results.

    Parameters
    ----------
    decomposition_string : str
        decompositiong string with delimited steps
    delimiter : str
        delimiter at the end of decomposition steps

    Returns
    -------
    str
        Returns formatted text string, numbered steps, each step in a new line
    """
    text_decomposition = ""
    # remove last delimiter
    if decomposition_string[-1] == delimiter:
        decomposition_string = decomposition_string[:-1]
    # break into steps
    steps = decomposition_string.split(delimiter)
    for i in range(len(steps)):
        text_decomposition += '\n\t' + str(i+1) + '. ' + steps[i]
    return text_decomposition


import sys
sys.stdin = open('11967.txt')
import collections

dr = [0, 0, -1, 1]
dc = [-1, 1, 0, 0]
def bfs():
    deq = collections.deque()
    deq.append((1, 1))
    visit = [[0 for _ in range(N + 1)] for _ in range(N + 1)]
    visit[1][1] = 1
    result = 1
    while deq:
        row, col = deq.popleft()
        # print(row, col, on)
        # 1. 불켜기
        for w in range(len(on_off[row][col])):
            r, c = on_off[row][col][w]
            # print(r, c)
            if room[r][c] == 0:
                result += 1
                room[r][c] = 1
            visit = [[0 for _ in range(N + 1)] for _ in range(N + 1)]

        on_off[row][col].clear()
        # for i in room:
        #     print(*i)

        # 2. 이동하기
        for w in range(4):
            nr = row + dr[w]
            nc = col + dc[w]

            if 1 <= nr < N + 1 and 1 <= nc < N + 1 and visit[nr][nc] == 0:
                if room[nr][nc] == 1:
                    visit[nr][nc] = 1
                    deq.append((nr, nc))

    return result

N, M = map(int, input().split())

room = [[0 for _ in range(N + 1)] for _ in range(N + 1)]
on_off = [[[] for _ in range(N + 1)] for _ in range(N + 1)]
room[1][1] = 1

for _ in range(M):
    r1, c1, r2, c2 = map(int, input().split())

    on_off[r1][c1].append((r2, c2))

print(bfs())


# jukebox.py

# A GUI interface that shows you artists, their albums,
# and the songs in those albums.

# Lecture: 221. Simple Database Browser

# I left is debug print statements.

# - Jim

import sqlite3
import tkinter

# Connect to the DB:
music_db_conn = sqlite3.connect("./music.sqlite")

# Which artist is presently selected?
# I'm not a fan of global variables.
# However, some artists may have albums of the same name.
# So, we need to know the artist's name in order to get the
# right album to display the songs.
# There's no way to pass the artist info to the function that
# pulls the songs from an album (at least there is no way
# that I know of w/ my current python skill.)
artist_id: None = None

# We are creating own class for the listbox + scrolling:

class Scrollbox(tkinter.Listbox):

    # This class has a listbox + a scrollbar.
    # It's parent's class is a listbox.  We add
    # in a scrollbar.

    def __init__(self, window, **kwargs):
        # Create the list box.
        super().__init__(window, **kwargs)

        # Add the scroll bar, and associate its movement to the list box.
        self.scrollbar = tkinter.Scrollbar(window, orient=tkinter.VERTICAL, command=self.yview)

        # Associate movement w/i the list box with the scroll bar's location.
        self["yscrollcommand"] = self.scrollbar.set

    # Overwriting the parent's (Listbox)  "grid" function.
    def grid(self, row, column, sticky="nsw", rowspan=1, columnspan=1, **kwargs):
        # Put the listbox on the left, and the scrollbar on the right.
        super().grid(row=row, column=column, sticky=sticky, rowspan=rowspan, columnspan=columnspan, **kwargs)
        self.scrollbar.grid(row=row, column=column, sticky="nse", rowspan=rowspan)

# End of classes

# Start of functions:

def get_albums(event):

    # global artist_id: We need to share this info so that we can
    # select the correct album.  Some album names are common like
    # "Greatest Hits".
    global artist_id

    # Get the artist name from the selection.
    list_box = event.widget
    # "list_box" is the list box had is calling
    # this function.  It has a function called
    # "curselection()".  It returns a tuple of
    # the currently selected items - their indexes.  (You can
    # have more than 1 selection.)  So, we are
    # just going to take item [0] in the tuple
    # b/c mostly likely we only selected 1 item.
    # curselection --> I guess it stands for "current selection".

    # The new version of tkinter has the "binding()" function
    # send an event, to the bound function, when you select a
    # different list box (call it ListBoxB).  Let's call this a
    # "deselect" of your original list box.  So, ListBoxA will send
    # a "ListboxSelect" event to its attached function.  And,
    # "curselection" will contain an empty tuple b/c no items,
    # in ListBoxA, are activitely/currently selected.  You need
    # to deal with that:  (Really dumb design in my opinion!
    # However, m/b I'll learn it's usefulness in time.)  You,
    # now, need to test for that!

    print("curselection is '{}'.".format(list_box.curselection()))
    if not list_box.curselection():
        # We have a deselection.
        # False call.
        # Do nothing.  Return.
        print("it's null")
        return

    # If you are here, then you have a value from "curselection()".
    # This time, the function was not called as a result of a deselection!

    # Get the albums for the artist, and populate the list box:
    index = list_box.curselection()[0]
    print("index is: '{}', type: '{}'.".format(index, type(index)))

    # This will be the actual value at that index aka the
    # value that was populated into it: the name of the band (as a string).

    # Either form below is fine.
    # selected_artist_name = tuple([list_box.get(album_index)])
    selected_artist_name = list_box.get(index)
    print("selected_artist_name is: '{}', type: '{}'.".format(selected_artist_name, type(selected_artist_name)))

    # We want the id number associated w/ the artist name.
    # This way we can use it to find the artist's albums in the album table.
    # The ".execute" statement returns an iterable.
    # We want grab the 1 result from it --> .fetchone()
    # Note: The replacement field list needs to be a tuple.
    # Also, is you just do tuple(some_string), the tuple will make each
    # character 1 element in the tuple.  Hence, we need to declare the
    # contents of "selected_artist_name" as 1 item in the tuple via [...].
    artist_id = music_db_conn.execute("SELECT artists._id FROM artists WHERE artists.name=?",
                                      tuple([selected_artist_name])).fetchone()

    print("Artist ID is: ", artist_id)

    # Get the artist's albums from the DB:
    albums_for_artist = []

    # The "?" replacement expects a tuple.  "artist_id" is a tuple b/c that is
    # what ".execute" returns.
    for row in music_db_conn.execute("SELECT albums.name FROM albums WHERE "
                                     "albums.artist = ? ORDER BY albums.name",  artist_id):
        # Remember ".execute" returns a tuple even when
        # you only get 1 item per row returned.  So, you
        # specifically have to get access the item via row[0].
        print(row[0])
        albums_for_artist.append(row[0])

    # Display the albums in the album scroll box:
    # (The "album_label_var" holds what is displayed
    # in the list box.
    album_label_var.set(tuple(albums_for_artist))

    # Reset the songs list as we are now onto a new artist.
    song_label_var.set(("Choose an album",))

    print("-----------------------------")

def get_songs(song_event):
    print("In 'get_songs'.")
    # What list box called this function?
    list_box = song_event.widget

    # The new version of tkinter has the "binding()" function
    # send an event, to the bound function, when you select a
    # different list box (call it ListBoxB).  Let's call this a
    # "deselect" of your original list box.  So, ListBoxA will send
    # a "ListboxSelect" event to its attached function.  And,
    # "curselection" will contain an empty tuple b/c no items,
    # in ListBoxA, are activitely/currently selected.  You need
    # to deal with that:  (Really dumb design in my opinion!
    # However, m/b I'll learn it's usefulness in time.)  You,
    # now, need to test for that!

    print("curselection is '{}'.".format(list_box.curselection()))
    if not list_box.curselection():
        # We have a deselection.
        # False call.
        # Do nothing.  Return.
        print("it's null")
        return

    # Get the item selected:
    # Note: > 1 item can be selected.  We are only
    # taking the 1st.
    # "curselection" = current selection
    # We are getting the index number not the album title.
    index = list_box.curselection()[0]
    print("index is: '{}', type: '{}'.".format(index, type(index)))

    # Get the album name.
    album_name = list_box.get(index)

    print(album_name, type(album_name))

    # Get the album id from the DB.
    # .execute creates an iterable.  Assume 1 entry selected (or if > 1, just
    # grab the 1st).  Use "fetchone()" to get 1 row.
    # .execute returns a tuple for each row.
    print("album name is: '{}'.  artist_id is '{}'.  artist_id type is: '{}'."
          .format(album_name, artist_id, type(artist_id)))

    # Note: artist_id is a tuple (w/ 1 entry).  It needs to be a single entry.
    # As a result, using: artist_id[0]
    album_id = music_db_conn.execute("SELECT _id FROM albums WHERE albums.name=? "
                                     "AND albums.artist=?", (album_name, artist_id[0])).fetchone()

    # sql_where = " WHERE " + self.field + "=? AND " + self.link_field + "=?"

    # Get the songs for this album:
    songs_for_album = []

    for row in music_db_conn.execute("SELECT songs.title FROM songs WHERE "
                                     "songs.album  = ? ORDER BY songs.track",  album_id):
        # .execute returns a tuple (of 1 element in this case).
        # Grab element 0
        songs_for_album.append(row[0])

        print(row)

    song_label_var.set(tuple(songs_for_album))
    print("*********************************")

# ===== Main Window =====
# Create the main GUI window:
main_window = tkinter.Tk()
main_window.title("Music DB Browser")
main_window.geometry('1024x768')

# Add the 4 columns.  Let 3 will have data.
# The last column, the one on the right will hold no data.
# It's just to put some space b/t the 3rd column, and the right
# side of the GUI.
main_window.columnconfigure(0, weight=2)    # Artists
main_window.columnconfigure(1, weight=2)    # Records
main_window.columnconfigure(2, weight=3)    # Songs
main_window.columnconfigure(3, weight=1)    # Just a spacer

# Add teh 4 rows.
main_window.rowconfigure(0, weight=1)    # For Column Headings
main_window.rowconfigure(1, weight=5)    # Artists, Records, & Songs
main_window.rowconfigure(2, weight=5)    # Artists cont.
main_window.rowconfigure(3, weight=1)    # Just a spacer

# Headings for the columns:
# (The teacher didn't see a need for assigning these to variables.
# I guess they won't change.)
tkinter.Label(main_window, text="Artists").grid(row=0, column=0)
tkinter.Label(main_window, text="Albums").grid(row=0, column=1)
tkinter.Label(main_window, text="Songs").grid(row=0, column=2)

# =====  Artists Scrollbox =====
# Remember: the artist listbox spans to 2 rows.
artist_listbox = Scrollbox(main_window)
# padx=(30, 0) --> Pad the left side by 30 pixels.
artist_listbox.grid(row=1, column=0, sticky='nsew', rowspan=2, padx=(30, 0))
artist_listbox.config(border=2, relief="sunken")

# Populate the list:
# (Put them in alphabetically.)
for artist_name in music_db_conn.execute("SELECT artists.name FROM artists ORDER BY  artists.name"):
    # You are iterating through a sorted list of the artists.
    # Add each 1 to the end of the list.
    # Note: The execute statement returns tuples!
    # So, you need to do artist_name[0].  Otherwise,
    # a tuple is put into the list box.
    artist_listbox.insert(tkinter.END, artist_name[0])

# When an artist is selected, call the "get_albums" function
# which will populate the Albums Scroll Box w/ their albums:
# Notes:
# 1) 'get_albums' is it's own function, and not part of the
# "Scrollbox" class.
# 2)  The "binding" will pass the event: "ListboxSelect" to
# the function.  It will pass the list box that triggered the
# function call.  (At that point (when the function is called),
# the list box will have a function called "curselection()"
# which will return a list of the currently selected items.
# (More than 1 item can be selected.)
# 3) curselection -> returns a list of the indexes!
# 4) IMPORTANT: new version of tkinter will cause:
# Unfortunately, the function "get_albums" will also get
# called on a deselect of this list box.  (When you
# select an element in another list box.)  And, we need to
# handle that in "get_songs".
artist_listbox.bind("<<ListboxSelect>>", get_albums)

# ===== Albums Scrollbox =====
# Initial text to display at the top of the scroll box.
# Later, it will hold/display a list of albums for the selected artist.
album_label_var = tkinter.Variable(main_window)
album_label_var.set(("Choose an artist",))

album_listbox = Scrollbox(main_window, listvariable=album_label_var)
album_listbox.grid(row=1, column=1, sticky="nsew", padx=(30, 0))
album_listbox.config(border=2, relief="sunken")

# Whenever an album is selected, call the "get_songs" function.
# Unfortunately, it will also get called on a deselect of this list box.
# (When you select an element in another list box.)
# And, we need to handle that in "get_songs".
album_listbox.bind("<<ListboxSelect>>", get_songs)

# ===== Song Scrollbox =====
# Initial text to display at the top of the scroll box.
# Later, it will hold/display a list of songs for the selected album.
song_label_var = tkinter.Variable(main_window)
song_label_var.set(("Choose an album",))

song_listbox = Scrollbox(main_window, listvariable=song_label_var)
song_listbox.grid(row=1, column=2, sticky="nsew", padx=(30, 0))
song_listbox.config(border=2, relief="sunken")

# ====== Main Loop =====
# Run the GUI:
main_window.mainloop()

# Done w/ the GUI.  Close the DB connection:
print("Closing the DB connection.")
music_db_conn.close()


import paddle.v2 as paddle
from paddle.utils.merge_model import merge_v2_model

# import network configuration
from mobilenet import mobile_net

if __name__ == "__main__":
    image_size = 224
    num_classes = 102
    net = mobile_net(3 * image_size * image_size, num_classes, 1.0)
    param_file = './mobilenet_flowers102.tar.gz'
    output_file = './mobilenet_flowers102.paddle'
    merge_v2_model(net, param_file, output_file)


import logging
import logging.handlers
import time
import datetime
import os
import random
import argparse
import sys
import re
# libs
import requests
from pyquery import PyQuery
# Local
import parse_post

class Topic():
    # Topic level stuff
    def get_topic_title(self):
        # Get the thread title
        thread_title_path = 'h2 > a'
        thread_title_element = self.d(thread_title_path)
        assert(thread_title_element)
        thread_title = thread_title_element.text()
        return thread_title

    def get_topic_id(self):
        # Get the thread ID
        thread_id_path = 'h2 > a'
        #print('get_topic_id() self.d: {0!r}'.format(self.d))
        thread_id_element = self.d(thread_id_path)
        thread_id_html = thread_id_element.outer_html()
        assert(thread_id_html)
        thread_id = re.search('<a\shref="./viewtopic\.php\?f=\d+&t=(\d+)(?:&start=\d+)?(?:&sid=\w+)?">', thread_id_html).group(1)
        return thread_id

    def get_board_id(self):
        # Get the board ID
        board_id_path = 'h2 > a'
        board_id_element = self.d(board_id_path)
        board_id_html = board_id_element.outer_html()
        assert(board_id_html)
        board_id = re.search('<a\shref="./viewtopic\.php\?f=(\d+)&t=\d+(?:&start=\d+)?(?:&sid=\w+)?">', board_id_html).group(1)
        return board_id

    def get_post_ids(self):
        # Get post IDs
        post_ids = re.findall('<div\sid="p(\d+)"\sclass="post\s(?:has-profile\s)?bg\d(?:\s*online\s*)?">', self.page_html)
        #print('Found {0} post_ids'.format(len(post_ids)))
        #print('post_ids: {0!r}'.format(post_ids))
        return post_ids
    # /Topic level stuff

    def parse_topic(self, page_html, board_id=None, topic_id=None, offset=None):
        """Parse a single page of posts
        return the information from the page as a dict"""
        self.page_html = page_html
        self.board_id = board_id
        self.topic_id = topic_id
        self.offset = offset

        self.d = PyQuery(self.page_html)

##        # Get thread level information
##        topic_dict = {}
##
##        topic_dict['title'] = self.get_topic_title()
##        topic_dict['thread_id'] = self.get_topic_id()
##        topic_dict['board_id'] = self.get_board_id()

        post_ids = self.get_post_ids()

        # Get post level information
        posts = []
        # With the post IDs, we can generate paths to the items we want
        for post_id in post_ids:
            #print('post_id: {0}'.format(post_id))
            # Lock ourselves to only this one post
            post_outer_html = self.d('#p{pid}'.format(pid=post_id)).outer_html()

            # Parse the post
            post = parse_post.Post()
            post_dict = post.parse_post(post_id, post_outer_html)
            # Store the post object away
            posts.append(post_dict)
    ##        if len(posts) == 200:# DEBUG
    ##            break# Stop at first post for debug
            continue
        #print('posts: {0!r}'.format(posts))

        #topic_dict['posts'] = posts

        #print('thread: {0!r}'.format(topic_dict))
        return posts

def parse_thread_page(page_html, board_id, topic_id, offset):
    """Parse a page of posts.
    Return a list of dicts, each dict being the values for a single post."""

    #return parse_viewtopic.parse_topic(page_html)

    topic = Topic()
    posts = topic.parse_topic(page_html)
    return posts

def split_thread_page_for_making_test_cases(topic_page_html_path, output_folder):
    with open(topic_page_html_path, 'r') as tf:
        topic_html = tf.read()
    post_ids = re.findall('<div\sid="p(\d+)"\sclass="post\s(?:has-profile\s)?bg\d(?:\s*online\s*)?">', topic_html)
    t = PyQuery(topic_html)
    for post_id in post_ids:
        post_outer_html = t('#p{pid}'.format(pid=post_id)).outer_html()
        post_filename = '{pid}.html'.format(pid=post_id)
        post_path = os.path.join(output_folder, post_filename)
        post_html = post_outer_html.encode('utf-8')# Fixes: UnicodeEncodeError: 'ascii' codec can't encode character u'\xbb' in position 537: ordinal not in range(128)
        with open(post_path, 'wb') as pf:
            pf.write(post_html)
        continue
    return

def main():
    pass

if __name__ == '__main__':
    main()
    # Test and debug stuff
    #file_path = os.path.join('debug','thread_page_response.htm')
    #file_path = os.path.join('debug','thread_page_response.b53.t2182.start2580.htm')
    #file_path = os.path.join('tests','aryion.b38.t45427.htm')
    #file_path = os.path.join('tests', 'phpbb.b64.t2377101.htm')
    #file_path = os.path.join('tests', 'aryion.b38.t44962.htm')
    #file_path = os.path.join('tests', 'phpbb.b64.t2103285.htm')
    #file_path = os.path.join('tests', 'electricalaudio.b5.t64830.htm')
    #file_path = os.path.join('tests', 'aryion.b53.t2182.offset2560.htm')
    ##file_path = os.path.join('tests', 'phpbb.b6.t362219.offset270.htm')
    file_path = os.path.join('tests', 'aryion.viewtopic.f38.t695.htm')
    #file_path = os.path.join('tests', 'phpbb.b6.t362219.offset270.htm')
    #file_path = os.path.join('tests', 'aryion.viewtopic.f38.t695.htm')
    #file_path = os.path.join('tests', 'phpbb.b6.t362219.offset270.htm')
    #file_path = os.path.join('tests', 'cichlid-forum.viewtopic.f4.t246181.htm')
    #file_path = os.path.join('tests', 'phpbb.b6.t2259706.offset15.htm')
    file_path = os.path.join('tests', 'aryion.viewtopic.f55.t11882.offset30.htm')# has swf attachment
    #file_path = os.path.join('tests', 'aryion.viewtopic.f79.t17592.htm')# has swf attachment
    #file_path = os.path.join('tests', 'phpbb.b64.t2377101.htm') # parse_s_thumbnails
    #file_path = os.path.join('tests', 'aryion.b53.t2182.offset2560.htm')

    split_thread_page_for_making_test_cases(
        topic_page_html_path=file_path,
        output_folder=os.path.join('tests', 'single_posts', 'aryion_new')
    )

    with open(file_path, 'r') as f:
        page_html = f.read()

    topic = Topic()
    result = topic.parse_topic(page_html)
    print('result: {0!r}'.format(result))


from random import choice

from flask import url_for
from pytest import mark

from shorter.start.environment import CRAWLERS

ENDPOINT = "side.robots"
EP_FVICO = "side.favicon"
EP_IMAGE = "side.logo"
EP_INDEX = "main.index"
EP_PAGES = "side.page"
EP_SHORT = "main.short"

@mark.usefixtures("session")
class TestSideRobots:
    @staticmethod
    @mark.usefixtures("ctx_app")
    def test_url():
        assert url_for(ENDPOINT) == "/robots.txt"

    @staticmethod
    def test_basic_view(visitor):
        res = visitor(ENDPOINT)
        txt = res.text.lower()

        assert "user-agent: *" in txt
        assert f"allow: {url_for(EP_INDEX)}$" in txt
        assert f"allow: {url_for(EP_FVICO)}" in txt
        assert f"allow: {url_for(EP_IMAGE)}" in txt
        assert f"allow: {url_for(EP_PAGES, name='')}" in txt

    @staticmethod
    def test_crawler_view(visitor):
        res = visitor(ENDPOINT, headers=[("User-Agent", choice(CRAWLERS))])
        txt = res.text.lower()

        assert "user-agent: *" in txt
        assert f"disallow: {url_for(EP_SHORT, symb='')}" in txt

    @staticmethod
    def test_headers(visitor):
        res = visitor(ENDPOINT)
        assert "text/plain" in res.headers.get("content-type")


import json_to_db
import os
from requests import get
import cairosvg
PATH = os.getcwd()+"/res/"
def printProgress (iteration, total, prefix = '', suffix = '', decimals = 1, barLength = 100):
	formatStr = "{0:." + str(decimals) + "f}"
	percent = formatStr.format(100 * (iteration / float(total)))
	filledLength = int(round(barLength * iteration / float(total)))
	bar = '#' * filledLength + '-' * (barLength - filledLength)
	sys.stdout.write('\r%s |%s| %s%s %s' % (prefix, bar, percent, '%', suffix)),
	if iteration == total:
		sys.stdout.write('\n')
	sys.stdout.flush()

if __name__ == '__main__':
	db = json_to_db.Json()
	#db.createDB() #Only Once
	db.json_to_db()
	db.db_insert()
	print(db.file_list)
	for filepath in db.file_list:
		url = "https://worldvectorlogo.com/download/"+filepath+".svg"
		print(url)
		with open(PATH+filepath+".svg", "wb") as file:
			response = get(url)
			file.write(response.content)
		path = os.getcwd()+"/res/"
		file_name = filepath
		cairosvg.svg2png(url=path+file_name+".svg",
                		write_to=path+file_name+".png",
                		dpi = 100
               			)


# SGAM: Building a Virtual 3D World through Simultaneous Generation and Mapping
# Authored by Yuan Shen, Wei-Chiu Ma and Shenlong Wang
# University of Illinois at Urbana-Champaign and Massachusetts Institute of Technology

import json
import numpy as np
from torch.utils.data import Dataset
from pathlib import Path
import networkx as nx
from PIL import Image
import pickle
import os

class PRNGMixin(object):
    """Adds a prng property which is a numpy RandomState which gets
    reinitialized whenever the pid changes to avoid synchronized sampling
    behavior when used in conjunction with multiprocessing."""

    @property
    def prng(self, seed=None):
        currentpid = os.getpid()
        if getattr(self, "_initpid", None) != currentpid:
            self._initpid = currentpid
            self._prng = np.random.RandomState(seed=seed) if seed is not None else np.random.RandomState()
        return self._prng

class Blender3dBase(Dataset, PRNGMixin):
    def __init__(self, split, dataset_dir, n_src=2, dataset=None, image_resolution=None):
        self.split = split
        self.image_resolution = image_resolution
        self.dataset = dataset
        self.dataset_dir = dataset_dir
        self.src_num = n_src
        self.grids = []
        self.cumulative_sum = [0]
        self.K = np.load(f"{self.dataset_dir}/K.npy")
        for grid_transform_path in sorted(Path(self.dataset_dir, self.split).glob("*")):
            print(grid_transform_path)
            with open(str(grid_transform_path / "transforms.json"), 'r') as f:
                curr_transform = json.load(f)
                g = self.build_graph_from_transform(curr_transform['frames'], grid_transform_path)
                self.grids.append(g)
                self.cumulative_sum.append(len(g.nodes) + self.cumulative_sum[-1])

    def build_graph_from_transform(self, transforms, grid_transform_path):
        g_path = f"{self.dataset_dir}/cache/{grid_transform_path.name[:-4]}_graph_{self.split}.txt"
        if os.path.exists(g_path):
           return pickle.load(open(g_path, 'rb'))
        g = nx.Graph()
        for i, transform in enumerate(transforms):
            c2w = np.array(transform['transform_matrix']) @ np.array([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]])
            w2c = np.linalg.inv(c2w)
            g.add_nodes_from([(i, {
                                    "R": w2c[:3, :3],
                                    "t": w2c[:3, 3],
                                    "position": c2w[:3, 3],
                                    "rgb_path": str(grid_transform_path / f"im_{i:05d}.png"),
                                    "depth_path": str(grid_transform_path / f"dm_{i:05d}.npy"),
                                    })])

        for i in range(len(transforms)-1):
            for j in range(i+1, len(transforms)):
                node_i = g.nodes[i]
                node_j = g.nodes[j]
                v = node_i['position'] - node_j['position']
                euclidean_dis = np.linalg.norm(v)
                if euclidean_dis <= 3: #TODO: to add as hyperparameters
                    g.add_edge(i, j, weight=euclidean_dis)
        pickle.dump(g, open(g_path, 'wb'))
        return g

    def __len__(self):
        return self.cumulative_sum[-1]

    def parse_idx(self, idx):
        for img_index, cur_cumsum in enumerate(self.cumulative_sum):
            if idx < self.cumulative_sum[img_index+1]:
                return img_index, idx - cur_cumsum

    def __getitem__(self, global_index):
        grid_id, idx = self.parse_idx(global_index)
        tgt_node = self.grids[grid_id].nodes[idx]
        tgt_neighbors = sorted(self.grids[grid_id][idx])
        if self.split == 'train':
            src_num = self.src_num
            src_nodes = [self.grids[grid_id].nodes[tgt_neighbors[k]] for k in self.prng.choice(len(tgt_neighbors), src_num)]
        else:
            state = np.random.RandomState(seed=global_index)
            tgt_neighbors = np.array(tgt_neighbors)
            state.shuffle(tgt_neighbors)
            src_num = self.src_num
            src_nodes = [self.grids[grid_id].nodes[k] for k in tgt_neighbors[:src_num]]
        img_dst = np.array(Image.open(tgt_node['rgb_path']))/127.5-1.0
        img_srcs = [np.array(Image.open(src_node['rgb_path']))/127.5-1.0 for src_node in src_nodes]

        dm_dst = np.load(tgt_node['depth_path'])#[..., None]
        dm_srcs = [np.load(src_node['depth_path']) for src_node in src_nodes]
        h, w = dm_dst.shape[:2]
        x = np.linspace(0, w - 1, w)
        y = np.linspace(0, h - 1, h)
        xs, ys = np.meshgrid(x, y)
        dm_dst = (dm_dst * self.K[0][0] / np.sqrt(self.K[0][0]**2 + (self.K[0][2] - ys - 0.5)**2 + (self.K[1][2] - xs - 0.5) ** 2))[..., None]
        dm_srcs = [(dm_src * self.K[0][0] / np.sqrt(self.K[0][0]**2 + (self.K[0][2] - ys - 0.5)**2 + (self.K[1][2] - xs - 0.5) ** 2))[..., None] for dm_src in dm_srcs]
        # for j in range(predicted_depth.shape[0]):
        #     for k in range(predicted_depth.shape[1]):
        #         predicted_depth[j, k] = predicted_depth[j, k] * K[0][0] / math.sqrt(K[0][0] ** 2 + (K[0][2] - j - 0.5) ** 2 + (K[1][2] - k - 0.5) ** 2)
        R_dst = tgt_node["R"]
        t_dst = tgt_node["t"]
        R_rels = []
        t_rels = []
        Ks = []
        K_invs = []
        T_tgt = np.eye(4)
        T_tgt[:3, :3] = R_dst
        T_tgt[:3, 3] = t_dst

        ## K
        h, w = img_dst.shape[:2]
        K = self.K
        K = K * self.image_resolution[1] / w
        K = K * self.image_resolution[0] / h
        for src_node in src_nodes:
            R_src = src_node["R"]
            t_src = src_node["t"]
            T_src = np.eye(4)
            T_src[:3, :3] = R_src
            T_src[:3, 3] = t_src
            T_rel = T_tgt @ np.linalg.inv(T_src)
            R_rel = T_rel[:3, :3]
            t_rel = T_rel[:3, 3]
            R_rels.append(R_rel)
            t_rels.append(t_rel)
            Ks.append(K)
            K_invs.append(np.linalg.inv(K))

        if self.image_resolution is not None and (self.image_resolution[0] != h or self.image_resolution[1] != w):
            ## img
            for i in range(len(img_srcs)):
                img_srcs[i] = img_srcs[i].resize((self.image_resolution[1], self.image_resolution[0]), resample=Image.LANCZOS)
            img_dst = img_dst.resize((self.image_resolution[1], self.image_resolution[0]), resample=Image.LANCZOS)

            ## depth
            for i in range(len(dm_srcs)):
                dm_srcs[i] = dm_srcs[i].resize((self.image_resolution[1], self.image_resolution[0]), resample=Image.LANCZOS)
            dm_dst = dm_dst.resize((self.image_resolution[1], self.image_resolution[0]), resample=Image.LANCZOS)
        mask = np.zeros(self.src_num)
        mask[:src_num] = 1
        while len(K_invs) < self.src_num:
            Ks.append(np.eye(3))
            K_invs.append(np.eye(3))
            R_rels.append(np.eye(3))
            t_rels.append(np.zeros(3))
            img_srcs.append(np.zeros_like(img_srcs[-1]))
            dm_srcs.append(np.zeros_like(dm_srcs[-1]))

        example = {
            "Ks": np.stack(Ks),
            "K_invs": np.stack(K_invs),
            "R_rels": np.stack(R_rels),
            "t_rels": np.stack(t_rels),
            "dst_img": img_dst, # rgb or rgbid
            "src_imgs": np.stack(img_srcs),
            "dst_depth": dm_dst,
            "src_depths": np.stack(dm_srcs),
            'src_masks': mask
        }

        for k in example:
            example[k] = example[k].astype(np.float32)

        return example

class Blender3dTrain(Blender3dBase):
    def __init__(self, dataset_dir=None, n_src=2, dataset=None, image_resolution=None):
        super().__init__(split='train', dataset_dir=dataset_dir, n_src=n_src, dataset=dataset, image_resolution=image_resolution)

class Blender3dValidation(Blender3dBase):
    def __init__(self, dataset_dir=None, n_src=2, dataset=None, image_resolution=None):
        super().__init__(split='val', n_src=n_src, dataset_dir=dataset_dir, dataset=dataset, image_resolution=image_resolution)

class Blender3dTest(Blender3dBase):
    def __init__(self, dataset_dir=None, n_src=2, dataset=None, image_resolution=None):
        super().__init__(split='test', n_src=n_src, dataset_dir=dataset_dir, dataset=dataset, image_resolution=image_resolution)


from datetime import datetime

from pytrends.request import TrendReq
from PersistenceLayer.ExternalAPIsORM.ExternalAPIsORMFacade import ExternalAPIsORMFacade

class GoogleTrendsManager:
    def __init__(self):
        self.pytrends = TrendReq(hl='en-US', tz=360, retries=3)
        self.ExternalOrm = ExternalAPIsORMFacade()
        self.trends = {}
        self.all_trends = self.ExternalOrm.get_all_trends()
        self.connect()

    def connect(self):
        try:
            self.trends = {}
            date = datetime.today().date()
            trends = self.pytrends.trending_searches().values.tolist()
            for trend in trends:
                if trend[0] not in self.all_trends.keys():
                    t_id = self.ExternalOrm.add_trend(trend[0], str(date))
                    self.trends[t_id] = {'keywords': trend[0]} # was trend instead of trend[0]
                    self.all_trends[trend[0]] = self.ExternalOrm.get_trend(t_id).copy()
                else:
                    flag = False
                    for trend_dict in self.all_trends[trend[0]]:
                        # print(trend_dict['date'])
                        # if type(trend_dict) == str:
                        #     trend_date = trend_dict
                        if type(trend_dict) == list:
                            trend_dict = trend_dict[0]
                            trend_date = trend_dict['date']
                        elif type(trend_dict) != dict:
                            trend_date = trend_dict
                            print(f"at GoogleTrendsManager.connect, the type of the trends dict is {type(trend_dict)}")

                        else:
                            trend_date = trend_dict['date']
                        if self.compare_dates(trend_date, date) == 0:
                        # if trend_dict['date'] == str(date):
                        #     print(f"at connect, trend_dict['id']= {trend_dict['id']}")
                        #     print(f"at connect, self.trends.keys()= {self.trends.keys()}")
                        #     print(f"flag = {flag}")
                            if trend_dict['id'] not in self.trends.keys() and not flag:
                                self.trends[trend_dict['id']] = {'keywords': trend[0]}
                            flag = True

                    if not flag:
                        t_id = self.ExternalOrm.add_trend(trend[0], str(date))
                        self.trends[t_id] = {'keywords': trend[0]}
                        self.all_trends[trend[0]].append(self.ExternalOrm.get_trend(t_id).copy())
        except:
            raise Exception("Connection problems")

    def get_trends(self):
        trends = {}
        for trend_name in self.all_trends:
            for trend in self.all_trends[trend_name]:
                stop = False
                if type(trend)==list:
                    if len(trend)>0:
                        print(f"at get_trends, trend type = {type(trend)}")
                        print(f"trends= {trends}")
                        trend=trend[0]
                    else:
                        stop=True
                        print(f"problem at get_trends, trend= {trend}")
                if not stop and (trend['date'] == str(datetime.today().date()) or trend['date'] == str(datetime.today())):
                    trends[self.all_trends[trend_name][0]['id']] = {'keywords': trend_name}
        for t_id in self.trends:
            if t_id not in trends.keys():
                trends[t_id] = self.trends[t_id]
        return trends

    def compare_dates(self, date1, date2):
        if type(date1) is str:
            if '-' in date1:
                date1 = datetime(int(date1[:4]), int(date1[5:7]), int(date1[8:])).date()
            else:
                date1 = datetime(int("20"+date1[6:]), int(date1[3:5]), int(date1[:2])).date()
        if type(date1) != type(datetime.today().date()) or type(date2) != type(datetime.today().date()):
            print(f"at googleTrendsManager.compare_dates     type of date1= {type(date1)},  type of date2= {type(date2)}")
            print(f"at googleTrendsManager.compare_dates     date1= {date1},  date2= {date2}")
        if date1.year != date2.year:
            if date1.year > date2.year:
                return 1
            else:
                return -1
        elif date1.month != date2.month:
            if date1.month > date2.month:
                return 1
            else:
                return -1
        elif date1.day == date2.day:
            return 0
        elif date1.day > date2.day:
            return 1
        else:
            return -1


"""
Given a list of integers, write a function that returns the largest sum of non-adjacent numbers.
Numbers can be 0 or negative.

For example, [2, 4, 6, 2, 5] should return 13, since we pick 2, 6, and 5. [5, 1, 1, 5]
should return 10, since we pick 5 and 5.

Follow-up: Can you do this in O(N) time and constant space?
"""
import pytest

from daily_problems.problem_0_to_100.problem_9 import largest_non_adjacent_sum

class TestLargestSum:
    def test_empty_list_should_return_null(self):
        assert largest_non_adjacent_sum([]) is None

    @pytest.mark.parametrize(
        "input_list,expected",
        [([1], 1), ([2], 2), ([3], 3), ([-10], -10), ([-20], -20), ([40], 40)],
    )
    def test_list_with_one_integer_should_return_the_integer(
        self, input_list, expected
    ):
        assert largest_non_adjacent_sum(input_list) == expected

    @pytest.mark.parametrize(
        "given_input,expected",
        [([1, 2], 2), ([5, -1], 5), ([10, 15], 15), ([30, 12], 30)],
    )
    def test_list_with_two_integers_should_return_the_higher_value(
        self, given_input, expected
    ):
        assert largest_non_adjacent_sum(given_input) == expected

    @pytest.mark.parametrize(
        "given_input,expected", [([1, 2, 3], 4), ([3, 5, 1], 5), ([-10, 15, 20], 20)]
    )
    def test_list_of_3_ints_should_return_max_non_adjacent_sum(
        self, given_input, expected
    ):
        assert largest_non_adjacent_sum(given_input) == expected

    @pytest.mark.parametrize(
        "given_input,expected",
        [([1, 2, 3, 4], 6), ([3, 1, 2, 4], 7), ([3, 4, -1, -3], 4)],
    )
    def test_list_of_4_ints_should_return_max_non_adjacent_sum(
        self, given_input, expected
    ):
        assert largest_non_adjacent_sum(given_input) == expected

    def test_sample_problem(self):
        assert largest_non_adjacent_sum([2, 4, 6, 2, 5]) == 13


import random
import argparse
import random
import time
import uuid
import json

import faker
import numpy as np

def random_timestamp_within(start, end, format, prop):
    """Get a time at a proportion of a range of two formatted times.

    start and end should be strings specifying times formated in the
    given format (strftime-style), giving an interval [start, end].
    prop specifies how a proportion of the interval to be taken after
    start.  The returned time will be in the specified format.
    """

    stime = time.mktime(time.strptime(start, format))
    etime = time.mktime(time.strptime(end, format))

    ptime = stime + prop * (etime - stime)
    return ptime

EVENTS = ['view', 'rate', 'buy']
RATINGS = [x for x in range(1, 5+1)]

def get_event():

    event = np.random.choice(EVENTS, 1, p=[0.6, 0.1, 0.3])[0]

    context = None

    if event == 'rate':
        return event, np.random.choice(RATINGS, 1)[0], context
    else:
        return event, None, context

def main(n_users, n_products, n_events, start_date, end_date, output):

    with open(output, 'w') as fp:

        for _ in range(n_events):

            event_type, weight, context = get_event()

            event = {
                'id': str(uuid.uuid4()),
                'type': event_type,
                'weight': weight,
                'timestamp': int(random_timestamp_within(start_date, end_date, '%Y-%m-%d', np.random.uniform(0, 1))),
                'context': context,
                'agent': {
                    'id': 'user-%d' % np.random.randint(1, n_users),
                    'type': 'user'
                },
                'element': {
                    'id': 'item-%d' % np.random.randint(1, n_products),
                    'type': 'product'
                }
            }

            fp.write(json.dumps(event))
            fp.write('\n')

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Generate fake ecommerce data.')
    parser.add_argument('--nusers', type=int, required=True, help='Number of users to generate')
    parser.add_argument('--nproducts', type=int, required=True, help='Number of users to generate')
    parser.add_argument('--nevents', type=int, required=True, help='Number of events to generate')
    parser.add_argument('--start_date', type=str, required=True, help='Starting date, format: yyyy-mm-dd')
    parser.add_argument('--end_date', type=str, required=True, help='End date, format: yyyy-mm-dd')
    parser.add_argument('--output', type=str, required=True, help='Output file, e.g. data.json')

    args = parser.parse_args()

    fake = faker.Factory()

    main(args.nusers, args.nproducts, args.nevents, args.start_date, args.end_date, args.output)


import importlib
import os
import types
from tensorflow.python.platform import tf_logging as logging

_TENSORFLOW_LAZY_LOADER_PREFIX = "_tfll"

class LazyLoader(types.ModuleType):
  """Lazily import a module, mainly to avoid pulling in large dependencies.

  `contrib`, and `ffmpeg` are examples of modules that are large and not always
  needed, and this allows them to only be loaded when they are used.
  """

  # The lint error here is incorrect.
  def __init__(self, local_name, parent_module_globals, name, warning=None):
    self._tfll_local_name = local_name
    self._tfll_parent_module_globals = parent_module_globals
    self._tfll_warning = warning

    # These members allows doctest correctly process this module member without
    # triggering self._load(). self._load() mutates parant_module_globals and
    # triggers a dict mutated during iteration error from doctest.py.
    # - for from_module()
    super().__setattr__("__module__", name.rsplit(".", 1)[0])
    # - for is_routine()
    super().__setattr__("__wrapped__", None)

    super().__init__(name)

  def _load(self):
    """Load the module and insert it into the parent's globals."""
    # Import the target module and insert it into the parent's namespace
    module = importlib.import_module(self.__name__)
    self._tfll_parent_module_globals[self._tfll_local_name] = module

    # Emit a warning if one was specified
    if self._tfll_warning:
      logging.warning(self._tfll_warning)
      # Make sure to only warn once.
      self._tfll_warning = None

    # Update this object's dict so that if someone keeps a reference to the
    #   LazyLoader, lookups are efficient (__getattr__ is only called on lookups
    #   that fail).
    self.__dict__.update(module.__dict__)

    return module

  def __getattr__(self, name):
    module = self._load()
    return getattr(module, name)

  def __setattr__(self, name, value):
    if name.startswith(_TENSORFLOW_LAZY_LOADER_PREFIX):
      super().__setattr__(name, value)
    else:
      module = self._load()
      setattr(module, name, value)
      self.__dict__[name] = value
      try:
        # check if the module has __all__
        if name not in self.__all__ and name != "__all__":
          self.__all__.append(name)
      except AttributeError:
        pass

  def __delattr__(self, name):
    if name.startswith(_TENSORFLOW_LAZY_LOADER_PREFIX):
      super().__delattr__(name)
    else:
      module = self._load()
      delattr(module, name)
      self.__dict__.pop(name)
      try:
        # check if the module has __all__
        if name in self.__all__:
          self.__all__.remove(name)
      except AttributeError:
        pass

  def __repr__(self):
    # Carefully to not trigger _load, since repr may be called in very
    # sensitive places.
    return f"<LazyLoader {self.__name__} as {self._tfll_local_name}>"

  def __dir__(self):
    module = self._load()
    return dir(module)

  def __reduce__(self):
    return importlib.import_module, (self.__name__,)

class KerasLazyLoader(LazyLoader):
  """LazyLoader that handles routing to different Keras version."""

  def __init__(  # pylint: disable=super-init-not-called
      self, parent_module_globals, mode=None, submodule=None, name="keras"):
    self._tfll_parent_module_globals = parent_module_globals
    self._tfll_mode = mode
    self._tfll_submodule = submodule
    self._tfll_name = name
    self._tfll_initialized = False

  def _initialize(self):
    """Resolve the Keras version to use and initialize the loader."""
    self._tfll_initialized = True
    package_name = None
    keras_version = None
    if os.environ.get("TF_USE_LEGACY_KERAS", None) in ("true", "True", "1"):
      try:
        import tf_keras  # pylint: disable=g-import-not-at-top,unused-import

        keras_version = "tf_keras"
        if self._tfll_mode == "v1":
          package_name = "tf_keras.api._v1.keras"
        else:
          package_name = "tf_keras.api._v2.keras"
      except ImportError:
        logging.warning(
            "Your environment has TF_USE_LEGACY_KERAS set to True, but you "
            "do not have the tf_keras package installed. You must install it "
            "in order to use the legacy tf.keras. Install it via: "
            "`pip install tf_keras`"
        )
    else:
      try:
        import keras  # pylint: disable=g-import-not-at-top

        if keras.__version__.startswith("3."):
          # This is the Keras 3.x case.
          keras_version = "keras_3"
          package_name = "keras._tf_keras.keras"
        else:
          # This is the Keras 2.x case.
          keras_version = "keras_2"
          if self._tfll_mode == "v1":
            package_name = "keras.api._v1.keras"
          else:
            package_name = "keras.api._v2.keras"
      except ImportError:
        raise ImportError(  # pylint: disable=raise-missing-from
            "Keras cannot be imported. Check that it is installed."
        )

    self._tfll_keras_version = keras_version
    if keras_version is not None:
      if self._tfll_submodule is not None:
        package_name += "." + self._tfll_submodule
      super().__init__(
          self._tfll_name, self._tfll_parent_module_globals, package_name
      )
    else:
      raise ImportError(  # pylint: disable=raise-missing-from
          "Keras cannot be imported. Check that it is installed."
      )

  def __getattr__(self, item):
    if item in ("_tfll_mode", "_tfll_initialized", "_tfll_name"):
      return super(types.ModuleType, self).__getattribute__(item)
    if not self._tfll_initialized:
      self._initialize()
    if self._tfll_keras_version == "keras_3":
      if (
          self._tfll_mode == "v1"
          and not self._tfll_submodule
          and item.startswith("compat.v1.")
      ):
        raise AttributeError(
            "`tf.compat.v1.keras` is not available with Keras 3. Keras 3 has "
            "no support for TF 1 APIs. You can install the `tf_keras` package "
            "as an alternative, and set the environment variable "
            "`TF_USE_LEGACY_KERAS=True` to configure TensorFlow to route "
            "`tf.compat.v1.keras` to `tf_keras`."
        )
      elif (
          self._tfll_mode == "v2"
          and not self._tfll_submodule
          and item.startswith("compat.v2.")
      ):
        raise AttributeError(
            "`tf.compat.v2.keras` is not available with Keras 3. Just use "
            "`import keras` instead."
        )
      elif self._tfll_submodule and self._tfll_submodule.startswith(
          "__internal__.legacy."
      ):
        raise AttributeError(
            f"`{item}` is not available with Keras 3."
        )
    module = self._load()
    return getattr(module, item)

  def __repr__(self):
    if self._tfll_initialized:
      return (
          f"<KerasLazyLoader ({self._tfll_keras_version}) "
          f"{self.__name__} as {self._tfll_local_name} mode={self._tfll_mode}>"
      )
    return "<KerasLazyLoader>"

  def __dir__(self):
    if not self._tfll_initialized:
      self._initialize()
    return super().__dir__()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import re
import os
import time
import logging
from . import config
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
import watchdog.events as events
from threading import Thread
from .cleepapi import CleepApi
from .file import File
from collections import deque

class ActionFileSync():
    def __init__(self, module=None):
        self.module = module

class ActionRestart():
    def __init__(self, frontend=False):
        self.frontend = frontend

class ActionsExecutor(Thread):
    """
    Actions executor
    Implements queue to execute sequentially actions
    """
    def __init__(self, rpc_url):
        Thread.__init__(self)
        Thread.daemon = True

        self.logger = logging.getLogger(self.__class__.__name__)
        self.running = True
        self.__queue = deque(maxlen=200)
        self.file = File()
        self.cleep = CleepApi(rpc_url)

    def stop(self):
        """
        Stop process
        """
        self.running = False

    def add_action(self, action):
        """
        Add an action to execute
        """
        self.__queue.appendleft(action)

    def __execute_action(self, action):
        """
        Execute specified action

        Args:
            action (dict): action to execute
        """
        if isinstance(action, ActionFileSync):
            if action.module is None:
                self.file.core_sync()
            else:
                self.file.module_sync(action.module)

        elif isinstance(action, ActionRestart):
            if action.frontend:
                self.cleep.restart_frontend()
            else:
                self.cleep.restart_backend()

    def run(self):
        """
        Main process: unqueue action and process it
        """
        while self.running:
            try:
                action = self.__queue.pop()
                self.__execute_action(action)

            except IndexError:
                #no action available
                time.sleep(0.25)

            except:
                #error occured during action execution
                self.logger.exception('Error during action execution:')

class CleepHandler(FileSystemEventHandler):
    """
    Watchdog handler for Cleep
    """
    REJECTED_FILENAMES = [
        '4913', #vim furtive temp file to check user permissions
        '.gitignore'
    ]
    REJECTED_EXTENSIONS = [
        '.swp', #vim
        '.swpx', #vim
        '.swx', #vim
        '.tmp', #generic?
        '.offset', #pygtail
        '.pyc', #python compiled
        '.log' #log file
    ]
    REJECTED_PREFIXES = [
        '.',
        '~'
    ]
    REJECTED_SUFFIXES = [
        '~'
    ]
    REJECTED_DIRS = [
        '.git',
        '.vscode',
        '.editor',
        'backend/__pycache__',
    ]
    DEBOUNCE = 0.25 #seconds

    def __init__(self, actions_executor):
        self.__last_times = {
            'backend': 0,
            'frontend': 0
        }
        self.logger = logging.getLogger(self.__class__.__name__)
        self.actions_executor = actions_executor

    def __is_event_dropped(self, event):
        """
        Analyse event and return True if event must be dropped
        Return:
            bool: True if event must be dropped
        """
        #drop dir modified event or created file (that always comes with modified file)
        if type(event) in (events.DirModifiedEvent, events.FileCreatedEvent):
            self.logger.debug('Filtered: DirModifiedEvent or FileCreatedEvent')
            return True

        #filter invalid event
        if not event:
            self.logger.debug('Filtered: invalid event')
            return True

        #filter event on current script
        if event.src_path == '.%s' % __file__:
            self.logger.debug('Filtered: event on current script')
            return True

        #filter root event
        if event.src_path == '.':
            self.logger.debug('Filtered: root event')
            return True

        #filter invalid extension
        src_ext = os.path.splitext(event.src_path)[1]
        if src_ext in self.REJECTED_EXTENSIONS:
            self.logger.debug('Filtered: invalid extension')
            return True

        #filter by prefix
        for prefix in self.REJECTED_PREFIXES:
            if event.src_path.startswith(prefix):
                self.logger.debug('Filtered: source prefix')
                return True
            if getattr(event, 'dest_path', None) and event.dest_path.startswith(prefix):
                self.logger.debug('Filtered: destination prefix')
                return True

        #filter by suffix
        for suffix in self.REJECTED_SUFFIXES:
            if event.src_path.endswith(suffix):
                self.logger.debug('Filtered: source suffix')
                return True
            if getattr(event, 'dest_path', None) and event.dest_path.endswith(prefix):
                self.logger.debug('Filtered: destination suffix')
                return True

        #filter by filename
        for filename in self.REJECTED_FILENAMES:
            if event.src_path.endswith(filename):
                self.logger.debug('Filtered: filename')
                return True

        #filter by dir
        parts = event.src_path.split(os.path.sep)
        for folder in self.REJECTED_DIRS:
            if folder in parts:
                self.logger.debug('Filtered: directory')
                return True

        return False

    def __debounce(self, now, last_time):
        if now<(last_time+self.DEBOUNCE):
            return True
        return False

    def __change_on_module(self, event):
        """
        Change on module

        Args:
            event: watchdog event

        Returns:
            module name (string) or None if nothing found
        """
        pattern = '^%s/(.*?)/(?:frontend|backend|scripts)/.*$' % (config.MODULES_SRC)
        #self.logger.debug('Module pattern: %s' % pattern)

        if hasattr(event, 'src_path'):
            res = re.findall(pattern, event.src_path)
            if res and len(res)>0:
                return res[0]

        if hasattr(event, 'dest_path'):
            res = re.findall(pattern, event.dest_path)
            if res and len(res)>0:
                return res[0]

        return None

    def __change_on_core(self, event):
        """
        Change on core

        Args:
            event: watchdog event

        Returns:
            True if change occurs in core, False otherwise
        """
        pattern_core = '^(%s/(?!tests|modules)(.*\.py)|%s/.*)$' % (config.CORE_SRC, config.HTML_SRC)
        pattern_bin = '^%s/cleep' % (config.BIN_SRC)
        #self.logger.debug('Core pattern: %s' % pattern_core)
        #self.logger.debug('Bin pattern: %s' % pattern_bin)

        if hasattr(event, 'src_path'):
            if re.search(pattern_core, event.src_path):
                return True
            if re.search(pattern_bin, event.src_path):
                return True

        if hasattr(event, 'dest_path'):
            if re.search(pattern_core, event.dest_path):
                return True

        return False

    def __change_on_frontend(self, event):
        """
        Detect change on frontend

        Args:
            event: watchdog event

        Returns:
            True if change on frontend, False otherwise
        """
        pattern = '^(%s/|%s/(?:.*?)/frontend/).*$' % (config.HTML_SRC, config.MODULES_SRC)
        #self.logger.debug('Frontend pattern: %s' % pattern)

        if hasattr(event, 'src_path'):
            res = re.search(pattern, event.src_path)
            if res:
                return True

        if hasattr(event, 'dest_path'):
            res = re.search(pattern, event.dest_path)
            if res:
                return True

        return False

    def on_any_event(self, event):
        try:
            now = time.time()
            self.logger.debug('Event: %s' % event)
            if not self.__is_event_dropped(event):
                self.__last_action_time = now

                #is event on backend or frontend ?
                frontend = self.__change_on_frontend(event)

                #debounce ?
                if self.__debounce(now, self.__last_times[('frontend' if frontend else 'backend')]):
                    self.logger.debug('  Dropped by debounce')
                    return

                #is event on module ?
                change_on_core = False
                change_on_module = self.__change_on_module(event)
                if change_on_module is None:
                    change_on_core = self.__change_on_core(event)

                #push action and update last action time
                if change_on_module:
                    self.logger.debug('  Change on module "%s"' % change_on_module)
                    self.actions_executor.add_action(ActionFileSync(change_on_module))
                elif change_on_core:
                    self.logger.debug('  Change on core')
                    self.actions_executor.add_action(ActionFileSync())
                else:
                    self.logger.debug('  Dropped due to useless change')
                    return
                if frontend:
                    self.logger.debug('  Change on frontend')
                    self.actions_executor.add_action(ActionRestart(True))
                    self.__last_times['frontend'] = now
                else:
                    self.logger.debug('  Change on backend')
                    self.actions_executor.add_action(ActionRestart())
                    self.__last_times['backend'] = now

        except:
            self.logger.exception('Error occured during watcher event processing:')

class CleepWatchdog():
    """
    Cleep core and module watchdog
    """
    def __init__(self, rpc_url):
        """
        Constructor

        Args:
            rpc_url (str): Cleep RPC url
        """
        self.logger = logging.getLogger(self.__class__.__name__)
        self.actions_executor = ActionsExecutor(rpc_url)

    def watch(self):
        """
        Start watchdog that monitors filesystem changes on Cleep sources
        """
        path = '%s/' % config.REPO_DIR
        self.actions_executor.start()

        if not os.path.exists(path):
            self.logger.error('Please get Cleep sources using "cleep-cli getcore" command')
            return False

        event_handler = CleepHandler(self.actions_executor)
        observer = Observer()
        observer.schedule(event_handler, path, recursive=True)
        self.logger.info('Cleep watchdog is running on "%s" (CTRL-C to stop)' % path)
        observer.start()
        try:
            while True:
                time.sleep(0.25)
        except KeyboardInterrupt:
            observer.stop()
            self.actions_executor.stop()
        observer.join()

        return True


import sys
import os
import yaml
import re
import urllib.request

from .drawio_parse.parse import xml2yaml
from .utils.get_yaml import get_yaml

DEBUG_FLAG = True

LEFT = 0
CENTER = 1
RIGHT = 2

class widget_configure:
    def __init__(self):
        self.x_center = 0
        self.y_center = 0
        self.x_left = 0
        self.x_right = 0
        self.y_top = 0
        self.width = 50
        self.height = 50
        self.text = ""
        self.items = []
        self.max = 100
        self.min = 0
        self.step = 1
        self.default = 0
        self.stylesheet_str = ""
        self.font = "Arial"
        self.font_size = 10

class label_configure(widget_configure):
    def is_url(self, path: str) -> bool:
        pattern = "https?://[\w/:%#\$&\?\(\)~\.=\+\-]+"
        if re.match(pattern, path):
            return True
        else:
            return False

    def __init__(self, yaml_abs_path: str, target_key: str) -> None:
        super().__init__()
        self.stylesheet_str = str()
        self.yaml_abs_path_file = yaml_abs_path
        self.target_key = target_key

        self.position = LEFT

        self.debug = DEBUG_FLAG

        self.yaml_data = get_yaml(self.yaml_abs_path_file)

        # exit when no yaml data
        if self.yaml_data is None:
            print("no yaml data")
            sys.exit()
        self.yaml_data = self.yaml_data[self.target_key]

    # include yaml from yaml ---------------------------------------------------
        if "include" in self.yaml_data:
            path = self.yaml_data["include"]["path"]
            key = self.yaml_data["include"]["key"]
            # is url or path
            if self.is_url(path):
                # is url -> save to ~/.cache/pyamlside2/yaml/***.yaml and load it
                # exists ~/.cache/pyamlside2/yaml/***.yaml ?
                if not os.path.exists(os.path.expanduser("~/.cache/pyamlside2/yaml/" + os.path.basename(path))):
                    os.makedirs(os.path.expanduser(
                        "~/.cache/pyamlside2/yaml"), exist_ok=True)
                    urllib.request.urlretrieve(path, os.path.expanduser(
                        "~/.cache/pyamlside2/yaml/") + os.path.basename(path))
                    print("download yaml file: " + path)
                else:
                    print("yaml file is already downloaded (Please delete ~/.cache/pyamlside2/yaml/" +
                          os.path.basename(path) + " to download again)")
                path = os.path.expanduser(
                    "~/.cache/pyamlside2/yaml/") + os.path.basename(path)

            with open(path, 'r') as f:
                self.yaml_data = yaml.load(f, Loader=yaml.FullLoader)
            self.yaml_data = label_configure(path, key).yaml_data
            print("yaml_data: " + str(self.yaml_data))
            # return force
            return

    # Using common tag --------------------------------------------------------

        if "type" in self.yaml_data:
            self.type = self.yaml_data["type"]
        else:
            print("type is undefined.")
            sys.exit()

        # switching position 1 ------------------
        if "x_center" in self.yaml_data:
            self.position = CENTER
            self.x_center = int(float(self.yaml_data["x_center"]))
        if "y_center" in self.yaml_data:
            self.y_center = int(float(self.yaml_data["y_center"]))
        # switching position 2 ------------------
        if "x_left" in self.yaml_data:
            self.position = LEFT
            self.x_left = int(float(self.yaml_data["x_left"]))
        # switching position 3 ------------------
        if "x_right" in self.yaml_data:
            self.position = RIGHT
            self.x_right = int(float(self.yaml_data["x_right"]))

        if "y_top" in self.yaml_data:
            self.y_top = int(float(self.yaml_data["y_top"]))

        if "width" in self.yaml_data:
            self.width = int(float(self.yaml_data["width"]))

        if "height" in self.yaml_data:
            self.height = int(float(self.yaml_data["height"]))

        if "text" in self.yaml_data:
            self.text = self.yaml_data["text"]

        if "items" in self.yaml_data:
            self.items = self.yaml_data["items"]

        if "max" in self.yaml_data:
            self.max = int(float(self.yaml_data["max"]))

        if "min" in self.yaml_data:
            self.min = int(float(self.yaml_data["min"]))

        if "step" in self.yaml_data:
            self.step = int(float(self.yaml_data["step"]))

        if "default" in self.yaml_data:
            self.default = int(float(self.yaml_data["default"]))

        if "style" in self.yaml_data:
            if "font" in self.yaml_data["style"]:
                font_list = self.yaml_data["style"]["font"].split(" ")

                for fl_s in font_list:
                    if fl_s.endswith("px") :
                        self.font_size = int(float(fl_s.replace("px", "")))

            if "font-family" in self.yaml_data["style"]:
                self.font = self.yaml_data["style"]["font-family"]

    # StyleSheet ---------------------------------------------------------------
        if "style" in self.yaml_data:
            # if include key
            if "include" in self.yaml_data["style"]:
                path = self.yaml_data["style"]["include"]["path"]
                key = self.yaml_data["style"]["include"]["key"]
                # is url or path
                print("path: " + path)
                if self.is_url(path):
                    # is url -> save to ~/.cache/pyamlside2/yaml/***.yaml and load it
                    # exists ~/.cache/pyamlside2/yaml/***.yaml ?
                    if not os.path.exists(os.path.expanduser("~/.cache/pyamlside2/yaml/" + os.path.basename(path))):
                        os.makedirs(os.path.expanduser(
                            "~/.cache/pyamlside2/yaml"), exist_ok=True)
                        urllib.request.urlretrieve(path, os.path.expanduser(
                            "~/.cache/pyamlside2/yaml/") + os.path.basename(path))
                        print("download yaml file: " + path)
                    else:
                        print("yaml file is already downloaded (Please delete ~/.cache/pyamlside2/yaml/" +
                              os.path.basename(path) + " to download again)")
                    print("yaml_data: " + str(self.yaml_data))
                    path = os.path.expanduser(
                        "~/.cache/pyamlside2/yaml/") + os.path.basename(path)

                with open(path, 'r') as f:
                    self.stylesheet_str = yaml.load(f, Loader=yaml.FullLoader)
                self.stylesheet_str = label_configure(
                    path, key).stylesheet_str
                # print("stylesheet_str: " + str(self.stylesheet_str))

            else:
                for key, value in self.yaml_data["style"].items():
                    # configure stylesheet
                    if (key == "background-image"):
                        # check path "./" or "../" ... (relative path)
                        # image_path : url(./image/back-image.png) -> ./image/back-image.png
                        image_path = value.replace("url(", "").replace(")", "")
                        if (image_path[0] == "." or image_path[0] == ".."):
                            basepath = os.path.dirname(self.yaml_abs_path_file)
                            image_path = os.path.join(basepath, image_path)
                            print("------------------")
                            print("image_path: " + image_path)
                            print("------------------")
                        self.stylesheet_str += key + \
                            ": url(" + image_path + ");"
                    else:
                        self.stylesheet_str += key + ": " + str(value) + "; "
            # print(self.stylesheet_str)

    # Label Rect ---------------------------------------------------------------
        if "rect" in self.yaml_data:
            if "include" in self.yaml_data["rect"]:
                path = self.yaml_data["rect"]["include"]["path"]
                key = self.yaml_data["rect"]["include"]["key"]
                if self.is_url(path):
                    # is url -> save to ~/.cache/pyamlside2/yaml/***.yaml and load it
                    # exists ~/.cache/pyamlside2/yaml/***.yaml ?
                    if not os.path.exists(os.path.expanduser("~/.cache/pyamlside2/yaml/" + os.path.basename(path))):
                        os.makedirs(os.path.expanduser(
                            "~/.cache/pyamlside2/yaml"), exist_ok=True)
                        urllib.request.urlretrieve(path, os.path.expanduser(
                            "~/.cache/pyamlside2/yaml/") + os.path.basename(path))
                        print("download yaml file: " + path)
                    else:
                        print("yaml file is already downloaded (Please delete ~/.cache/pyamlside2/yaml/" +
                              os.path.basename(path) + " to download again)")
                    path = os.path.expanduser(
                        "~/.cache/pyamlside2/yaml/") + os.path.basename(path)

                with open(path, 'r') as f:
                    self.rect_width = yaml.load(f, Loader=yaml.FullLoader)
                self.rect_width = label_configure(path, key).rect_width
                self.rect_height = label_configure(path, key).rect_height
            else:
                self.rect_width = self.yaml_data["rect"]["width"]
                self.rect_height = self.yaml_data["rect"]["height"]
        else:
            pass
            self.rect_width = self.width
            self.rect_height = self.height

        self.width = self.rect_width
        self.height = self.rect_height

    # debug ---------------------------------------------------------------
        # if "debug" in self.yaml_data:
        #     self.debug = self.yaml_data["debug"]
        # else:
        #     self.debug = False

        if self.position == LEFT:
            self.x = self.x_left
            self.y = self.y_top
        elif self.position == CENTER:
            self.x = self.x_center - self.width // 2
            self.y = self.y_center - self.height // 2
        elif self.position == RIGHT:
            self.x = self.x_right - self.width
            self.y = self.y_top - self.height


from flask import Flask, request, Response
import jsonpickle
import numpy as np
app = Flask(__name__)
import requests as req

url = 'http://localhost:80/Edge/3/1'

@app.route("/capture/", methods=['POST'])
def capture():

    r = request
    # convert string of image data to uint8
    nparr = np.fromstring(r.data, np.uint8)
    # decode image
    img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
    img_name = "opencv_frame_{}.png".format(0)
    # do some fancy processing here....
    cv2.imwrite(img_name, img)

    # build a response dict to send back to client
    response = {'message': 'image received. size={}x{}'.format(img.shape[1], img.shape[0])
                }
    # encode response using jsonpickle
    response_pickled = jsonpickle.encode(response)

    return Response(response=response_pickled, status=200, mimetype="application/json")

if __name__ == "__main__":
    app.run(host='0.0.0.0', port=90)


import numpy as np
from .get_dcm import get_dcm
from .q_mult import q_mult
from numpy import linalg as LA
import math

def get_theta_error_sun(poskep):
    for i in range(2, 6):
        poskep[0, i] = poskep[0, i]*math.pi/180
    sun = sun_sensor()  # return column vector
    z = getDCM(bV, sV, bI, sI) * np.matrix([0, 0, 1]).getH()
    theta = math.acos(np.dot(sun, z))
    vecu = np.cross(sun, z)
    uma = LA.norm(vecu)
    vecu = vecu/uma
    thetadegrees = math.asin(uma) * 180/math.pi
    alpha = 90-thetadegrees
    qref = np.matrix([[vecu*math.sin((alpha/2)*math.pi/180)],
                      [math.cos((alpha/2)*math.pi/180)]])
    return qref


import argparse
from pathlib import Path
import typing

import numpy as np
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from PIL import Image

import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
import torchvision.transforms.functional as F
from torchinfo import summary

from omegaconf import DictConfig, OmegaConf

import hdrpy

from deepy.data.dataset import SelfSupervisedDataset
from deepy.data.vision.visiondataset import UnorganizedImageFolder
import deepy.data.transform
import deepy.data.vision.transform
from deepy.train.trainer import RegressorTrainer
from deepy.train.extension import (
    IntervalTrigger,
    MinValueTrigger,
    MaxValueTrigger,
    ModelSaver,
    HistorySaver
)

import transform as mytransform
import util as myutil
import train_id as tid

def get_transform(cfg: DictConfig):
    pre_transforms = None

    dict_cfg_crop = cfg.dataset.transforms.paired_random_resized_crop.param
    if dict_cfg_crop.interpolation == 'Image.BICUBIC':
        interpolation = Image.BICUBIC

    transforms = deepy.data.transform.PairedCompose([
        deepy.data.transform.ToPairedTransform(
            torchvision.transforms.ToTensor()),
        deepy.data.vision.transform.PairedPowerTwoResize(interpolation=interpolation),
        deepy.data.transform.SeparatedTransform(
            transform=mytransform.ReinhardTMO(),
            target_transform=mytransform.RandomEilertsenTMO()),
    ])

    return pre_transforms, transforms

def get_dataset(cfg: DictConfig):
    cwd = Path.cwd()
    p = Path(cwd) / cfg.dataset.path

    testset = SelfSupervisedDataset(
        UnorganizedImageFolder(
            str(p / "test"),
            pre_load=False,
            loader=hdrpy.io.read, is_valid_file=myutil.is_valid_file)
    )

    return testset

def predict(path, dataset, net, device):
    p = Path(path)
    (p / 'imgs').mkdir(parents=True, exist_ok=True)
    transforms = deepy.data.transform.Compose(
        [torchvision.transforms.ToTensor(),
         mytransform.ResizeToMultiple(divisor=16, interpolation=Image.BICUBIC),
         mytransform.RandomEilertsenTMO()])

    post_transforms = mytransform.KinoshitaITMO()
    target_transforms = deepy.data.transform.Compose(
        [torchvision.transforms.ToTensor(),
         mytransform.ReinhardTMO()])

    with torch.no_grad():
        for i in range(len(dataset)):
            q = dataset.dataset.samples[i]
            q = Path(q)

            print('{:04d}/{:04d}: File Name {}'.format(i, len(dataset), q.name))

            sample, hdr_target = dataset[i]
            sample = transforms(sample).unsqueeze(0).float().to(device)
            ldr_predict = net(sample).to('cpu').clone().detach().squeeze(0)
            hdr_predict = post_transforms(torch.clamp(ldr_predict, 0, 1))
            height, width, _ = hdr_target.shape
            sample = sample.to('cpu').clone().detach().squeeze(0)
            sample = F.resize(sample, (height, width), Image.BICUBIC)
            ldr_predict = F.resize(ldr_predict, (height, width), Image.BICUBIC)
            hdr_predict = F.resize(hdr_predict, (height, width), Image.BICUBIC)
            ldr_target = target_transforms(hdr_target)

            hdrpy.io.write(
                p / 'imgs' / ('{:04d}'.format(i) + '_' + q.stem + '_input.jpg'),
                sample.clone().detach().numpy().transpose((1, 2, 0)))
            hdrpy.io.write(
                p / 'imgs' / ('{:04d}'.format(i) + '_' + q.stem + '_output.jpg'),
                ldr_predict.clone().detach().numpy().transpose((1, 2, 0)))
            hdrpy.io.write(
                p / 'imgs' / ('{:04d}'.format(i) + '_' + q.stem + '_prediction.pfm'),
                hdr_predict.clone().detach().numpy().transpose((1, 2, 0)))
            hdrpy.io.write(
                p / 'imgs' / ('{:04d}'.format(i) + '_' + q.stem + '_reinhard.jpg'),
                ldr_target.clone().detach().numpy().transpose((1, 2, 0)))
            hdrpy.io.write(
                p / 'imgs' / ('{:04d}'.format(i) + '_' + q.stem + '_target.pfm'),
                hdr_target)

def main(cfg: DictConfig, train_id:str) -> None:
    cwd = Path.cwd()
    myutil.print_config(cfg)
    # Setting seed
    myutil.set_random_seed(0)

    model_file_name = "{}_best.pth".format(cfg.model.name)

    # Checking history directory
    history_dir = cwd / 'history' / train_id
    if (history_dir / model_file_name).exists():
        pass
    else:
        return

    # Setting result directory
    # All outputs will be written into (p / 'result' / train_id).
    if not (cwd / 'result').exists():
        (cwd / 'result').mkdir(parents=True)
    result_dir = cwd / 'result' / train_id
    if result_dir.exists():
        # removing files in result_dir?
        pass
    else:
        result_dir.mkdir(parents=True, exist_ok=True)

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    # Testing
    testset = get_dataset(cfg)
    net = myutil.get_model(cfg)
    net.module.load_state_dict(torch.load(str(history_dir / model_file_name)))
    predict(result_dir, testset, net, device)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--history_dir', type=str,
                        default='./history',
                        help='Directory path for searching trained models')
    args = parser.parse_args()
    p = Path.cwd() / args.history_dir
    for q in p.glob('**/config.yaml'):
        cfg = OmegaConf.load(str(q))
        cfg.loader.num_workers = 0
        train_id = q.parent.name
        main(cfg, train_id)