#-----------------------------------------------------------------------------------------------------
# Author: cgarcia@umw.edu
# Created: 9/12/2020
# About: This file provides uniform wrappers for each algorithm. Each algorithm wrapper takes a
#        BaseProblem and param dict as input and returns a solution as a dict in the form:
#        {'solution':<json-able dict containing solution attributes>, 'objective_f':<obj>, 'elapsed_time':time}
#
# Revision History:
# Date        Author            Change Description
# 9/12/2020   cgarcia           Initial version.
#-----------------------------------------------------------------------------------------------------

import alns
import ga
from structs import *
from functools import reduce

# Converts an ordered list of Nodes and a corresponding AssignmentSet object
# to a json-writable/readable solution in the following form:
# [{'worker':<worker id>, 'job_node':<job node id>, 'role':<role id>, 'start_time':<job start time>}]
def generic_solution_to_dict(job_nodes, assignment_set):
    aset = assignment_set
    assigned_job_nodes = [a for a in job_nodes if a.node_id in aset.scheduled_node_ids]
    a = reduce(lambda x,y: x+y, [[{'job_node':j.node_id, 'start_time':j.start_time, 'worker':a[0], 'role':a[1]}
                                  for a in aset.node_assignment_dict[j.node_id]] for j in assigned_job_nodes] + [[]])
    return {'assignments':a, 'reward':aset.reward}

# This method applies ALNS to the problem using the given parameters.
def alns_algorithm(prob, params):
    initial_sol = alns.ALNSEncodedSolution(prob)
    max_iter = params['max_iter']
    max_unimprove_time = params['max_unimprove_time']
    reaction_factor = params['reaction_factor']
    cost_f = params['cost_f']
    accept_method = params['accept_method']
    update_period = params['update_period']
    repair_fs = params['repair_fs']
    destroy_fs = params['destroy_fs']
    weight_deltas = params['weight_deltas']
    initial_sol = sorted([r(alns.ALNSEncodedSolution(prob)) for r in repair_fs], key=lambda x:cost_f(x))[0] # Start with best initial solution.
    sol = alns.alns(initial_sol, max_iter, max_unimprove_time, reaction_factor, cost_f,
                    accept_method, update_period, repair_fs, destroy_fs, weight_deltas)
    job_nodes = sol['solution'].job_nodes
    asmt = alns.decode(sol['solution'])
    return {'solution':generic_solution_to_dict(job_nodes, asmt),
            'objective_f':asmt.reward, 'elapsed_time':sol['elapsed_time']}


import logging
import time
import os
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--dataname', type=str, default='flickr', help='Dataset name: flickr/coco/nuswide')
parser.add_argument('--bits', type=int, default=32, help='16/32/64/128')
parser.add_argument('--epochs', type=int, default=500, help='The epoch of training stage.')
# parser.add_argument('--gpuID', type=str, default='0', help='The GPU ID')
args = parser.parse_args()

if args.dataname == 'flickr':
    DIR = '../Data/raw_mir.mat'
elif args.dataname == 'nuswide':
    DIR = '../Data/raw_nus.mat'
elif args.dataname == 'coco':
    DIR = '../Data/raw_coco.mat'
else:
    print('Dataname Error!')
    DIR = ''

DATASET_NAME = args.dataname
CODE_LEN = args.bits
# GPU_ID = args.gpuID
FLAG_savecode = False

BETA = 0.6
LAMBDA1 = 0.1
LAMBDA2 = 0.1
NUM_EPOCH = 80
#NUM_EPOCH = 1

LR_IMG = 0.001
LR_TXT = 0.01
EVAL_INTERVAL = 40

BATCH_SIZE = 128 # 32
MU = 1.5
ETA = 0.4

MOMENTUM = 0.9
WEIGHT_DECAY = 5e-4

NUM_WORKERS = 1
EPOCH_INTERVAL = 2

MODEL_DIR = './checkpoint'
EVAL = False # EVAL = True: just test, EVAL = False: train and eval

logger = logging.getLogger('train')
logger.setLevel(logging.INFO)
now = time.strftime("%Y%m%d%H%M%S",time.localtime(time.time()))
log_name = now + '_log.txt'
log_dir = './log'
if not os.path.exists(log_dir):
    os.makedirs(log_dir)
txt_log = logging.FileHandler(os.path.join(log_dir, log_name))
txt_log.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
txt_log.setFormatter(formatter)
logger.addHandler(txt_log)

stream_log = logging.StreamHandler()
stream_log.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
stream_log.setFormatter(formatter)
logger.addHandler(stream_log)

logger.info('--------------------------Current Settings--------------------------')
logger.info('EVAL = %s' % EVAL)
logger.info('DATASET_NAME = %s' % DATASET_NAME)
logger.info('BETA = %.4f' % BETA)
logger.info('LAMBDA1 = %.4f' % LAMBDA1)
logger.info('LAMBDA2 = %.4f' % LAMBDA2)
logger.info('NUM_EPOCH = %d' % NUM_EPOCH)
logger.info('LR_IMG = %.4f' % LR_IMG)
logger.info('LR_TXT = %.4f' % LR_TXT)
logger.info('BATCH_SIZE = %d' % BATCH_SIZE)
logger.info('CODE_LEN = %d' % CODE_LEN)
logger.info('MU = %.4f' % MU)
logger.info('ETA = %.4f' % ETA)
logger.info('MOMENTUM = %.4f' % MOMENTUM)
logger.info('WEIGHT_DECAY = %.4f' % WEIGHT_DECAY)
# logger.info('GPU_ID =  %d' % GPU_ID)
logger.info('NUM_WORKERS = %d' % NUM_WORKERS)
logger.info('EPOCH_INTERVAL = %d' % EPOCH_INTERVAL)
logger.info('EVAL_INTERVAL = %d' % EVAL_INTERVAL)
logger.info('--------------------------------------------------------------------')


import sys
import unittest
import os

import coverage

from flask_script import Manager

from app import create_app, db

manager = Manager(create_app)

@manager.command
def createdb(drop_first=False):
    """Creates the database."""
    if drop_first:
        db.drop_all()
    db.create_all()

@manager.command
def test():
    """Runs unit tests."""
    os.environ['FLACK_CONFIG'] = 'testing'
    omit_items = [
        'manage.py',
        '*venv*/*',
        '*/serverapp351/*'
    ]

    # start coverage engine
    cov = coverage.Coverage(branch=True)
    cov.start()

    # run tests
    tests = unittest.TestLoader().discover('.')
    ok = unittest.TextTestRunner(verbosity=2).run(tests).wasSuccessful()

    # print coverage report
    cov.stop()
    cov.report(omit=omit_items)

    sys.exit(0 if ok else 1)

if __name__ == '__main__':
    manager.run()


from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
import json
import re
import socket
import cgi
from urllib.parse import parse_qs
from optparse import OptionParser
import os
from threading import Thread
import sys
import errno

# Optional command line argument
z = 1 # number of clients to wait before starting threads
c = 1 # in-memory cache flag

# in-memory cache
cache = {}
invalidation_flag = {} # buy() and restock() invalidate, successful query resets

leader_id = 0 # unitialized leader_id; determined by leader election
port_map_order = {} # main channel for handling order requests; key refers to replica id
port_map_leader = {} # for leader election

def initializeRepMap():
    global port_map_order
    global port_map_leader
    port_map_order[1] = 11111
    port_map_order[2] = 11112
    port_map_order[3] = 11113
    port_map_leader[1] = 21111
    port_map_leader[2] = 21112
    port_map_leader[3] = 21113

def push_cache(name, result):
    global cache
    global invalidation_flag
    cache[name] = result
    invalidation_flag[name] = False

# buy() and restock() invalidate cache to maintain cache consistency
def invalidate_item(name):
    global invalidation_flag
    if name is not None:
        invalidation_flag[name] = True

# catalog service sends invalidation flag upon restocking an item
def listen_restock():
    host = '127.0.0.1'
    port = 12545  # catalog_service restock port
    s1 = socket.socket()
    s1.connect((host, port))
    incoming = s1.recv(1024).decode()
    while incoming:
        names = re.split("\s", incoming)
        for name in names:
            if name != "":
                invalidate_item(name)
                print("received restock notification for {}".format(name))
        incoming = s1.recv(1024).decode()
    s1.close()

def LeaderElection():
    global leader_id
    global port_map_leader
    print("--Leader Election--")
    # ping each replica starting with highest ID in decreasing order
    for rid in range(3, 0, -1):
        host = '127.0.0.1'
        port = port_map_leader[rid]
        try:
            s = socket.socket()
            s.connect((host, port))
            msg = "ping"
            s.send(msg.encode())
            incoming = s.recv(1024).decode()
            if incoming == "present": # new leader chosen
                leader_id = rid
                print("New Leader: {}".format(leader_id))
                break
            s.close()
        except ConnectionRefusedError:
            print("order service with ID {} not present".format(rid))

    # notify all replicas of new leader
    for rid in range(3, 0, -1):
        host = '127.0.0.1'
        port = port_map_leader[rid]
        try:
            s = socket.socket()
            s.connect((host, port))
            msg = "New Leader: {}".format(leader_id)
            s.send(msg.encode())
            s.close()
        except ConnectionRefusedError:
            pass # node not online

class httpHandler(BaseHTTPRequestHandler):
    def do_GET(self):
        global z
        global d
        global c
        global leader_id
        global port_map_order
        if z > 0:
            z -= 1
            while z > 0:
                pass
        self.send_response(200)
        self.send_header("Content-type", "text/html")
        self.end_headers()

        # GET /products/<product_name>
        route_regex = re.compile(r"^/products/(.+)$")
        path = self.path
        match = route_regex.match(path)
        host = '127.0.0.1'
        port = 12645  # catalog_service
        if match: # match exists for /products/<product name>
            name = match.group(1)
            if c == 1: # check cache
                lookup = cache.get(name)
                if lookup is not None:
                    if not invalidation_flag.get(name):
                        self.wfile.write(lookup.encode())
                        # print("cache hit")
                        return
            # contact catalog_service to call query method
            # print("cache miss")
            s = socket.socket()
            s.connect((host, port))
            msg = "Query " + name
            s.send(msg.encode())
            incoming = s.recv(1024).decode()  # results of call
            data = json.loads(incoming)
            # if query successful then push to cache
            if data.get("error") is None:
                push_cache(name, incoming)
            self.wfile.write(incoming.encode())  # write back to client
            s.close()
            return

        # GET /orders/<order_number>
        route_regex = re.compile(r"^/orders/(.+)$")
        path = self.path
        match = route_regex.match(path)
        host = '127.0.0.1'
        if match:  # match exists for /orders/<order_number>
            order_id = match.group(1)
            # if order service fails, elect new leader and try again
            while True:
                port = port_map_order[leader_id]  # order_service leader node
                s = socket.socket()
                try:
                    s.connect((host, port))
                    msg = "getOrder " + order_id  # call query method
                    s.send(msg.encode())
                    incoming = s.recv(1024)  # results of call
                    self.wfile.write(incoming)  # write back to client
                    s.close()
                    break # successfully interacted with order_service, exit loop
                except ConnectionRefusedError:
                    print("Leader node of order service not present")
                    LeaderElection()
                except IOError as e:
                    if e.errno == errno.EPIPE:
                        print("Leader node of order service failed")
                    LeaderElection()

    def do_POST(self):
        global d
        # Parse POST form
        ctype, pdict = cgi.parse_header(self.headers['content-type'])
        if ctype == 'multipart/form-data':
            postvars = cgi.parse_multipart(self.rfile, pdict)
        elif ctype == 'application/x-www-form-urlencoded':
            length = int(self.headers['content-length'])
            postvars = parse_qs(
                self.rfile.read(length),
                keep_blank_values=1)
        else:
            postvars = {}
        data = {}
        for key, value in postvars.items():  # clean POST form into dictionary of strings
            data[key.decode()] = value[0].decode()

        self.send_response(200)
        self.send_header("Content-type", "text/html")
        self.end_headers()

        if self.path == "/orders":
            host = '127.0.0.1'
            while True:
                port = port_map_order[leader_id]  # order_service leader node
                s = socket.socket()
                try:
                    s.connect((host, port))
                    msg = "processOrder " + json.dumps(data)
                    s.send(msg.encode())
                    incoming = s.recv(1024)  # results of call
                    msg = None
                    try:
                        result = int(incoming.decode())
                        if result >= 0: # succesful order
                            msg = str(result)
                            if c == 1:
                                invalidate_item(data.get("name")) # invalidate item in cache after successful buy()
                        if result == -1:
                            msg = json.dumps(
                                {"error": {"code": 404, "message": "product not found"}})
                        if result == -2:
                            msg = json.dumps(
                                {"error": {"code": 404, "message": "product out of stock"}})
                    except:
                        msg = incoming.decode()
                    self.wfile.write(msg.encode())  # write back to client
                    s.close()
                    break # successfully interacted with order_service, exit loop
                except ConnectionRefusedError:
                    print("Leader node of order service not present")
                    LeaderElection()
                except IOError as e:
                    if e.errno == errno.EPIPE:
                        print("Leader node of order service failed")
                    LeaderElection()

def main():
    # Optional command line argumentz (default 1) specifies number
    # of clients to wait before starting threads
    parser = OptionParser()
    parser.add_option('-z', default=1, help='Parameter for probability of sending order request', action='store',
                      type='int', dest='z')
    parser.add_option('-c', default=1, help='in-memory cache flag', action='store',
                      type='int', dest='c')
    (options, args) = parser.parse_args()
    global z
    global d
    global c
    z = options.z
    c = options.c

    initializeRepMap()
    LeaderElection()

    # assign thread to listen to catalog server for invalidation notifications due to restocking an item
    if c == 1:
        t1 = Thread(target = listen_restock, args=())
        t1.start()
    host = '0.0.0.0'
    PORT = 8001
    server = ThreadingHTTPServer((host, PORT), httpHandler)
    server.serve_forever()

if __name__ == "__main__":
    main()


""" Core definition of Structure metadata """
from __future__ import annotations

from typing import List, Optional, Type, TypeVar

from pydantic import Field
from pymatgen.core.composition import Composition
from pymatgen.core.periodic_table import Element
from pymatgen.core.structure import Structure, Molecule

from emmet.core.base import EmmetBaseModel
from emmet.core.symmetry import SymmetryData, PointGroupData

T = TypeVar("T", bound="StructureMetadata")
S = TypeVar("S", bound="MoleculeMetadata")

class StructureMetadata(EmmetBaseModel):
    """
    Mix-in class for structure metadata
    """

    # Structure metadata
    nsites: int = Field(None, description="Total number of sites in the structure.")
    elements: List[Element] = Field(None, description="List of elements in the material.")
    nelements: int = Field(None, description="Number of elements.")
    composition: Composition = Field(None, description="Full composition for the material.")
    composition_reduced: Composition = Field(
        None,
        title="Reduced Composition",
        description="Simplified representation of the composition.",
    )
    formula_pretty: str = Field(
        None,
        title="Pretty Formula",
        description="Cleaned representation of the formula.",
    )
    formula_anonymous: str = Field(
        None,
        title="Anonymous Formula",
        description="Anonymized representation of the formula.",
    )
    chemsys: str = Field(
        None,
        title="Chemical System",
        description="dash-delimited string of elements in the material.",
    )
    volume: float = Field(
        None,
        title="Volume",
        description="Total volume for this structure in Angstroms^3.",
    )

    density: float = Field(None, title="Density", description="Density in grams per cm^3.")

    density_atomic: float = Field(
        None,
        title="Packing Density",
        description="The atomic packing density in atoms per cm^3.",
    )

    symmetry: SymmetryData = Field(None, description="Symmetry data for this material.")

    @classmethod
    def from_composition(
        cls: Type[T],
        composition: Composition,
        fields: Optional[List[str]] = None,
        **kwargs,
    ) -> T:

        fields = (
            [
                "elements",
                "nelements",
                "composition",
                "composition_reduced",
                "formula_pretty",
                "formula_anonymous",
                "chemsys",
            ]
            if fields is None
            else fields
        )
        composition = composition.remove_charges()

        elsyms = sorted(set([e.symbol for e in composition.elements]))

        data = {
            "elements": elsyms,
            "nelements": len(elsyms),
            "composition": composition,
            "composition_reduced": composition.reduced_composition.remove_charges(),
            "formula_pretty": composition.reduced_formula,
            "formula_anonymous": composition.anonymized_formula,
            "chemsys": "-".join(elsyms),
        }

        return cls(**{k: v for k, v in data.items() if k in fields}, **kwargs)

    @classmethod
    def from_structure(
        cls: Type[T],
        meta_structure: Structure,
        fields: Optional[List[str]] = None,
        **kwargs,
    ) -> T:

        fields = (
            [
                "nsites",
                "elements",
                "nelements",
                "composition",
                "composition_reduced",
                "formula_pretty",
                "formula_anonymous",
                "chemsys",
                "volume",
                "density",
                "density_atomic",
                "symmetry",
            ]
            if fields is None
            else fields
        )
        comp = meta_structure.composition.remove_charges()
        elsyms = sorted(set([e.symbol for e in comp.elements]))
        symmetry = SymmetryData.from_structure(meta_structure)

        data = {
            "nsites": meta_structure.num_sites,
            "elements": elsyms,
            "nelements": len(elsyms),
            "composition": comp,
            "composition_reduced": comp.reduced_composition,
            "formula_pretty": comp.reduced_formula,
            "formula_anonymous": comp.anonymized_formula,
            "chemsys": "-".join(elsyms),
            "volume": meta_structure.volume,
            "density": meta_structure.density,
            "density_atomic": meta_structure.volume / meta_structure.num_sites,
            "symmetry": symmetry,
        }
        return cls(**{k: v for k, v in data.items() if k in fields}, **kwargs)

class MoleculeMetadata(EmmetBaseModel):
    """
    Mix-in class for molecule metadata
    """

    charge: float = Field(None, description="Charge of the molecule")
    spin_multiplicity: int = Field(None, description="Spin multiplicity of the molecule")
    natoms: int = Field(None, description="Total number of atoms in the molecule")
    elements: List[Element] = Field(None, description="List of elements in the molecule")
    nelements: int = Field(None, title="Number of Elements")
    composition: Composition = Field(None, description="Full composition for the molecule")
    formula_alphabetical: str = Field(
        None,
        title="Alphabetical Formula",
        description="Alphabetical molecular formula",
    )
    chemsys: str = Field(
        None,
        title="Chemical System",
        description="dash-delimited string of elements in the molecule",
    )

    symmetry: PointGroupData = Field(None, description="Symmetry data for this molecule")

    @classmethod
    def from_molecule(
        cls: Type[S], meta_molecule: Molecule, fields: Optional[List[str]] = None, **kwargs
    ) -> MoleculeMetadata:

        fields = (
            [
                "charge",
                "spin_multiplicity",
                "natoms",
                "elements",
                "nelements",
                "composition",
                "formula_alphabetical",
                "chemsys",
                "symmetry",
            ]
            if fields is None
            else fields
        )
        comp = meta_molecule.composition
        elsyms = sorted(set([e.symbol for e in comp.elements]))
        symmetry = PointGroupData.from_molecule(meta_molecule)

        data = {
            "charge": meta_molecule.charge,
            "spin_multiplicity": meta_molecule.spin_multiplicity,
            "nsites": len(meta_molecule),
            "elements": elsyms,
            "nelements": len(elsyms),
            "composition": comp,
            "formula_alphabetical": comp.alphabetical_formula,
            "chemsys": "-".join(elsyms),
            "symmetry": symmetry,
        }

        return cls(**{k: v for k, v in data.items() if k in fields}, **kwargs)


import openai
import cv2
from matplotlib import pyplot as plt
import json
import key
import numpy as np
import urllib.request
import serial

# print the original image before pixelation
#  cv2_imshow(image)
# this function prints the input image
def printImage(img):
    # we can select different
    rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    plt.imshow(rgb)

# this image will print a grid of images
def printImageGrid(imglst):
    rbg_img = None
    for i, img in enumerate(imglst):
      plt.subplot(1, 2, i+1)
      # COLOR_RGB2BGR565
      # COLOR_BGR2RGB
      rgb_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
      plt.imshow(rgb_img)
    return rgb_img

# this function pixelates the image to a given height and width
def pixelate(img, w, h):
    height, width = img.shape[:2]

    # Resize input to "pixelated" size
    # INTER_LINEAR is a bilinear interpolation resampling method that uses the distanceweighted
    # average of the four nearest pixel values to estimate a new pixel value
    temp = cv2.resize(img, (w, h), interpolation=cv2.INTER_LINEAR)

    # Initialize output image
    # determines the “nearest” neighboring pixel and assumes its intensity value
    return cv2.resize(temp, (width, height), interpolation=cv2.INTER_NEAREST)

def genImages():
  # import the openAI API key
  openai.api_key = key.OPENAI_KEY

  # implement the DALL E image generation
  imagePrompt = input("enter a fire idea for a picture: ") # this is where the image prompt will go

  # create the image using DALL E and the image prompt
  rawImages = openai.Image.create(
      prompt = imagePrompt,
      n=2,
      size = "1024x1024"
  )

  # initialize the image lists
  img_values = []
  # this loops through generated Dall-E URLs and generates a list of pictures
  for url_dict in rawImages['data']:
    img_url = url_dict['url']
    with urllib.request.urlopen(img_url) as url_response:
      image_data = url_response.read()
    image = np.asarray(bytearray(image_data), dtype=np.uint8)
    # imglst.append(cv2.imdecode(image, cv2.IMREAD_COLOR))
    image = cv2.imdecode(image, cv2.IMREAD_COLOR)
    # here, we would likely send the picture data to the arduino

    # img_values.append(np.asarray(bytearray(pixelate(image, 32, 32))))
    img_values.append(np.asarray(pixelate(image, 32, 32)))

  test_img1 = img_values[0]
  test_img2 = img_values[1]

  resized_img1 = cv2.resize(test_img1, (32, 32))
  resized_img2 = cv2.resize(test_img2, (32,32))

  concat_images = np.concatenate((resized_img1, resized_img2), axis = 1)
  # If you need to save the image one more time:
  cv2.imwrite('concat_img.jpg', concat_images)
  print(concat_images.tolist())

  return concat_images

def main():
  # If you need to generate new images uncomment:
  # genImages()

  img = cv2.imread("concat_img.jpg")
  out_rgbs = img.tolist()
  ser = serial.Serial('/dev/tty.usbmodem2101', 9600)
  json_string = json.dumps(out_rgbs)
  ser.write(json_string.encode())

if __name__ == "__main__":
  main()


import sys
from typing import (
    List,
    Tuple,
    Optional,
)

lookup = {
    "0": "0000",
    "1": "0001",
    "2": "0010",
    "3": "0011",
    "4": "0100",
    "5": "0101",
    "6": "0110",
    "7": "0111",
    "8": "1000",
    "9": "1001",
    "A": "1010",
    "B": "1011",
    "C": "1100",
    "D": "1101",
    "E": "1110",
    "F": "1111",
}

def read_input(input_file: str) -> str:
    with open(input_file) as f:
        return f.readlines()[0].strip()

class Packet:
    def __init__(self, version: int, type_id: int) -> None:
        self.version = version
        self.type_id = type_id

class Literal(Packet):
    def __init__(self, version: int, type_id: int, bin_str: str, start: int) -> None:
        super().__init__(version, type_id)

        self.literal: int
        self.length: int

        self._read_str(bin_str, start)

    def _read_str(self, bin_str: str, start: int) -> None:
        lit_start = start + 6
        offset = 5
        literal = ""

        def read_val() -> None:
            nonlocal literal

            literal = f"{literal}{bin_str[lit_start + 1 : lit_start + offset]}"

        while bin_str[lit_start : lit_start + offset][0] == "1":
            read_val()
            lit_start += offset

        # Read the last one (with '0' prefix)
        read_val()
        lit_start += offset

        self.literal = int(literal, base=2)
        self.length = lit_start - start

    def __str__(self) -> str:
        return f"{self.literal}"

    def __repr__(self) -> str:
        return f"{self.literal}"

class Operator(Packet):
    def __init__(self, version: int, type_id: int, bin_str: str, start: int) -> None:
        super().__init__(version, type_id)

        self.literals: List[Literal] = []
        self.operators: List[Operator] = []
        self.length: int = 0
        self.type_length_id = int(bin_str[start + 6])

        self._read_str(bin_str, start)

    def _read_str(self, bin_str: str, start: int) -> None:
        if self.type_length_id == 0:
            length = int(bin_str[start + 7 : start + 22], base=2)
            sub_start = start + 22
            should_continue = True
            length_read = 0

            while should_continue:
                version = int(bin_str[sub_start : sub_start + 3], base=2)
                type_id = int(bin_str[sub_start + 3 : sub_start + 6], base=2)
                if type_id == 4:
                    lit = Literal(version, type_id, bin_str, sub_start)
                    self.literals.append(lit)
                    sub_start += lit.length
                    length_read += lit.length
                    if length_read == length:
                        should_continue = False
                else:
                    op = Operator(version, type_id, bin_str, sub_start)
                    self.operators.append(op)
                    # sub_offset += op.length
        else:
            count = int(bin_str[start + 7 : start + 18], base=2)
            sub_start = start + 18

            for _ in range(count):
                version = int(bin_str[sub_start : sub_start + 3], base=2)
                type_id = int(bin_str[sub_start + 3 : sub_start + 6], base=2)
                if type_id == 4:
                    lit = Literal(version, type_id, bin_str, sub_start)
                    self.literals.append(lit)
                    sub_start += lit.length
                else:
                    op = Operator(version, type_id, bin_str, sub_start)
                    self.operators.append(op)

    def __str__(self) -> str:
        return f"{self.literals}"

    def __repr__(self) -> str:
        return f"{self.literals}"

def get_literal(bin_str: str) -> Tuple[int, int]:
    start = 6
    offset = 5
    literal = ""

    def read_val(start: int) -> None:
        nonlocal literal

        literal = f"{literal}{bin_str[start + 1 : start + offset]}"

    while bin_str[start : start + offset][0] == "1":
        read_val(start)
        start += offset

    # Read the last one (with '0' prefix)
    read_val(start)
    start += offset

    return (int(literal, base=2), start)

def convert_to_binary(string: str) -> str:
    bin_str = ""
    for c in string:
        bin_str = f"{bin_str}{lookup[c]}"
    return bin_str

def main(input_file: str) -> None:
    line = read_input(input_file)
    bin_str = convert_to_binary(line)
    start = 0

    lits: List[Literal] = []
    ops: List[Operator] = []

    while start < len(bin_str):
        version = int(bin_str[start : start + 3], base=2)
        type_id = int(bin_str[start + 3 : start + 6], base=2)

        if type_id == 4:
            lit = Literal(version, type_id, bin_str, start)
            lits.append(lit)
            start += 100
        else:
            op = Operator(version, type_id, bin_str, start)
            ops.append(op)
            start += 100

        continue
        nxt = 0

        if type_id == 4:
            literal, nxt = get_literal(bin_str)
        else:
            type_length_id = int(bin_str[start + 6])
            if type_length_id == 0:
                length = int(bin_str[start + 7 : start + 22], base=2)
                sub_start = start + 22
                sub_offset = 0
                while sub_offset < length:
                    version = int(bin_str[sub_start : sub_start + 3], base=2)
                    type_id = int(bin_str[sub_start + 3 : sub_start + 6], base=2)
                    assert type_id == 4, f"type_id is NOT 4!!! is {type_id}"
                    literal, sub_nxt = get_literal(bin_str[sub_start + sub_offset :])
                    sub_offset += sub_nxt
                nxt = sub_start + sub_offset
            else:
                count = int(bin_str[start + 7 : start + 18], base=2)
                sub_start = start + 18
                sub_offset = 0
                for _ in range(count):
                    version = int(bin_str[sub_start : sub_start + 3], base=2)
                    type_id = int(bin_str[sub_start + 3 : sub_start + 6], base=2)
                    assert type_id == 4, f"type_id is NOT 4!!! is {type_id}"
                    literal, sub_nxt = get_literal(bin_str[sub_start + sub_offset :])
                    print("literal", literal)
                    sub_offset += sub_nxt
                nxt = sub_start + sub_offset
        while nxt % 8 != 0:
            nxt += 1

        start = nxt
    #  packet = Packet(line)
    #  print("version", packet.version)
    #  print("type_id", packet.type_id)
    #  print("literal", packet.literal)
    print(lits)
    print(ops)

if __name__ == "__main__":
    main(sys.argv[1] if len(sys.argv) == 2 else "day16/op3")


import time #To implement delays
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities #to log network traffic
from playsound import playsound

Surrey_driver_licensing_label = "/html/body/div/div[1]/div/mat-dialog-container/app-search-modal/div[2]/div/div[3]/div[2]"
Surrey_claim_centre_label = "/html/body/div/div[1]/div/mat-dialog-container/app-search-modal/div[2]/div/div[4]/div"

Label = Surrey_claim_centre_label # Change this for different locations

# Chrome Profiles
options = webdriver.ChromeOptions()
options.add_argument('--user-data-dir=C:/Users/sshao/AppData/Local/Google/Chrome/User Data')
options.add_argument('--profile-directory=Profile 1')
#implement capabilities
capabilities = DesiredCapabilities.CHROME
# capabilities["loggingPrefs"] = {"performance": "ALL"}  # chromedriver < ~75
capabilities["goog:loggingPrefs"] = {"performance": "ALL"}  # chromedriver 75+

browser = webdriver.Chrome('C:\webdrivers\chromedriver',
options=options, desired_capabilities=capabilities) #Open Browser
browser.get("https://onlinebusiness.icbc.com/webdeas-ui/home") #Open ICBC Website

playsound('C:/Users/sshao/Desktop/icbc bot/ohyeah.mp3')

def scrape_date():
    # Click the Surrey label to refresh the data
    Surrey_label = browser.find_element_by_xpath(Label)
    Surrey_label.click()

    try:
        WebDriverWait(browser, 600).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, "div.date-title"))
        )
        print("waiting")
    finally:
        print("Date Located. Returning data: ")

    # Find earliest date
    earliest_date = browser.find_element_by_css_selector("div.date-title")
    date1 = earliest_date.text
    return(date1)
def press_next():
    nextbutton_location = "/html/body/app-root/app-home/mat-card/div[3]/div[2]/button"
    nextbutton = browser.find_element_by_xpath(nextbutton_location)
    nextbutton.click()
def getting_to_searching():
    # Fill in Last Name
    lastname_box_location = "/html/body/app-root/app-login/mat-card/mat-card-content/form/span[1]/div[1]/mat-form-field/div/div[1]/div[3]/input"
    Lastname = "Shao"
    lastnamebox = browser.find_element_by_xpath(lastname_box_location)
    lastnamebox.clear()
    lastnamebox.send_keys(Lastname)

    # Fill in Drivers License Number
    driverlicensenumber = "nah"
    licensebox_location = "/html/body/app-root/app-login/mat-card/mat-card-content/form/span[1]/div[2]/mat-form-field/div/div[1]/div[3]/input"
    licensebox = browser.find_element_by_xpath(licensebox_location)
    licensebox.clear()
    licensebox.send_keys(driverlicensenumber)

    # Fill in Keyword (Password)
    keyword = "nah"
    keywordbox_location = "/html/body/app-root/app-login/mat-card/mat-card-content/form/span[2]/div[1]/mat-form-field/div/div[1]/div[3]/input"
    keywordbox = browser.find_element_by_xpath(keywordbox_location)
    keywordbox.clear()
    keywordbox.send_keys(keyword)

    #check box
    checkbox = browser.find_element_by_xpath("/html/body/app-root/app-login/mat-card/mat-card-content/form/span[2]/div[3]/mat-checkbox/label/div")
    checkbox.click()

    #Sign in
    signinbox = browser.find_element_by_xpath("/html/body/app-root/app-login/mat-card/mat-card-content/form/div[3]/button[2]")
    signinbox.click()

    try:
        WebDriverWait(browser, 600).until(
        EC.presence_of_element_located((By.XPATH, "/html/body/app-root/app-driver/div/mat-card/div[5]/div[1]/app-appointments/div/div[2]/div/div[4]/button[1]"))
    )
    finally:
        print("Reschedule appointment button located. Delaying 0.1s, then pressing.")
    time.sleep(0.1)
    #reschedule appointmnet
    reschedule_button = browser.find_element_by_xpath("/html/body/app-root/app-driver/div/mat-card/div[5]/div[1]/app-appointments/div/div[2]/div/div[4]/button[1]")
    reschedule_button.click()

    try:
        WebDriverWait(browser, 600).until(
        EC.presence_of_element_located((By.XPATH, "/html/body/div/div[2]/div/mat-dialog-container/app-cancel/div/div/div[2]/button[2]"))
    )
    finally:
        print("Confirm button located. Delaying 0.1s, then pressing.")
    time.sleep(0.1)
    #confirm
    confirm_button = browser.find_element_by_xpath("/html/body/div/div[2]/div/mat-dialog-container/app-cancel/div/div/div[2]/button[2]")
    confirm_button.click()

    try:
        WebDriverWait(browser, 600).until(
        EC.presence_of_element_located((By.XPATH, "/html/body/div/div[1]/div/mat-dialog-container/app-search-modal/div/div/form/div[1]/div[1]/mat-form-field/div/div[1]/div[3]/input"))
    )
    finally:
        print("Page Loaded. Delaying 0.1s, then entering data.")
    time.sleep(0.1)

    #Fill in Location
    surrey_location_first_letter = "S"
    surrey_location_second_letter = "u"
    surrey_location_third_letter = "r"
    surrey_location_fifth_letter = "e"
    location_box = browser.find_element_by_xpath("/html/body/div/div[1]/div/mat-dialog-container/app-search-modal/div/div/form/div[1]/div[1]/mat-form-field/div/div[1]/div[3]/input")
    location_box.clear()
    location_box.send_keys(surrey_location_first_letter)
    time.sleep(0.1)
    location_box.send_keys(surrey_location_second_letter)
    time.sleep(0.1)
    location_box.send_keys(surrey_location_third_letter)
    time.sleep(0.1)
    location_box.send_keys(surrey_location_third_letter)
    time.sleep(0.1)
    location_box.send_keys(surrey_location_fifth_letter)
    time.sleep(1)

    #click dropdown
    try:
        WebDriverWait(browser, 600).until(
        EC.presence_of_element_located((By.XPATH, "/html/body/div/div[2]/div/div"))
    )
    finally:
        print("Dropdown located. Delaying 0.1s, then pressing.")
    time.sleep(0.1)
    dropdown_result = browser.find_element_by_xpath("/html/body/div/div[2]/div/div")
    dropdown_result.click()

    #Search
    time.sleep(0.5)
    search_button = browser.find_element_by_xpath("/html/body/div/div/div/mat-dialog-container/app-search-modal/div/div/form/div[2]/button")
    search_button.click()

    time.sleep(3)
    #Check Surrey
    try:
        WebDriverWait(browser, 600).until(
        EC.presence_of_element_located((By.XPATH,Label))
    )
    finally:
        print("Surrey location located. Delaying 0.1s, then parsing.")
    time.sleep(0.1)
    Surrey_label = browser.find_element_by_xpath(Label)
    Surrey_label.click()

press_next()
getting_to_searching()

time.sleep(0.2)
#Collecting Reference Date (most likely today's date)
try:
    WebDriverWait(browser, 60).until(
    EC.presence_of_element_located((By.CSS_SELECTOR, "div.date-title"))
    )
    print("waiting")
finally:
    print("Element located. Delaying 0.1s, then parsing.")
time.sleep(0.1)
reference_date = browser.find_element_by_css_selector("div.date-title").text
print("This is the reference date: ")
print(reference_date)

# Random status condition
is_same_thing = False

while not is_same_thing:
    new_date = scrape_date()
    if new_date == reference_date:
        print("No change.")
    else:
        print("NEW DATE FOUND!")
        print(new_date)
        is_same_thing = True
        playsound('C:/Users/sshao/Desktop/icbc bot/ohyeah.mp3')
    time.sleep(5) # Wait 5 seconds then check again

print("Go register")

# Find later dates, change the 1 at the end in span
#browser.find_element_by_xpath("/html/body/div/div[2]/div/mat-dialog-container/app-eligible-tests/div/div[2]/mat-button-toggle-group/div/span[1]/div")


import os
from dataclasses import dataclass

from utils.factories.logger_factory import LoggerFactory
from utils.ncbi_database import NCBIDatabase
from utils.gene_util import get_opposite_dna
from utils.str_util import StrConverter

@dataclass
class GeneStreamAnalysis:
    data_path: str
    input_path: str
    output_directory: str
    mode: str = 'rna'
    limit: int = 200

    def __post_init__(self):
        self.inter_path = self.input_path if self.mode == 'inter' else None
        self.rna_path = self.input_path if self.mode == 'rna' else None
        file_name = os.path.basename(self.input_path)
        file_prefix = StrConverter.extract_file_name(file_name)
        suffix = 'stream_%d' % self.limit if self.mode == 'rna' else 'gene'
        self.result_path = os.path.join(self.output_directory, '%s_%s_result.txt' % (file_prefix, suffix))
        self.gene_reader = NCBIDatabase(self.data_path)
        self.logger = LoggerFactory()
        self.headers = {}
        self.inv_headers = []

    def get_utr_between(self, first, second):
        left = self.gene_reader.gene_segments[first].cds[1]
        right = self.gene_reader.gene_segments[second].cds[0] - 1
        return self.gene_reader.dna_code[left:right]

    def work_for_gene_index(self, index, start, end):
        gene_segment = self.gene_reader.gene_segments[index]
        assert gene_segment.cds[0] == min(start, end)
        assert gene_segment.cds[1] == max(start, end)
        seq = self.gene_reader.dna_code[gene_segment.cds[0] - 1:gene_segment.cds[1]]
        upstream = self.gene_reader.dna_code[max(gene_segment.cds[0] - self.limit - 1, 0):gene_segment.cds[0] - 1]
        downstream = self.gene_reader.dna_code[gene_segment.cds[1]:gene_segment.cds[1] + self.limit]
        if start > end:
            seq = get_opposite_dna(seq[::-1])
            upstream, downstream = get_opposite_dna(downstream[::-1]), get_opposite_dna(upstream[::-1])
        return seq, upstream, downstream

    def work_for_gene(self, gene_idx, gene_name, start, end, fw):
        if gene_name.find('->') >= 0:
            gene_name = gene_name[:gene_name.index('->')]
        if gene_name not in self.gene_reader.gene_name_segment_map:
            self.logger.info("%s not found in data" % gene_name)
            return
        cnt = 1
        fw.write('%d. %s\n' % (gene_idx, gene_name))
        for idx in self.gene_reader.gene_name_segment_map[gene_name]:
            seq, up, down = self.work_for_gene_index(idx, start, end)
            fw.write('%d)\n' % cnt)
            fw.write('position\t%d %s %d\n' % (
                self.gene_reader.gene_segments[idx].cds[0], '->' if start < end else '<-',
                self.gene_reader.gene_segments[idx].cds[1]))
            fw.write('product\t%s\n' % self.gene_reader.gene_segments[idx].product)
            fw.write('GeneID\t%s\n' % self.gene_reader.gene_segments[idx].gene_id)
            fw.write('stream\t%s\n' % seq)
            if up: fw.write('upstream\t%s\n' % up)
            if down: fw.write('downstream\t%s\n' % down)
            fw.write('\n')
            cnt += 1

    def check_inter(self, fw):
        for line in open(self.inter_path, 'r', encoding='utf8'):
            line = line.strip()
            if line == '': continue
            left, right = map(int, line.split(','))
            up, down = None, None
            for gene_segment in self.gene_reader.gene_segments:
                if max(gene_segment.cds) < left:
                    if not up or max(up.cds) < max(gene_segment.cds):
                        up = gene_segment
                if min(gene_segment.cds) > right:
                    if not down or min(down.cds) > min(gene_segment.cds):
                        down = gene_segment
            fw.write('%s:\n' % line)
            if up:
                fw.write('up-gene\t%s\nup-position\t%s\nup-product\t%s\n' % (
                    up.gene, '-'.join(map(str, up.cds)), up.product))
            if down:
                fw.write('down-gene\t%s\ndown-position\t%s\ndown-product\t%s\n' % (
                    down.gene, '-'.join(map(str, down.cds)), down.product))
            fw.write('\n')

    def generate_header(self, items):
        for idx, col_name in enumerate(items.strip().split('\t')):
            self.headers[col_name] = idx
            self.inv_headers.append(col_name)

    def run(self):
        with open(self.result_path, 'w', encoding='utf8') as fw:
            if self.mode == 'rna':
                lines = open(self.rna_path, 'r', encoding='utf8').readlines()
                self.generate_header(lines[0])
                for gene_idx, line in enumerate(lines[1:]):
                    items = line.split('\t')
                    gene_name, start, end = items[self.headers['gene']], int(
                        items[self.headers['map_start_pos']]), int(items[self.headers['map_end_pos']])
                    self.work_for_gene(gene_idx, gene_name.strip(), start, end, fw)
            elif self.mode == 'inter':
                self.check_inter(fw)
            else:
                raise ValueError(self.mode)


import pandas as pd
from utils import num_filter

def main():
    # S&P Industries by Ticker and Company
    industry_df = pd.read_csv('sp500_Industries.csv')
    industry_df = industry_df.rename(columns={'Symbol': 'Ticker'})

    # S&P Individual Stock Prices
    price_df = pd.read_csv('sp500_Price.csv').drop('Unnamed: 0', axis=1)
    price_df = num_filter(price_df, 'Date', ['2010-01-01', '2022-12-31'])
    price_df = price_df[['Ticker', 'Date', 'Volume', 'Adj Close']]

    # SIZE = # of stocks
    count_df = pd.DataFrame(industry_df.groupby('Sector').count().drop('Name', axis=1)).reset_index()
    count_df.columns = ['Sector', 'Count']

    # Y AXIS = close
    price2020_df = num_filter(price_df, 'Date', ['2022-01-01', '2022-12-31'])
    cdf = pd.merge(industry_df, price2020_df, on=['Ticker'])
    close_df = pd.DataFrame(cdf.groupby('Sector').mean(numeric_only=True)['Adj Close']).reset_index()
    close_df.columns = ['Sector', 'Close']

    rows = []
    for stock in list(set(price_df.Ticker.values)):
        df = price_df[price_df.Ticker == stock]['Adj Close']
        growth = ((df.iloc[-1] - df.iloc[0]) / df.iloc[0]) * 100
        row = stock, round(growth, 2)
        rows.append(row)

    growth_df = pd.DataFrame(rows, columns=['Ticker', 'Growth (%)'])
    growth_df = pd.merge(industry_df, growth_df, on=['Ticker'])
    growth_df = growth_df.groupby('Sector').mean(numeric_only=True)

    df = pd.merge(pd.merge(growth_df, close_df, on='Sector'), count_df, on='Sector')

    df.to_csv('sp500_growth.csv')

if __name__ == '__main__':
    main()

S&P 500 Dashboard Python Files and Datasets/[RUN AND SAVE] sp500_growth.py


import sys
from collections import deque

N, L, R = map(int, sys.stdin.readline().split())
graph = [list(map(int, sys.stdin.readline().split())) for _ in range(N)]
dx = [-1,1,0,0]
dy = [0,0,-1,1]
move_true = False

def bfs(x,y):
    #국경 간 이동 유무
    global move_true
    q = deque()
    q.append((x,y))
    #방문 처리
    visited[x][y] = True
    #연합 인구수
    sum_human = graph[x][y]
    #연합 수
    count = 1
    #연합의 위치
    near_country = [(x,y)]
    while q:
        x, y = q.popleft()
        for i in range(4):
            nx = x + dx[i]
            ny = y + dy[i]
            if nx < 0 or nx >= N or ny < 0 or ny >= N:
                continue
            if not visited[nx][ny] and L <= abs(graph[nx][ny]-graph[x][y]) <= R:
                sum_human += graph[nx][ny]
                count += 1
                visited[nx][ny] = True
                q.append((nx,ny))
                near_country.append((nx,ny))
    #인구 이동 발생 시
    if count > 1:
        move_true = True
        cal_human = sum_human//count
        for cx, cy in near_country:
            graph[cx][cy] = cal_human
    return move_true

#인구 이동 일수
day_result = 0
while True:
    visited = [[False]*N for _ in range(N)]
    move_true = False
    for i in range(N):
        for j in range(N):
            if not visited[i][j]:
                move_true = bfs(i,j)
    if not move_true:
        break
    day_result += 1
print(day_result)


import cv2
import matplotlib.pyplot as plt
import cvlib as cv
import urllib.request
import numpy as np
from cvlib.object_detection import draw_bbox
import concurrent.futures
from cvzone.HandTrackingModule import HandDetector
import math
import cvzone
#
# detector = HandDetector(detectionCon=0.8,maxHands=1)
# x = [300,250,200,150,130,115,100,80,70,66]
# y = [12,14,19,23,27,34,38,46,52,62]
url = 'http://192.168.231.1/cam-hi.jpg'
im = None
classNames = []
classFile = 'coco.names'

with open(classFile,'rt') as f:
    classNames = f.read().rstrip('\n').split('\n')
# print(classNames)
configPath = 'ssd_mobilenet_v3_large_coco_2020_01_14.pbtxt'
weightspath = 'frozen_inference_graph.pb'

net =cv2.dnn_DetectionModel(weightspath,configPath)
# coff = np.polyfit(x,y,2)

net.setInputSize(320,320)
net.setInputScale(1.0/127.5)
net.setInputMean((127.5,127.5,127.5))
net.setInputSwapRB(True)
# fourcc = cv2.VideoWriter_fourcc(*'XVID')
# out = cv2.VideoWriter('output.avi',fourcc, 20.0, (640,480))

def run1():
    cv2.namedWindow("live transmission", cv2.WINDOW_AUTOSIZE)
    while True:
        img_resp = urllib.request.urlopen(url)
        imgnp = np.array(bytearray(img_resp.read()), dtype=np.uint8)
        im = cv2.imdecode(imgnp, -1)

        cv2.imshow('live transmission', im)
        key = cv2.waitKey(5)
        if key == ord('q'):
            break

    cv2.destroyAllWindows()

def run2():
    cv2.namedWindow("detection", cv2.WINDOW_AUTOSIZE)
    while True:
        img_resp = urllib.request.urlopen(url)
        imgnp = np.array(bytearray(img_resp.read()), dtype=np.uint8)
        im = cv2.imdecode(imgnp, -1)
        classIds, confidencess, bbox = net.detect(im, confThreshold=0.5)
        if len(classIds) != 0:
            for classId, confidence, box in zip(classIds.flatten(), confidencess.flatten(), bbox):
                cv2.rectangle(im, box, color=(0, 255, 0), thickness=3)
                cv2.putText(im, classNames[classId - 1].capitalize(), (box[0] + 10, box[1] + 30), cv2.FONT_ITALIC, 1,
                            (255, 255, 0), 2)
                cv2.putText(im, str(round(confidence * 100, 2)), (box[0] + 40, box[1] + 50), cv2.FONT_ITALIC, 0.5,
                            (255, 255, 0), 1)
        # out.write(im)
        # result = cv2.VideoWriter('filename.avi',
        #                          cv2.VideoWriter_fourcc(*'MJPG'),
        #                          10, size)
        cv2.imshow('detection', im)
        key = cv2.waitKey(5)
        if key == ord('q'):
            break
    cv2.destroyAllWindows()

# def run3():
#     cv2.namedWindow("live distance", cv2.WINDOW_AUTOSIZE)
#     while True:
#         img_resp = urllib.request.urlopen(url)
#         imgnp = np.array(bytearray(img_resp.read()), dtype=np.uint8)
#         im = cv2.imdecode(imgnp, -1)
#         hands, img = detector.findHands(im)
#         if hands:
#             lmList = hands[0]['lmList']
#             x, y, w, h = hands[0]['bbox']
#
#             # print(lmList[5])
#             x1, y1, z1 = lmList[5]
#             x2, y2, z2 = lmList[17]
#             # print(x1)
#             distance = int(math.sqrt((y2 - y1) ** 2 + (x2 - x1) ** 2))
#             A, B, C = coff
#             distanceCM = A * distance ** 2 + B * distance + C
#             print(distanceCM)
#             # print(distance)
#             # print(distance)
#             cvzone.putTextRect(img, f'{int(distanceCM)}cm', (x, y))
#
#             # print(abs(x2-x1),distance)
#         cv2.imshow("distance", im)
#         key = cv2.waitKey(5)
#         if key == ord('q'):
#             break
#
#     cv2.destroyAllWindows()

if __name__ == '__main__':
    print("started")
    with concurrent.futures.ProcessPoolExecutor() as executer:
        f1 = executer.submit(run1)
        f2 = executer.submit(run2)
        # f3 = executer.submit(run3)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('monitor', '0009_config'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='config',
            name='blah',
        ),
        migrations.AlterField(
            model_name='config',
            name='predicate',
            field=models.CharField(max_length=50, blank=True),
            preserve_default=True,
        ),
    ]


# coding: utf8
from django.db import models
from orum import Orum
from period import Period

__author__ = 'Demyanov Kirill'

class OrumCorrection(models.Model):
    orum = models.ForeignKey(
        Orum
    )
    period = models.ForeignKey(
        Period,
        null=True
    )
    kwh = models.DecimalField(
        u'Корректировка в kwh',
        decimal_places=5,
        max_digits=17,
        default=0
    )


from abc import abstractmethod
from typing import Iterable, Optional

import numpy as np
from init_args_serializer import Serializable
from rcsenv import JointLimitException, RcsSimEnv

import pyrado
from pyrado.environments.sim_base import SimEnv
from pyrado.spaces.base import Space
from pyrado.spaces.box import BoxSpace
from pyrado.spaces.empty import EmptySpace
from pyrado.tasks.base import Task
from pyrado.utils.data_types import RenderMode

def to_pyrado_space(space) -> [BoxSpace, EmptySpace]:
    """
    Convert the box space implementation from RcsPySim to the one of Pyrado.

    :param space: a space from RcsPySim
    :return: a Pyrado `BoxSpace` or an Pyrado`EmptySpace` if `None` was given
    """
    if space is None:
        return EmptySpace
    return BoxSpace(space.min, space.max, labels=space.names)

class RcsSim(SimEnv, Serializable):
    """Base class for RcsPySim environments. Uses Serializable to facilitate proper serialization."""

    def __init__(
        self,
        envType: str,
        task_args: dict,
        dt: float = 0.01,
        max_steps: int = pyrado.inf,
        init_state: Optional[np.ndarray] = None,
        checkJointLimits: bool = True,
        joint_limit_penalty: float = -1e3,
        state_mask_labels: Optional[Iterable[str]] = None,
        **kwargs,
    ):
        """
        Constructor

        .. note::
            The joint type (i.e. position or torque control) is set in the associated xml files in `RcsPySim/config`.

        :param envType: environment type name as defined on the C++ side
        :param task_args: arguments for the task construction
        :param dt: integration step size in seconds
        :param max_steps: max number of simulation time steps
        :param domain_param: initial domain param values
        :param init_state: initial state sampler can be a callable or one fixed state
        :param checkJointLimits: flags if the joint limits should be ignored or not passed to the C++ constructor
        :param joint_limit_penalty: cost returned on termination due to joint limits. This is a different from the
                                    state bounds since `RcsPySim` return an error when the joint limits are violated.
        :param state_mask_labels: if `None`, the complete observation space (defined on the C++ side) will be used as
                                  state space. Pass an iterable of stings to select dimensions from the observation
                                  space as the state space. This is also affects the tasks.
        :param kwargs: keyword arguments which are available for `RcsSim` on the C++ side. These arguments will not
                       be stored in the environment object, thus are saved e.g. when pickled.
        """
        if not isinstance(envType, str):
            raise pyrado.TypeErr(given=envType, expected_type=str)
        if not isinstance(task_args, dict):
            raise pyrado.TypeErr(given=task_args, expected_type=dict)
        if dt <= 0:
            raise pyrado.ValueErr(given=dt, g_constraint="0.0 (float)")

        Serializable._init(self, locals())

        # Initialize basic variables
        super().__init__(dt, max_steps)
        self._check_joint_limits = checkJointLimits

        # Create Rcs-based implementation (RcsSimEnv comes from the pybind11 module)
        self._sim = RcsSimEnv(dt=dt, envType=envType, checkJointLimits=self._check_joint_limits, **kwargs)

        # Setup the initial domain parameters
        self._domain_param = self._unadapt_domain_param(self._sim.domainParam)

        if joint_limit_penalty > 0:
            raise pyrado.ValueErr(given=joint_limit_penalty, le_constraint="0")
        self._joint_limit_penalty = joint_limit_penalty

        # Initial init state space is taken from C++ which can be overwritten by setting a new init_space
        self._init_space = to_pyrado_space(self._sim.initStateSpace)

        # By default, the state space is a subset of the observation space. Set this to customize in subclass.
        if state_mask_labels is None:
            self.state_mask = None
        else:
            # Mask / redefine the state space, e.g. to match the init state space. This is affects the tasks.
            self.state_mask = self.obs_space.create_mask(state_mask_labels)

        # Dummy initialization, must be set by the derived classes
        self.init_state = None
        self.task_args = task_args
        self._task = self._create_task(self.task_args)

    @property
    def state_space(self) -> Space:
        """Derives the state space from the observation space using _state_from_obs or state_mask."""
        obs_space = self.obs_space
        # Check if _state_from_obs was overridden
        if self._state_from_obs.__func__ != RcsSim._state_from_obs:
            return BoxSpace(self._state_from_obs(obs_space.bound_lo), self._state_from_obs(obs_space.bound_up), None)
        # Check if there is a state mask
        if self.state_mask is not None:
            return obs_space.subspace(self.state_mask)
        # Identical to obs space
        return obs_space

    @property
    def obs_space(self) -> Space:
        return to_pyrado_space(self._sim.observationSpace)

    @property
    def act_space(self) -> Space:
        return to_pyrado_space(self._sim.actionSpace)

    @property
    def init_space(self) -> Space:
        return self._init_space

    @init_space.setter
    def init_space(self, space: Space):
        if not isinstance(space, Space):
            raise pyrado.TypeErr(given=space, expected_type=Space)
        self._init_space = space

    @property
    def task(self) -> Task:
        return self._task

    @abstractmethod
    def _create_task(self, task_args: dict) -> Task:
        # Needs to implemented by subclasses
        raise NotImplementedError

    @property
    def domain_param(self) -> dict:
        return self._unadapt_domain_param(self._sim.domainParam)

    @domain_param.setter
    def domain_param(self, domain_param: dict):
        if not isinstance(domain_param, dict):
            raise pyrado.TypeErr(given=domain_param, expected_type=dict)
        # Update the internal parameters. The New domain parameters will be applied on reset().
        self._domain_param.update(domain_param)

        # Update task
        self._task = self._create_task(self.task_args)

    @classmethod
    def get_nominal_domain_param(cls):
        """
        Get the nominal a.k.a. default domain parameters.

        .. note::
            It is highly recommended to have the same values as in the associated physics config file (p<NAME>.xml),
            since the nominal domain parameters are not set explicitly from Pyrado (only when randomizing).
        """
        raise NotImplementedError

    def _state_from_obs(self, obs: np.ndarray) -> np.ndarray:
        """
        Retrieve the system state from the observation. In most cases, the system state is a part of the observation.
        This function is to be used when the observations include additional information.
        The default implementation is based off `self.state_mask` which is set in sub-classes of `RcsSim`.

        :param obs: observation from the environment
        :return: state of the environment
        """
        obs_ = obs.copy()
        if self.state_mask is not None:
            obs_ = obs_[self.state_mask]
        return obs_

    def _adapt_domain_param(self, params: dict) -> dict:
        """
        Changes the domain parameters before passing them to the Rcs simulation.
        One use case is for example the rolling friction coefficient which is usually given unit-less but the Vortex
        physics engine expects it to be multiplied with the body's curvature radius.

        :param params: domain parameters to adapt
        :return: adapted parameters
        """
        return params

    def _unadapt_domain_param(self, params: dict) -> dict:
        """
        Changes the domain parameters coming from to the Rcs simulation.

        .. note::
            This function is called from the constructor.

        :param params: domain parameters to revert the previously done adaptation
        :return: unadapted parameters
        """
        return params

    def _get_state(self, state_dict: dict):
        state_dict["domain_param"] = self.domain_param
        state_dict["init_state"] = self.init_state

    def _set_state(self, state_dict: dict, copying: bool = False):
        self.domain_param = state_dict["domain_param"]
        self.init_state = state_dict["init_state"]

    def _disturbance_generator(self) -> (np.ndarray, None):
        """Provide an artificial disturbance. For example a force on a body in the physics simulation."""
        return None

    def reset(self, init_state: np.ndarray = None, domain_param: dict = None) -> np.ndarray:
        # Reset time
        self._curr_step = 0

        # Reset the state
        if init_state is None:
            # Sample from the init state space
            init_state = self._init_space.sample_uniform()
        else:
            if not init_state.shape == self._init_space.shape:
                raise pyrado.ShapeErr(given=init_state, expected_match=self._init_space)

        # Reset the task
        self._task.reset(env_spec=self.spec)

        # Use stored domain parameters if not overwritten
        if domain_param is None:
            domain_param = self._domain_param

        # Forward to C++ implementation
        obs = self._sim.reset(domainParam=self._adapt_domain_param(domain_param), initState=init_state)
        self.state = self._state_from_obs(obs)

        return obs

    def step(self, act: np.ndarray) -> tuple:
        # Current reward depending on the state (before step) and the (unlimited) action
        remaining_steps = self._max_steps - (self._curr_step + 1) if self._max_steps is not pyrado.inf else 0
        self._curr_rew = self._task.step_rew(self.state, act, remaining_steps)

        # Apply actuator limits
        act = self.limit_act(act)

        # Get the disturbance to be applied on the Rcs side
        disturbance = self._disturbance_generator()

        # Dynamics are calculated in the Rcs simulation
        try:
            obs = self._sim.step(act, disturbance)
        except JointLimitException:
            # Joint limits exceeded! Return (obs, rew, done, info) directly after this failure.
            return self._sim.lastObservation, self._joint_limit_penalty, True, dict(t=self._curr_step * self._dt)

        self.state = self._state_from_obs(obs)  # only for the Python side

        self._curr_step += 1

        # Check if the task or the environment is done
        done = self._task.is_done(self.state)
        if self._curr_step >= self._max_steps:
            done = True

        if done:
            # Add final reward if done
            self._curr_rew += self._task.final_rew(self.state, remaining_steps)

        return obs, self._curr_rew, done, dict()

    def render(self, mode: RenderMode = RenderMode(text=True), render_step: int = 1):
        if self._curr_step % render_step == 0:
            # Call base class
            super().render(mode)

            # Forward to Rcs GUI
            if mode.video:
                self._sim.render()

    def save_config_xml(self, fileName: str):
        """
        Save environment configuration as xml file for use on the C++ side.

        :param fileName: output file name
        """
        self._sim.saveConfigXML(fileName)

    def get_body_position(self, bodyName: str, refFrameName: str, refBodyName: str) -> np.ndarray:
        """
        Get the position of a body in the simulators config graph.
        This function uses code coped from `Rcs` to transform the position depending on a reference frame and/or body.

        :param bodyName: name of the body in the graph
        :param refFrameName: name of the reference frame, pass '' to use world coordinates
        :param refBodyName: name of the reference body, pass '' to use world coordinates
        :return: x,y,z positions in a reference frame coordinates relative to a reference bodies
        """
        return self._sim.getBodyPosition(bodyName, refFrameName, refBodyName)

    def get_body_extents(self, bodyName: str, shapeIdx: int = 0) -> np.ndarray:
        """
        Get the dimensions of a body in the simulators config graph.
        This function uses code coped from `Rcs` to transform the position depending on a reference frame and/or body.

        .. note::
            Depending on the kind of shape (e.g. box, sphere, torus, ect.) the extends mean different things.

        :param bodyName: name of the body in the graph
        :param shapeIdx: index of the shape in the `Body` node, defaults to the first shape of the body
        :return: x,y,z positions in a reference frame coordinates relative to a reference bodies
        """
        return self._sim.getBodyExtents(bodyName, shapeIdx)


from django import template

register = template.Library()

@register.filter
def show_exact_length(items_list, fixed_length):
    length = len(list(items_list))
    result = []
    if length > fixed_length:

        for idx in range(0, length, fixed_length):
            result.append(items_list[idx:(idx + fixed_length)])
        return result
    return None


"""
Action for calculating first two principal component of the RFM table and add the value of those component to a new dataframe
Inputs:
    - segment_data: Dataframe with RFM values and segments
    - customer_id_column: Customer identification column of the segment_data
    - segment_column: Segment labels columns of segment_data
Output:
    - Modified Dataframe with RFM values and  the first two principal components values
"""""
import traceback
import pandas as pd
from dft import df_plot
from sklearn.preprocessing import StandardScaler
from dft.base_execution_handler import BaseExecutionHandler
from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder
import dirty_cat
import numpy as np

class ExecutionHandler(BaseExecutionHandler):
    def execute(self, segment_data, customer_id_column, segment_column):

        from sklearn.decomposition import PCA
        try:
            pca = PCA(2)  # calculating the first two principal component
            data = segment_data.drop([customer_id_column, segment_column], axis=1)
            data = self.numeric_column_scale(data)
            data = self.categorical_column_encode(data)
            df = pca.fit_transform(data)
            PC1 = data.columns[pca.components_[0].argmax()] + " (Feature 1)"
            PC2 = data.columns[pca.components_[1].argmax()] + " (Feature 2)"
            d = pd.DataFrame(df)
            d[segment_column] = segment_data[segment_column]
            d.rename(columns={0: PC1, 1: PC2}, inplace=True)
            table = pd.pivot(d, values=PC2,
                             columns=segment_column).reset_index()
            n = len(d[segment_column].unique())
            table[PC1] = d[PC1]
            df_plot.multiple_series_scatter_chart('scatter plot', PC1, list(table.columns[1:n]), table)
        except Exception as e:
            print(e)
            print(traceback.format_exc())
            raise
        return table

    def categorical_column_encode(self, data: pd.DataFrame):
        columns = list(data.select_dtypes(include=['object']).columns)
        distinct_values = dict(map(lambda x: (x, len(pd.unique(data[x]))), columns))
        least_distinct_columns = list(dict(filter(lambda x: x[1] <= 5, distinct_values.items())).keys())
        mid_distinct_columns = list(dict(filter(lambda x: 5 < x[1] <= 10, distinct_values.items())).keys())
        most_distinct_values = list(dict(filter(lambda x: 10 < x[1], distinct_values.items())).keys())

        if len(least_distinct_columns):
            data = self.one_hot_encode(data, least_distinct_columns, drop_columns=True)
        if len(mid_distinct_columns):
            data = self.original_encode(data, mid_distinct_columns)
        if len(most_distinct_values):
            data = self.similarity_encode(data, most_distinct_values)

        return data

    def one_hot_encode(self, data: pd.DataFrame, columns, drop_columns):
        try:
            encoder = OneHotEncoder(sparse=False, handle_unknown='ignore').fit(data[columns])

            encoded_cols = list(encoder.get_feature_names_out(columns))

            data[encoded_cols] = encoder.transform(data[columns])
            if drop_columns:
                data = data.drop(columns, axis=1)

            for col in columns:
                print(col + " is " + "encoded")
            return data

        except Exception as e:
            raise type(e)(e)

    def original_encode(self, data: pd.DataFrame, columns):
        try:
            encoder = OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=np.nan).fit(data[columns])
            new_names = [column + '_' + 'encode' for column in columns]
            data[new_names] = pd.DataFrame(encoder.transform(data[columns]))
            data.drop(columns, axis=1, inplace=True)
            for col in columns:
                print(col + " is " + "encoded")
            return data
        except Exception as e:
            raise type(e)(e)

    def similarity_encode(self, data: pd.DataFrame, columns):
        try:
            for col in columns:
                encoder = dirty_cat.SimilarityEncoder(similarity='ngram')
                encoded_df = pd.DataFrame(encoder.fit_transform(data[[col]]))
                encoded_df.columns = [col + '_' + str(i) for i in encoded_df.columns]
                data = pd.concat([data, encoded_df], axis=1)
                data.drop(col, axis=1, inplace=True)
                print(col + " is " + "encoded")

            return data
        except Exception as e:
            raise type(e)(e)

    def numeric_column_scale(self, data: pd.DataFrame):
        from sklearn.preprocessing import MinMaxScaler
        columns = list(data.select_dtypes(include=['int', 'float']).columns)
        for col in columns:
            if data[col].dtype == 'float':
                scaler = MinMaxScaler().fit(data[[col]])
                data[col] = scaler.transform(data[[col]])

                print(col + ' Is Normalized')
            else:
                if data[col].nunique() / len(data[col]) > 0.5:
                    scaler = MinMaxScaler().fit(data[[col]])
                    data[col] = scaler.transform(data[[col]]).round(2)
                    print(col + ' Is Normalized')
                else:
                    dummies = pd.get_dummies(data[col])
                    dummies.columns = [col + '_' + str(i) for i in list(dummies.columns)]
                    data = pd.concat([data, dummies], axis=1)
                    data = data.drop(col, axis=1)
                    print(col + ' Is Normalized')

        return data


# -*- coding: utf-8 -*-
"""
Created on Sat Jan 26 13:45:49 2019

@author: vlade
"""
### This is a script that takes a look at the famous Titanic dataset (downloaded from Kaggle)
### The idea here is to predict whether the passenger would survive

import os
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import statsmodels.api as sm
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix

os.chdir("C:\\Users\\vlade\\Documents\\Python Scripts\\Kaggle\\Titanic") # Change directory to your own
train = pd.read_csv("train.csv")
test = pd.read_csv("test.csv")
#train = pd.concat([train, test])

# Taking a brief look at the data
train.info()
train.describe()
train.head()

# Look for missing data
plt.figure(figsize = (10, 6))
sns.heatmap(train.isnull(),yticklabels=False,cbar=False,cmap='viridis')

### Age needs to be filled in, cabin column should be dropped.
### To fill in age we could build a simple regression model that takes age
### as a function of sex and pclass
sns.boxplot(data = train, x = "Pclass", y = "Age", hue = "Sex")

### Some feature engineering
###
### 1) Turn the title of passenger name into dummy variable

ser = pd.Series(train["Name"])
names = ser.str.extract(r'([A-Z]{1}[a-z]*\.)')
train["Name"] = names

### From the boxplot we can see that sex and pclass seem to be good predictors
### of age. Now we fill in with average for different classes and sex.
cols = ["Sex", "Embarked", "Name"]
train = pd.get_dummies(train, columns = cols, drop_first = True)

# Drop unused series
train.drop(["Cabin", "PassengerId", "Ticket"], inplace = True, axis = 1)

def impute_age(cols):
    Age = cols[0]
    Pclass = cols[1]
    Sex = cols[2]

    if pd.isnull(Age):
        return train.loc[(train["Pclass"] == Pclass)
                         & (train["Sex_male"] == Sex)]["Age"].mean()
    else:
        return Age

train["Age"] = train[["Age", "Pclass", "Sex_male"]].apply(impute_age, axis = 1)

X_train, X_test, y_train, y_test = train_test_split(train.drop('Survived',axis=1),
                                                    train['Survived'], test_size=0.40,
                                                    random_state=101)

### Begin looking at different machine learning methods
###
### 1) Logistic Regression

from sklearn.linear_model import LogisticRegression

logmodel = LogisticRegression()
logmodel.fit(X_train, y_train)
log_pred = logmodel.predict(X_test)

print(classification_report(y_test, log_pred))
print(confusion_matrix(y_test, log_pred))

### 2) K Nearest Neighbours

from sklearn.neighbors import KNeighborsClassifier

error_rate = []

# Use elbow method to select n_neighbours - may take some time
for i in range(1,40):

    knn = KNeighborsClassifier(n_neighbors=i)
    knn.fit(X_train,y_train)
    pred_i = knn.predict(X_test)
    error_rate.append(np.mean(pred_i != y_test))

plt.figure(figsize=(10,6))
plt.plot(range(1,40),error_rate,color='blue', linestyle='dashed', marker='o',
         markerfacecolor='red', markersize=10)
plt.title('Error Rate vs. K Value')
plt.xlabel('K')
plt.ylabel('Error Rate')

# Graph shows somewhere between 5 and 10. I will choose 7.

knmodel = KNeighborsClassifier(n_neighbors = 7)
knmodel.fit(X_train,y_train)
kn_pred = knmodel.predict(X_test)

print(classification_report(y_test, kn_pred))
print(confusion_matrix(y_test, kn_pred))

### 3) Random Forests

from sklearn.ensemble import RandomForestClassifier

rfc = RandomForestClassifier(n_estimators = 200)
rfc.fit(X_train, y_train)
rfc_pred = rfc.predict(X_test)

print(classification_report(y_test, rfc_pred))
print(confusion_matrix(y_test, rfc_pred))


import torch
import torchvision

from maskrcnn_benchmark.structures.bounding_box import BoxList
from maskrcnn_benchmark.structures.segmentation_mask import SegmentationMask

from maskrcnn_benchmark.structures.ke import textKES
from maskrcnn_benchmark.structures.mty import MTY

DEBUG = 0

class WordDataset(torchvision.datasets.coco.CocoDetection):
    def __init__(
        self, ann_file, root, remove_images_without_annotations, transforms=None
    ):
        super(WordDataset, self).__init__(root, ann_file)
        # sort indices for reproducible results
        self.ids = sorted(self.ids)

        # filter images without detection annotations
        if remove_images_without_annotations:
            self.ids = [
                img_id
                for img_id in self.ids
                if len(self.coco.getAnnIds(imgIds=img_id, iscrowd=None)) > 0
            ]

        self.json_category_id_to_contiguous_id = {
            v: i + 1 for i, v in enumerate(self.coco.getCatIds())
        }
        self.contiguous_category_id_to_json_id = {
            v: k for k, v in self.json_category_id_to_contiguous_id.items()
        }
        self.id_to_img_map = {k: v for k, v in enumerate(self.ids)}
        self.transforms = transforms

    def kes_encode(self, kes):
        kes_encode = []
        for i in kes:
            mnx = i[0]
            mny = i[1]
            assert(len(i)%3 == 0)
            npts = int(len(i)/3-2)
            for index in range(npts):
                i[3+index*3]  = (i[3+index*3]+mnx)/2
                i[4+index*3]  = (i[4+index*3]+mny)/2
            kes_encode.append(i)
        return kes_encode

    def kes_gen(self, kes):
        kes_gen_out = []
        for i in kes:
            mnx = i[0]
            mny = i[1]
            cx= i[27]
            cy= i[28]
            assert(len(i)%3 == 0)
            ot = [mnx, i[3],i[6],i[9],i[12], cx,\
                  mny, i[16],i[19],i[22],i[25], cy]
            kes_gen_out.append(ot)
        return kes_gen_out

    def __getitem__(self, idx):
        img, anno = super(WordDataset, self).__getitem__(idx)

        # filter crowd annotations
        # TODO might be better to add an extra field
        anno = [obj for obj in anno if obj["iscrowd"] == 0]

        boxes = [obj["bbox"] for obj in anno]
        if DEBUG: print('len(boxes)', len(boxes), boxes[0])
        boxes = torch.as_tensor(boxes).reshape(-1, 4)  # guard against no boxes
        target = BoxList(boxes, img.size, mode="xywh").convert("xyxy")

        classes = [obj["category_id"] for obj in anno]
        if DEBUG: print('len(classes)', len(classes), classes[0])
        classes = [self.json_category_id_to_contiguous_id[c] for c in classes]
        classes = torch.tensor(classes)
        target.add_field("labels", classes)

        masks = [obj["segmentation"] for obj in anno]
        if DEBUG: print('len(masks)', len(masks), masks[0])
        masks = SegmentationMask(masks, img.size)
        target.add_field("masks", masks)

        if anno and 'keypoints' in anno[0]:
            kes = [obj["keypoints"] for obj in anno]
            kes = self.kes_gen(kes)
            if DEBUG: print('len(kes)', len(kes), kes[0])
            kes = textKES(kes, img.size)
            target.add_field("kes", kes)

        if anno and 'match_type' in anno[0]:
            mty = [obj["match_type"] for obj in anno]
            mty = MTY(mty, img.size)
            target.add_field("mty", mty)

        target = target.clip_to_image(remove_empty=True)

        if self.transforms is not None:
            img, target = self.transforms(img, target)

        return img, target, idx

    def get_img_info(self, index):
        img_id = self.id_to_img_map[index]
        img_data = self.coco.imgs[img_id]
        return img_data


import json
import pandas as pd
import xml.etree.ElementTree as ET
import pm4py
import uuid

def convert_to_sorted_list(s:str):
    if type(s) != str:return s
    l = json.loads(s.replace("\'","\""))
    if type(l)==list: l.sort()
    return l

def generate_data_disclosed_uuid(s:pd.Series,id_key="@_id",category_key="@category",legal_bases_key="tilt:legalBases",purposes_key="tilt:purposes")->dict:
    legal_bases = convert_to_sorted_list(s["tilt:legal_bases"])
    purposes = convert_to_sorted_list(s["tilt:purposes"])
    element={
        category_key:s["tilt:data_disclosed"],
        purposes_key:purposes,
        legal_bases_key:legal_bases
    }
    return uuid.uuid5(uuid.NAMESPACE_URL,str(element))

def build_data_disclosed_element(s:pd.Series)->dict:
    legal_bases = [{"reference": i ,"description":None} for i in convert_to_sorted_list(s["tilt:legal_bases"])]
    purposes = [{"purpose": i ,"description":None} for i in convert_to_sorted_list(s["tilt:purposes"])]
    return {
        "_id":s["id"],
        "category":s["tilt:data_disclosed"],
        "legalBases":legal_bases,
        "purposes":purposes
    }

def get_data_disclosed_df(df):
    tilt_columns = [tilt_column for tilt_column in df.columns if tilt_column.startswith("tilt:")]
    for c in tilt_columns:
        df[c] = df[c].apply(convert_to_sorted_list)
    dd_df = df[["ident:eid","concept:name"]+tilt_columns]
    for c in [c for c in tilt_columns if c != "tilt:data_disclosed"]:
        dd_df[c] = dd_df[c].apply(lambda x: str(x))
    dd_df = dd_df.set_index(["ident:eid","concept:name"]+[c for c in tilt_columns if c != "tilt:data_disclosed"])
    dd_df = dd_df.apply(lambda x: pd.Series(x).explode()).reset_index()
    dd_df["id"] = dd_df[tilt_columns].apply(generate_data_disclosed_uuid,axis=1)
    unique_dd = dd_df.drop_duplicates(subset="id")
    unique_dd["tilt:element"] = unique_dd.apply(build_data_disclosed_element,axis=1)
    return pd.merge(left=dd_df,right=unique_dd[["id","tilt:element"]], on="id")

def append_tilt_information_to_bpmn(bpmn,df):
    ns = {
    "bpmn":"http://www.omg.org/spec/BPMN/20100524/MODEL",
    "tilt":"mytilturi.com"}
    ET.register_namespace("tilt","mytiltURI.com")
    ET.register_namespace("bpmn","http://www.omg.org/spec/BPMN/20100524/MODEL")
    ET.register_namespace("bpmndi","http://www.omg.org/spec/BPMN/20100524/DI")
    ET.register_namespace("omgdc","http://www.omg.org/spec/DD/20100524/DC")
    ET.register_namespace("omgdi","http://www.omg.org/spec/DD/20100524/DI")
    ET.register_namespace("xsi","http://www.w3.org/2001/XMLSchema-instance")
    ET.register_namespace("xsd","http://www.w3.org/2001/XMLSchema")

    pm4py.write_bpmn(bpmn,"/tmp/temp.bpmn")
    tree = ET.parse("/tmp/temp.bpmn")
    root = tree.getroot()

    for task in root.find("bpmn:process",ns).findall("bpmn:task",ns):
        extensionElements = ET.SubElement(task,"bpmn:extensionElements")
        concept_name = task.attrib["name"]
        relevant_df = df[df["concept:name"]==concept_name].drop_duplicates("id")
        for id in relevant_df["id"].value_counts().index:
            elements = relevant_df[relevant_df["id"]==id]["tilt:element"].values
            if len(elements) != 0:
                dataElement = elements[0]
            else:
                continue
            dataDisclosed = ET.SubElement(extensionElements,"tilt:dataDisclosed")
            dataDisclosed.attrib["_id"] = str(id)
            dataDisclosed.attrib["category"] = dataElement["category"]
            for x in dataElement["legalBases"]:
                lb = ET.SubElement(dataDisclosed,"tilt:legalBases")
                lb.attrib["reference"] = x["reference"]
            for x in dataElement["purposes"]:
                lb = ET.SubElement(dataDisclosed,"tilt:purposes")
                lb.attrib["purpose"] = x["purpose"]
    tree.write("/tmp/temp_with_tilt.bpmn")
    with open("/tmp/temp_with_tilt.bpmn") as f:
        xml = f.read()
        f.close()
    return xml


# Code adapted from https://github.com/henzler/neuraltexture/blob/master/code/models/core_modules/standard_block.py

from torch import nn
import torch
from plan2scene.texture_gen.nets.core_layers import normalization, non_linearity

class StandardBlock(nn.Module):
    def __init__(self, type, nf_in, nf_out, kernel_size, stride, padding, norm_type=None, activation_type=None,
                 dropout_ratio=0.0, bias=True, padding_mode='zeros'):
        super().__init__()

        # shortcut if features have different size
        self.n_features_in = nf_in
        self.n_features_out = nf_out

        self.norm_type = norm_type
        self.bias = bias
        norm = normalization(norm_type)
        activation = non_linearity(activation_type)

        if type == 'linear':
            layer = torch.nn.Linear
            dropout = torch.nn.Dropout
            self.layer = layer(nf_in, nf_out, bias)

        elif type == 'conv_2d':
            layer = torch.nn.Conv2d
            dropout = torch.nn.Dropout2d
            self.layer = layer(nf_in, nf_out, kernel_size, stride, padding, bias, padding_mode=padding_mode)

        elif type == 'conv_transpose_2d':
            layer = torch.nn.ConvTranspose2d
            dropout = torch.nn.Dropout2d
            self.layer = layer(nf_in, nf_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias,
                               padding_mode=padding_mode)

        elif type == 'conv_3d':
            layer = torch.nn.Conv3d
            dropout = torch.nn.Dropout3d
            self.layer = layer(nf_in, nf_out, kernel_size, stride, padding, bias, padding_mode=padding_mode)

        elif type == 'conv_transpose_3d':
            layer = torch.nn.ConvTranspose3d
            dropout = torch.nn.Dropout3d
            self.layer = layer(nf_in, nf_out, kernel_size, stride, padding, biaspadding_mode=padding_mode)

        else:
            raise NotImplementedError

        self.activation = activation()
        self.dropout = dropout(dropout_ratio)

        if self.norm_type == 'specular':
            self.norm = norm(self.conv)
        else:
            self.norm = norm(nf_out)

    def forward(self, input, style=None):

        if self.norm_type == 'specular':
            output = self.norm(input)

        elif self.norm_type == 'adain':
            output = self.layer(input)

            output = self.norm(output, style)
        else:
            output = self.layer(input)

            # output = self.norm(output)

        output = self.activation(output)
        output = self.dropout(output)

        return output

class Conv2dBlock(StandardBlock):
    def __init__(self, nf_in, nf_out, kernel_size, stride, padding, norm_type=None, activation_type=None,
                 dropout_ratio=0.0, bias=True, padding_mode='zeros'):
        super().__init__('conv_2d', nf_in, nf_out, kernel_size, stride, padding, norm_type, activation_type,
                         dropout_ratio, bias, padding_mode=padding_mode)


import logging
import pymongo
from pymongo import MongoClient
from pymongo.collection import Collection
from functools import reduce
from datetime import datetime, timedelta
import os
import numpy as np

try:
    profile
except NameError:
    profile = lambda x: x

class Orderbooks:

    time_of_start_snapshot = None
    actual_end = None
    corrupt_order_book_count = 0

    def __init__(
            self,
            *,
            orderbooks_collection: Collection,
            envId: int,
            exchange: str,
            market: str,
            depth: float,
            start: datetime,
            end: datetime,
    ):

        self.start = start
        self.end = end
        self.depth = depth

        assert start < end

        self.buy_orderbook = []
        self.sell_orderbook = []

        start_snapshot = Orderbooks.get_start_snapshot(
            envId,
            exchange,
            market,
            start,
            orderbooks_collection=orderbooks_collection)

        self.buy_orderbook = start_snapshot["buy"]
        self.sell_orderbook = start_snapshot["sell"]

        # Ensure both buy and sell OBs are sorted
        self.buy_orderbook.sort(reverse=True)
        self.sell_orderbook.sort(reverse=False)

        self.time_of_start_snapshot = start_snapshot["ts"]

        last_orderbook = Orderbooks.get_last_orderbook(
            envId,
            exchange,
            market,
            start,
            end,
            orderbooks_collection=orderbooks_collection)

        self.actual_end = last_orderbook["ts"]

        if __debug__:
            logging.debug('start_snapshot: %r', start_snapshot)
            logging.debug('actual_end: %r', self.actual_end)

        # Setup an iterator to drive the simulation
        self.iter = iter(
            orderbooks_collection.find(
                filter={
                    "e": envId,
                    "x": exchange,
                    "m": market,
                    "ts": {
                        "$gte": self.time_of_start_snapshot,
                        "$lt": end
                    },
                    "s": {
                        "$exists": True
                    },
                    "V": "V", # valid flag
                }
            )
        )

    """
            ).sort([
                ("e", pymongo.ASCENDING),
                ("x", pymongo.ASCENDING),
                ("m", pymongo.ASCENDING),
                ("ts", pymongo.ASCENDING),
            ])
    """

    def __str__(self):
        return str(self.__class__) + '\n' + '\n'.join(
            '{} = {}'.format(
                item,
                self.__dict__[item]) for item in self.__dict__)

    def __iter__(self):
        return self

    @staticmethod
    def get_start_snapshot(
        envId: int,
        exchange: str,
        market: str,
        start: datetime,
        orderbooks_collection
    ):

        earlier_snapshots = list(orderbooks_collection.find(
            filter={
                "e": envId,
                "x": exchange,
                "m": market,
                # Look for a snapshot within the last 3 hours
                "ts": {
                    "$lte": start,
                    "$gte": start - timedelta(milliseconds=3*3600000)
                },
                "s": True,
            }))

        if len(earlier_snapshots) > 0:

            # Work with the most recent one
            start_snapshot = reduce(
                lambda x, y: x if x['ts'] > y['ts'] else y,
                earlier_snapshots,
            )

        else:

            # Nothing earlier; look for the next snapshot
            snapshots = list(
                orderbooks_collection.find(
                    filter={
                        "e": envId,
                        "x": exchange,
                        "m": market,
                        "ts": {
                            "$gte": start,
                        },
                        "s": True,
                    }).sort("ts", 1).limit(1)
            )
            if len(snapshots) == 0:
                raise StopIteration
            else:
                start_snapshot = snapshots[0]

        return start_snapshot

    @staticmethod
    def get_first_orderbook(
        envId: int,
        exchange: str,
        market: str,
        start: datetime,
        orderbooks_collection,
    ):

        # Nothing earlier; look for the next snapshot
        orderbooks = list(
            orderbooks_collection.find(
                filter={
                    "e": envId,
                    "x": exchange,
                    "m": market,
                    "ts": {
                        "$gte": start,
                    },
                })
            .sort("ts", 1).limit(1)
        )
        if len(orderbooks) == 0:
            return None
        else:
            return orderbooks[0]

    @staticmethod
    def get_last_orderbook(
        envId: int,
        exchange: str,
        market: str,
        start: datetime,
        end: datetime,
        orderbooks_collection,
    ):

        # Nothing earlier; look for the next snapshot
        orderbooks = list(
            orderbooks_collection.find(
                filter={
                    "e": envId,
                    "x": exchange,
                    "m": market,
                    "ts": {
                        "$gt": start,
                        "$lt": end,
                    },
                }).sort("ts", -1).limit(1)
        )

        if len(orderbooks) == 0:
            return None
        else:
            return orderbooks[0]

    @staticmethod
    def count_orderbooks(
        envId: int,
        exchange: str,
        market: str,
        start: datetime,
        end: datetime,
        orderbooks_collection,
    ):

        return orderbooks_collection.count_documents(
            filter={
                "e": envId,
                "x": exchange,
                "m": market,
                "ts": {
                    "$gte": start,
                    "$lt": end,
                },
                "V": "V",
                "s": {
                    "$exists": True
                }
            })

    @staticmethod
    def count_corrupt_orderbooks(
        envId: int,
        exchange: str,
        market: str,
        start: datetime,
        end: datetime,
        orderbooks_collection,
    ):

        return orderbooks_collection.count_documents(
            filter={
                "e": envId,
                "x": exchange,
                "m": market,
                "ts": {
                    "$gte": start,
                    "$lte": end,
                },
                "V": "C",
            })

    @profile
    def next(self):

        while True: # Until a clean OB

            while True: # Until prelimary OBs are removed

                orderbook = self.iter.next()

                if orderbook == None:
                    raise StopIteration

                assert "s" in orderbook

                if orderbook["s"] == True:

                    # Snapshot
                    self.buy_orderbook = orderbook["buy"]
                    self.sell_orderbook = orderbook["sell"]

                else:

                    # Delta
                    self.apply_deltas(orderbook)

                    orderbook["buy"] = self.buy_orderbook
                    orderbook["sell"] = self.sell_orderbook

                # First orderbooks not needed
                if self.start.replace(tzinfo=None) <= orderbook["ts"].replace(tzinfo=None):
                    break

            # Sanity check: best buy strictly less than best sell
            try:
                assert orderbook["buy"][0][0] < orderbook["sell"][0][0]
                break

            except AssertionError as err:
                # Log the first instance, otherwise just keep a count
                if self.corrupt_order_book_count == 0:
                    msg = f'OB Corruption: bestBuy={orderbook["buy"][0][0]} >= bestSell={orderbook["sell"][0][0]} @ {orderbook["ts"]}'
                    logging.error (msg)
                self.corrupt_order_book_count += 1

        orderbook["buy"] = np.array(orderbook["buy"], dtype=float)
        orderbook["sell"] = np.array(orderbook["sell"], dtype=float)

        assert len(orderbook["buy"]) > 0
        assert len(orderbook["sell"]) > 0

        return orderbook

    def apply_deltas(self, orderbook):

        assert self.buy_orderbook != None
        assert self.sell_orderbook != None

        # Apply the deltas to both sides
        self.apply_delta(self.buy_orderbook, orderbook["buy"])
        self.apply_delta(self.sell_orderbook, orderbook["sell"])

        self.buy_orderbook.sort(reverse=True)
        self.sell_orderbook.sort(reverse=False)

        # Ensure buy_orderbook has unique entries
        assert len(self.buy_orderbook) == len(
            set([x[0] for x in self.buy_orderbook]))

        # Ensure sell_orderbook has unique entries
        assert len(self.sell_orderbook) == len(
            set([x[0] for x in self.sell_orderbook]))

    def apply_delta(self, orderbook, deltas):

        for delta in deltas:

            for idx, entry in enumerate(orderbook):
                if entry[0] == delta[1]:
                    del orderbook[idx]

            if delta[0] == 0:   # Insert
                orderbook.append([delta[1], delta[2]])

            elif delta[0] == 1:  # Remove
                pass

            elif delta[0] == 2:  # Update
                orderbook.append([delta[1], delta[2]])

    @staticmethod
    def apply_depth(depth: float, orderbook: np.array) -> np.array:

        if depth == 0:
            return orderbook

        i = 0

        for entry in np.cumsum(np.prod(orderbook, axis=1)):

            i += 1

            if entry >= depth:
                break

        return orderbook[0:i]