## 다종목 스윙(해당일 종가매수 해당일 종가매도) 백테스팅 템플릿

import create_data_base
import logic
import config

import pandas as pd
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import win32com
from datetime import datetime
import time
import sqlite3

resultdata = []
flag = 0

class Core():

    ## 파라미터 설정
    def __init__(self, stock_universe, k):

        self.stock_universe = stock_universe ## 종목 바스켓

        self.trade_fee = config.trade_fee  ## 왕복 수수료
        self.trade_tax = config.trade_tax
        self.position_size = config.position_size

    ## 종목별 포지션 설정
    def set_position(self, stock_code):

        stock_data = logic.add_technical_analyze_data(stock_code)
        position_array = logic.set_position(stock_data)

        return position_array

    ## 포지션에 따른 수익률 계산
    def calculate_profit(self, stock_code):

        position_data = self.set_position(stock_code)
        stock_data = logic.add_technical_analyze_data(stock_code)
        profit_array = [1] ## 첫날에는 매수를 하지 않기 때문에 수익률이 0%

        today = 1
        yesterday = 0

        while today < len(position_data):

            ## 매수 포지션
            ## 수익률 = 종가에 매수하였으므로 전일 수익률 * (1 - 증권 거래수수료)
            if position_data[yesterday] == 'Hold' and position_data[today] == 'Buy':
                temp_profit = 1 * (1 - (self.trade_fee/100))
                profit_array.append(temp_profit)

            ## 매수 유지 포지션
            ## 수익률 = 전일 수익률 * 전일 종가 / 금일 종가
            elif position_data[yesterday] == 'Buy' and position_data[today] == 'Buy':
                temp_profit = 1 * (stock_data.at[today, "Close"] / stock_data.at[yesterday, "Close"])
                profit_array.append(temp_profit)

            ## 매도 포지션
            ## 수익률 = 전일 수익률 * 전일 종가 / 금일 종가 * (1 - 증권 거래수수료 - 증권거래세)
            elif position_data[yesterday] == 'Buy' and position_data[today] == 'Hold':
                temp_profit = 1 * (stock_data.at[today, "Close"] / stock_data.at[yesterday, "Close"]) * (1 - (self.trade_fee/100) - (self.trade_tax/100))
                profit_array.append(temp_profit)

            ## 현금 보유 포지션
            else:
                temp_profit = 1
                profit_array.append(temp_profit)

            today += 1
            yesterday += 1

        return profit_array

    ## 각 종목별 수익률 데이터 합성
    def merge_total_trade_data(self):

        ## 총 수익 데이터 저장용 Dataframe 선언
        total_trade_data = pd.DataFrame()

        ## 종목별 수익률 로드후 데이터프레임에 저장
        for i in range(len(self.stock_universe)):
            temp_profit_data = self.calculate_profit(self.stock_universe[i])
            temp_profit_data.reverse()
            total_trade_data.insert(len(total_trade_data.columns), "%s" %self.stock_universe[i], pd.Series(temp_profit_data))
            i += 1

        ## 종목별 수익률의 평균 산출
        total_trade_data = total_trade_data.reindex(index=total_trade_data.index[::-1])
        total_trade_tata = total_trade_data.fillna(1, inplace=True)
        total_trade_data["profit_average"] = total_trade_data.mean(axis=1)

        print(total_trade_data)
        return total_trade_data

    ## 수익률에 따른 잔고 반영
    def calculate_account_data(self):

        ## 기본 잔고 = 1000
        account_data = [1000]
        profit_data = self.merge_total_trade_data()

        ## 기본 잔고 = 1000
        for i in range(len(profit_data.index)):
            temp_account_data = account_data[i] * self.position_size * profit_data.at[i, "profit_average"] + account_data[i] * (1 - self.position_size)
            account_data.append(temp_account_data)
            i += 1

        account_data_dataframe = pd.DataFrame(account_data, columns = ['Basket'])
        account_data_dataframe = pd.concat([account_data_dataframe, profit_data], axis=1)

        temp_date_data = logic.add_technical_analyze_data(self.stock_universe[0])
        account_data_dataframe = pd.concat([account_data_dataframe, temp_date_data['Date']], axis=1)

        print(account_data_dataframe)
        return account_data_dataframe


# mod1.py

# loop comprehensions
loop = [[(num1, num2) for num2 in range(5)] for num1 in range(5)]
print(type(loop))
print(loop)

loop2 = [i for i in range(0, 50)]
print(loop2)

# loop comprehension with mixed list/tuples
movies = [("Gump", 1941), ("Terry", 2001), ("Wind", 1986), ("Gary", 1976),
          ("Tom", 1934), ("Ads", 2010), ("Grim", 2001), ("Gruff", 2019)]
gmovies = [(title, year) for (title, year) in movies if year < 2002 and title.startswith("G")]
print(gmovies)

# importing & creating modules - also see module.py

odds = [i for i in range(1, 11, 2)]
evens = [i for i in range(2, 11, 2)]
print(odds)
print(evens)
print(module.prodl(odds))
print(module.prodl(evens))
print(module.suml(odds))
print(module.suml(evens))

for path in sys.path:
    print(path)
print(iot.FunI())
print(sig.FunS())

import pygame

run = True
width = 400
height = 100
pygame.init()
screen = pygame.display.set_mode((width, height))
font = pygame.font.SysFont(None, 48)
text = font.render("Welcome to PyGame", True, (255, 255, 255))
screen.blit(text, ((width - text.get_width()) // 2, (height - text.get_height()) // 2))
pygame.display.flip()
while run:
    for event in pygame.event.get():
        if event.type == pygame.QUIT or event.type == pygame.MOUSEBUTTONUP or event.type == pygame.KEYUP:
            run = False

names = ["gary", "john", "terry", "berry", "gerry"]

for index, name in enumerate(names, start=1):
    print(index, name)

names = ["gary", "john", "terry", "berry", "gerry"]
nicks = ["carry", "con", "merry", "ferry", "very"]
list = [num for num in range(5)]

for name, nick, num in zip(names, nicks, list):         # use zip to combine sets of lists.
    print(name, nick, num)                              # stops when shortest list is exhausted


import argparse
import tensorflow as tf
import numpy as np
from models.model_builder import ModelBuilder
import cv2
import timeit
import pathlib

parser = argparse.ArgumentParser()
parser.add_argument("--batch_size",     type=int,
                    help="Evaluation batch size", default=1)
parser.add_argument("--num_classes",     type=int,
                    help="Model num classes", default=2)
parser.add_argument("--image_size",     type=tuple,
                    help="Model image size (input resolution)", default=(640, 360))
parser.add_argument("--checkpoint_dir", type=str,
                    help="Setting the model storage directory", default='./checkpoints/')
parser.add_argument("--weight_name", type=str,
                    help="Saved model weights directory", default='/0919/_0919_test_human_seg_640x360_pidnet_best_loss.h5')

args = parser.parse_args()

if __name__ == '__main__':
    # model = ModelBuilder(image_size=args.image_size, num_classes=args.num_classes).build_model()
    from models.model_zoo.PIDNet import PIDNet

    model = PIDNet(input_shape=(*args.image_size, 3), m=2, n=3, num_classes=args.num_classes,
                planes=32, ppm_planes=96, head_planes=128, augment=False).build()
    # model.load_weights(args.checkpoint_dir + args.model_weights, by_name=True)

    model.load_weights(args.checkpoint_dir + args.weight_name)
    model.summary()

    tflite_models_dir = pathlib.Path(args.checkpoint_dir + "quant_models/")
    tflite_models_dir.mkdir(exist_ok=True, parents=True)

    converter = tf.lite.TFLiteConverter.from_keras_model(model)
    converter.optimizations = [tf.lite.Optimize.DEFAULT]
    converter.target_spec.supported_types = [tf.float16]
    tflite_model = converter.convert()
    tflite_model_fp16_file = tflite_models_dir/"quant_test.tflite"
    tflite_model_fp16_file.write_bytes(tflite_model)

    # interpreter_fp16 = tf.lite.Interpreter(tflite_model)
    # interpreter_fp16.allocate_tensors()

    #  # Camera
    # frame_width = 1280
    # frame_height = 720
    # capture = cv2.VideoCapture(0)
    # capture.set(cv2.CAP_PROP_FRAME_WIDTH, frame_width)
    # capture.set(cv2.CAP_PROP_FRAME_HEIGHT, frame_height)

    # while cv2.waitKey(33) < 0:
    #     ret, frame = capture.read()

    #     start_t = timeit.default_timer()

    #     frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

    #     img = tf.image.resize(frame, size=args.image_size,
    #             method=tf.image.ResizeMethod.BILINEAR)
    #     img = tf.cast(img, tf.float32)
    #     img = tf.keras.applications.imagenet_utils.preprocess_input(x=img, mode='torch')

    #     img = tf.expand_dims(img, axis=0)

    #     output = model.predict(img)

    #     semantic_output = tf.math.argmax(output, axis=-1)

    #     output = semantic_output[0].numpy().astype(np.uint8) * 50

    #     terminate_t = timeit.default_timer()

    #     FPS = int(1./(terminate_t - start_t ))

    #     cv2.putText(output, str(FPS),(50, 70), cv2.FONT_HERSHEY_SIMPLEX, 1.2,
    #                     (200, 50, 0), 3, cv2.LINE_AA)
    #     cv2.imshow("VideoFrame", output)

    #     print(FPS)
    # capture.release()
    # cv2.destroyAllWindows()


import typing as t

# TODO: Good idea to maintain a count (length of ll) that gets updated with
#       each operation

class IndexOutOfRangeError(Exception):
    pass

class Node:
    def __init__(self, value: t.Any) -> None:
        self.value = value
        self.next = None

class LinkedList:
    def __init__(
        self,
        elements: t.Optional[t.Sequence[t.Any]] = None,
        key: t.Callable = lambda e: e,
    ) -> None:
        # TODO: Could remember the length and then update it
        self._key = key
        if not elements:
            self.head = None
        else:
            node = Node(elements[0])
            self.head = node
            for element in elements[1:]:
                new_node = Node(element)
                node.next = new_node
                node = new_node

    @property
    def length(self) -> int:
        if not self.head:
            return 0

        length = 0
        node = self.head
        while node:
            length += 1
            node = node.next
        return length

    @property
    def is_sorted(self) -> bool:
        if not self.head:
            return False
        if not self.head.next:
            return True
        node = self.head
        while node.next:
            value_current = node.value
            value_next = node.next.value
            if value_current > value_next:
                return False
            node = node.next
        return True

    def print(self, separator: str = ", ") -> None:
        if self.head:
            values = self._collect_all_values()
            print(separator.join(map(str, values)))

    def extend(self, ll: "LinkedList") -> None:
        if not ll.head:
            return
        if not self.head:
            self.head = ll.head
            return
        node = self.head
        while node.next:
            node = node.next
        node.next = ll.head

    def reverse(self) -> None:
        if not self.head:
            return
        node = self.head.next
        self.head.next = None
        while node:
            next_node = node.next
            node.next = self.head
            self.head = node
            node = next_node

    def get_index_of_value(self, value: t.Any) -> t.Optional[int]:
        if not self.head:
            return None
        node = self.head
        index = 0
        while node:
            if node.value == value:
                return index
            node = node.next
            index += 1

    def get_value_at_index(self, index: int) -> t.Optional[t.Any]:
        if not self.head:
            return None
        if index >= self.length or index < 0:
            raise IndexOutOfRangeError(
                "Wrong index value. Must be [0, len(LL) - 1]"
            )
        node = self.head
        i = 0
        while node:
            if i == index:
                return node.value
            i += 1
            node = node.next

    def prepend(self, value: t.Any) -> None:
        new_node = Node(value)
        if not self.head:
            self.head = new_node
            return
        new_node.next = self.head
        self.head = new_node

    def append(self, value: t.Any) -> None:
        new_node = Node(value)
        if not self.head:
            self.head = new_node
            return
        node = self.head
        while node.next:
            node = node.next
        node.next = new_node

    def insert_value_at(self, value: t.Any, index: int) -> None:
        new_node = Node(value)
        if not self.head:
            self.head = new_node
            return
        if index < 0:
            new_node.next = self.head
            self.head = new_node
            return
        node = self.head
        while node.next and index > 1:
            node = node.next
            index -= 1
        new_node.next = node.next
        node.next = new_node

    def delete_value(self, value: t.Any) -> bool:
        if not self.head:
            return False
        if self.head.value == value:
            if self.head.next:
                self.head = self.head.next
            else:
                self.head = None
            return True
        node = self.head
        while node.next and node.next.value != value:
            node = node.next
        if node.next:
            node.next = node.next.next
            return True
        return False

    def apply_function(self, func: t.Callable) -> None:
        """
        Modify each value of the LL
        :return:
        """
        if not self.head:
            return
        node = self.head
        while node:
            try:
                node.value = func(node.value)
            except Exception as e:
                print(
                    f"Failed while applying callable {func.__name__} to "
                    f"value {node.value}. Error: {e}"
                )
                raise e
            node = node.next

    def _collect_all_values(self) -> t.List[t.Any]:
        if not self.head:
            return []
        node = self.head
        values = []
        while node:
            values.append(node.value)
            node = node.next
        return values

    def insert_value_before(self):
        # TODO: Complete me
        pass

    def insert_value_after(self):
        # TODO: Complete me
        pass

    def insert_sorted_after(self):
        # TODO: Complete me
        pass

    def __len__(self) -> int:
        return self.length

    def __str__(self) -> str:
        if not self.head:
            return ""
        else:
            values = self._collect_all_values()
            return f"LinkedList: {' -> '.join(map(str, values))}"

if __name__ == "__main__":
    items = [4, 5, 1, 2, 7, 3]
    ll = LinkedList(items)

    print("Length:", ll.length, len(ll))

    print("\nPrinting LL:")
    ll.print("; ")
    print(ll)

    print("\nAppending item 100")
    ll.append(100)
    print(ll)

    print("\nPrepending item 228")
    ll.prepend(228)
    print(ll)

    print("\nGetting index of value 100")
    print(ll.get_index_of_value(100))

    print("\nGetting value at index 4")
    print(ll.get_value_at_index(4))

    print("\nInserting new value -10 at index 6")
    ll.insert_value_at(-10, index=6)
    print(ll)
    print("Inserting new values 0 and 1234 at -3 and 100 indices")
    ll.insert_value_at(0, index=-3)
    ll.insert_value_at(1234, index=100)
    print(ll)

    print("\nDeleting value -10", ll.delete_value(-10))
    print(ll)
    print("Deleting value 1234", ll.delete_value(1234))
    print(ll)
    print("Deleting value -1234", ll.delete_value(-1234))
    print(ll)

    print("\nApplying function x * 2 to all nodes")
    ll.apply_function(lambda e: e * 2)
    print(ll)

    print("\nIs LL sorted?", ll.is_sorted)
    new_ll = LinkedList([1, 2, 3, 4, 5])
    print(f"Is this LL: {new_ll} sorted? {new_ll.is_sorted}")

    print("\nReversing LL")
    ll.reverse()
    print(ll)


from sdv.model import Service

from sdv_model.proto.seats_pb2 import (
    CurrentPositionRequest,
    MoveComponentRequest,
    MoveRequest,
    Seat,
    SeatComponent,
    SeatLocation,
)
from sdv_model.proto.seats_pb2_grpc import SeatsStub

class SeatService(Service):
    """
    Seats service for getting and controlling the positions of the seats
    and their components in the vehicle.
    This definition corresponds to the COVESA Vehicle Service Catalog (VSC)
    comfort seats service definition
    (https://github.com/COVESA/vehicle_service_catalog)

    ...

    Methods
    ----------
    Move(seat=Seat)
        Set the desired seat position

    MoveComponent(seatLocation=SeatLocation, component=SeatComponent, position=int)
        Set a seat component position

    CurrentPosition(row=int, index=int)
        Get the current position of the addressed seat

    """

    def __init__(self):
        super().__init__()
        self._stub = SeatsStub(self.channel)

    async def Move(self, seat: Seat):
        """
        Summary
        -------
            Set the desired seat position

        Parameters
        ----------
        seat : Seat
            The seat which needs to be moved

        Returns
        -------
            * OK - Seat movement started
            * OUT_OF_RANGE - The addressed seat is not present in this vehicle
            * INVALID_ARGUMENT - At least one of the requested component positions is invalid
            * INTERNAL - A seat service internal error happened - see error message for details
        """
        response = await self._stub.Move(MoveRequest(seat=seat), metadata=self.metadata)
        return response

    async def MoveComponent(
        self,
        seatLocation: SeatLocation,
        component: SeatComponent,
        position: int,
    ):
        """
        Parameters
        ----------
        seatLocation : SeatLocation
            The location of the seat in which the component is present
        component : SeatComponent
            The component of the seat which needs to be moved
        position : int
            The units by which the component has to move

        Returns
        -------
            * OK - Seat movement started
            * OUT_OF_RANGE - The addressed seat is not present in this vehicle
            * NOT_FOUND - The addressed seat component is not supported by this seat/vehicle
            * INVALID_ARGUMENT - At least one of the requested component positions is invalid
            * INTERNAL - A seat service internal error happened - see error message for details

        """
        response = await self._stub.MoveComponent(
            MoveComponentRequest(
                seat=seatLocation,
                component=component,  # type: ignore
                position=position,
            ),
            metadata=self.metadata,
        )
        return response

    async def CurrentPosition(self, row: int, index: int):
        """
        Parameters
        ----------
        row : int
            The row of the seat
        index : int
            The index of the seat

        Returns
        -------
            * OK - Seat positions returned
            * OUT_OF_RANGE - The addressed seat is not present in this vehicle
        """
        response = await self._stub.CurrentPosition(
            CurrentPositionRequest(row=row, index=index),
            metadata=self.metadata,
        )
        return response


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging

from django.utils.translation import ugettext_lazy as _

from horizon import exceptions
from horizon import tables
from horizon import tabs

from openstack_dashboard.dashboards.project. \
    data_processing.utils import workflow_helpers as helpers

from openstack_dashboard.api import glance
from openstack_dashboard.api import network
from openstack_dashboard.api import neutron
from openstack_dashboard.api import nova

from openstack_dashboard.api import sahara as saharaclient

LOG = logging.getLogger(__name__)

class GeneralTab(tabs.Tab):
    name = _("General Info")
    slug = "cluster_details_tab"
    template_name = "project/data_processing.clusters/_details.html"

    def get_context_data(self, request):
        cluster_id = self.tab_group.kwargs['cluster_id']
        cluster_info = {}
        try:
            sahara = saharaclient.client(request)
            cluster = sahara.clusters.get(cluster_id)

            for info_key, info_val in cluster.info.items():
                for key, val in info_val.items():
                    if str(val).startswith(('http://', 'https://')):
                        cluster.info[info_key][key] = build_link(val)

            base_image = glance.image_get(request,
                                          cluster.default_image_id)

            if getattr(cluster, 'cluster_template_id', None):
                cluster_template = helpers.safe_call(
                    sahara.cluster_templates.get,
                    cluster.cluster_template_id)
            else:
                cluster_template = None

            if getattr(cluster, 'neutron_management_network', None):
                net_id = cluster.neutron_management_network
                network = neutron.network_get(request, net_id)
                net_name = network.name_or_id
            else:
                net_name = None

            cluster_info.update({"cluster": cluster,
                                 "base_image": base_image,
                                 "cluster_template": cluster_template,
                                 "network": net_name})
        except Exception as e:
            LOG.error("Unable to fetch cluster details: %s" % str(e))

        return cluster_info

def build_link(url):
    return "" + url + ""

class NodeGroupsTab(tabs.Tab):
    name = _("Node Groups")
    slug = "cluster_nodegroups_tab"
    template_name = (
        "project/data_processing.clusters/_nodegroups_details.html")

    def get_context_data(self, request):
        cluster_id = self.tab_group.kwargs['cluster_id']
        try:
            sahara = saharaclient.client(request)
            cluster = sahara.clusters.get(cluster_id)
            for ng in cluster.node_groups:
                if ng["flavor_id"]:
                    ng["flavor_name"] = (
                        nova.flavor_get(request, ng["flavor_id"]).name)
                if ng["floating_ip_pool"]:
                    ng["floating_ip_pool_name"] = (
                        self._get_floating_ip_pool_name(
                            request, ng["floating_ip_pool"]))

                if ng.get("node_group_template_id", None):
                    ng["node_group_template"] = helpers.safe_call(
                        sahara.node_group_templates.get,
                        ng["node_group_template_id"])
        except Exception:
            cluster = {}
            exceptions.handle(request,
                              _("Unable to get node group details."))

        return {"cluster": cluster}

    def _get_floating_ip_pool_name(self, request, pool_id):
        pools = [pool for pool in network.floating_ip_pools_list(
            request) if pool.id == pool_id]

        return pools[0].name if pools else pool_id

class Instance(object):
    def __init__(self, name=None, id=None, internal_ip=None,
                 management_ip=None):
        self.name = name
        self.id = id
        self.internal_ip = internal_ip
        self.management_ip = management_ip

class InstancesTable(tables.DataTable):
    name = tables.Column("name",
                         link=("horizon:project:instances:detail"),
                         verbose_name=_("Name"))

    internal_ip = tables.Column("internal_ip",
                                verbose_name=_("Internal IP"))

    management_ip = tables.Column("management_ip",
                                  verbose_name=_("Management IP"))

    class Meta:
        name = "cluster_instances"
        # Just ignoring the name.
        verbose_name = _(" ")

class InstancesTab(tabs.TableTab):
    name = _("Instances")
    slug = "cluster_instances_tab"
    template_name = "project/data_processing.clusters/_instances_details.html"
    table_classes = (InstancesTable, )

    def get_cluster_instances_data(self):
        cluster_id = self.tab_group.kwargs['cluster_id']

        try:
            sahara = saharaclient.client(self.request)
            cluster = sahara.clusters.get(cluster_id)

            instances = []
            for ng in cluster.node_groups:
                for instance in ng["instances"]:
                    instances.append(Instance(
                        name=instance["instance_name"],
                        id=instance["instance_id"],
                        internal_ip=instance.get("internal_ip",
                                                 "Not assigned"),
                        management_ip=instance.get("management_ip",
                                                   "Not assigned")))
        except Exception:
            instances = []
            exceptions.handle(self.request,
                              _("Unable to fetch instance details."))
        return instances

class ClusterDetailsTabs(tabs.TabGroup):
    slug = "cluster_details"
    tabs = (GeneralTab, NodeGroupsTab, InstancesTab, )
    sticky = True


from dataclasses import dataclass
from typing import List

@dataclass
class Book:
    title:str
    author:str
    borrower_name:str

def add_books(book_log: List[Book])->None:
    print("Title?")
    title = input("> ")
    print("Author(s)?")
    author = input("> ")
    book_log.append(Book(title,author,""))

def view_books(book_log: List[Book])->None:
    i = 0
    for book in book_log:
        i += 1
        if len(book.borrower_name) == 0:
            availability = "Available"
        else:
            availability = "Unavailable"
        print(f"{i}. [{availability}] {book.title} by {book.author}")

def loan_books(book_log: List[Book])->None:
    print("Which book?")
    i = 0
    for book in book_log:
        i += 1
        if len(book.borrower_name) == 0:
            print(f"{i}. {book.title} by {book.author}")
    which_book = int(input("> "))
    book = book_log[which_book -1]
    print("Borrower's name?")
    borrower = input("> ")
    book.borrower_name = borrower

def return_books(book_log: List[Book])->None:
        print("Title?")
        title = input("> ")
        for book in book_log:
            if book.title == title:
                print("Borrower's name?")
                borrower = input("> ")
                if book.borrower_name == borrower:
                    book.borrower_name = ""

def main()-> None:
    book_log = []
    while True:
        print("add [book], [view] inventory, [loan] book, [return] book, [quit]?")
        choice = input("> ")
        if choice == "book":
            add_books(book_log)
        if choice == "view":
            view_books(book_log)
        if choice == "loan":
            loan_books(book_log)
        if choice == "return":
            return_books(book_log)
        if choice == "quit":
            break
if __name__ == '__main__':
    main()


"""
credit to https://github.com/PredatH0r/XonStat/blob/master/xonstat/elo.py .

ELO algorithm to calculate player ranks

"""
from datetime import datetime
import logging
import math
from botocore.exceptions import ClientError
import json
import boto3
from collections import namedtuple
import time as _time

log_level = logging.INFO
logging.basicConfig(format='%(name)s:%(levelname)s:%(message)s')
logger = logging.getLogger("elo_calc")
logger.setLevel(log_level)
debug = False  # very local debug

class PlayerElo(object):
    def __init__(self, player_id=None, game_type_cd=None, elo=None):
        self.create_dt = datetime.utcnow()
        self.player_id = player_id
        self.game_type_cd = game_type_cd
        self.elo = 100.0
        self.g2_r = None
        self.g2_rd = None
        self.score = 0
        self.games = 0
        self.g2_games = 0
        self.g2_dt = None
        self.b_r = None
        self.b_rd = None
        self.b_games = 0
        self.b_dt = None

    def __repr__(self):
        return "<PlayerElo(pid=%s, gametype=%s, elo=%s, games=%s)>" % (self.player_id, self.game_type_cd, self.elo, self.games)

    def to_dict(self):
        return {
          'player_id':self.player_id, 'game_type_cd':self.game_type_cd, 'elo':self.elo, 'games':self.games,
          'g2_r':self.g2_r, 'g2_rd':self.g2_rd,  "g2_games":self.g2_games, "g2_dt":self.g2_dt,
          'b_r':self.b_r, 'b_rd':self.b_rd,  "b_games":self.b_games, "b_dt":self.b_dt
        }

class EloParms:
    def __init__(self, global_K = 15, initial = 100, floor = 80, logdistancefactor = math.log(10)/float(400), maxlogdistance = math.log(10)):
        self.global_K = global_K
        self.initial = initial
        self.floor = floor
        self.logdistancefactor = logdistancefactor
        self.maxlogdistance = maxlogdistance

class KReduction:
    #KREDUCTION = KReduction(900, 75, 0.5, 5, 15, 0.2)
    def __init__(self, fulltime, mintime, minratio, games_min, games_max, games_factor):
        self.fulltime = fulltime
        self.mintime = mintime
        self.minratio = minratio
        self.games_min = games_min
        self.games_max = games_max
        self.games_factor = games_factor

    def eval(self, mygames, mytime, matchtime):
        if mytime < self.mintime:
            return 0
        if mytime < self.minratio * matchtime:
            return 0
        if mytime < self.fulltime:
            k = mytime / float(self.fulltime)
        else:
            k = 1.0
        if mygames >= self.games_max:
            k *= self.games_factor
        elif mygames > self.games_min:
            k *= 1.0 - (1.0 - self.games_factor) * (mygames - self.games_min) / float(self.games_max - self.games_min)

        return k

# parameters for K reduction
# this may be touched even if the DB already exists

KREDUCTION = KReduction(900, 75, 0.5, 3, 10, 0.2)

# parameters for chess elo
# only global_K may be touched even if the DB already exists
# we start at K=200, and fall to K=40 over the first 20 games
ELOPARMS = EloParms(global_K=200)

def process_rtcwpro_elo(ddb_table, ddb_client, match_id, log_stream_name):
    "RTCWPro pipeline specific logic."
    t1 = _time.time()
    sk = match_id

    response = get_item("statsall", sk, ddb_table, log_stream_name)
    if "error" not in response:
        stats = json.loads(response["data"])
        logger.info("Retrieved statsall for " + str(len(stats)) + " players")
        stats = convert_stats_to_dict(stats)
    else:
        message = "Failed to retrieve statsall."
        logger.error(message)
        return message
        logger.error(json.dumps(response))

    response = get_item("wstatsall", sk, ddb_table, log_stream_name)
    if "error" not in response:
        wstats = json.loads(response["data"])
        logger.info("Retrieved wstatsall for " + str(len(wstats)) + " players")
    else:
        message = "Failed to retrieve wstatsall."
        logger.error(message)
        return message
        logger.error(json.dumps(response))

    response = get_item("match", sk + "2", ddb_table, log_stream_name)  # round 2
    if "error" not in response:
        match = json.loads(response["data"])
        match_region = response["lsipk"].split("#")[0]
        match_type = response["lsipk"].split("#")[1]
        match_region_type = match_region + "#" + match_type
        try:
            time_split = match["time_limit"].split(":")
            duration = int(time_split[0]) * 60 + int(time_split[1])
        except Exception:
            duration = 600
        winner = match["winner"]
    else:
        message = "Failed to retrieve match."
        logger.error(message)
        return message
        logger.error(json.dumps(response))

    real_name_request_list = prepare_player_name_list(stats)
    response = get_batch_items(real_name_request_list, ddb_table, log_stream_name)
    real_names = {}
    if "error" not in response:
        for result in response:
            guid = result["pk"].split("#")[1]
            if "data" in result:
                real_names[guid] = result["data"]
                logger.info("Retrieved " + guid + " name: " + result["data"])

    elo_item_list = prepare_player_elo_list(stats, match_region_type)
    response = get_batch_items(elo_item_list, ddb_table, log_stream_name)

    elo_dict = {}
    elo_games = {}
    if "error" not in response:
        for result in response:
            guid = result["pk"].split("#")[1]
            if match_region_type in result["sk"]:
                elo_dict[guid] = result["data"]
                elo_games[guid] = int(result["games"])
                logger.info("Retrieved " + match_region_type + "#elo" + " " + " " + real_names.get(guid,"no_name").ljust(20) + " elo:" + str(elo_dict[guid]) + " games " + str(elo_games[guid]))

    else:
        logger.error("Failed to retrieve any player elos.")
        logger.error(json.dumps(response))

    Player = namedtuple('Player', 'score duration games')

    new_wstats = {}
    for wplayer_wrap in wstats:
        for wplayer_guid, wplayer in wplayer_wrap.items():
            new_wplayer = {}
            for weapon in wplayer:
                new_wplayer[weapon["weapon"]] = weapon
            new_wstats[wplayer_guid] = new_wplayer

    player_scores = {}
    for guid, player_stats in stats.items():
        score_step_1 = player_stats["categories"].get("kills", 0)
        score_step_2 = score_step_1 \
            - wstat(new_wstats, guid, "Panzer", "kills") * .30 \
            - wstat(new_wstats, guid, "Artillery", "kills") * .30 \
            - wstat(new_wstats, guid, "Airstrike", "kills") * .30 \
            - wstat(new_wstats, guid, "Mauser", "kills") * .30

        win_multiplier = 1.5 if player_stats["team"] == winner else 1
        score_step_3 = int(score_step_2 * win_multiplier)

        player_scores[guid] = Player(score_step_3, duration, elo_games.get(guid,0)) # if elo is not there, default will be 0 anyway
        if debug:
            logger.info(player_stats.get("alias", "missing_alias").ljust(20) + str(player_stats["categories"].get("kills", 0)).ljust(5) + str(win_multiplier).ljust(5) + str(score_step_3).ljust(5) + str(player_scores[guid].games).ljust(5))

    (elo_deltas, elos) = process_elos(player_scores, elo_dict)

    items = []
    elo_delta_items = ddb_prepare_eloprogress_items(player_scores, elos, elo_deltas, match_id, match_region_type, real_names)
    player_elo_items = ddb_prepare_player_elo_items(elos, match_region_type, real_names)
    items.extend(elo_delta_items)
    items.extend(player_elo_items)

    try:
        ddb_batch_write(ddb_client, ddb_table.name, items)
    except Exception as ex:
        template = "An exception of type {0} occurred. Arguments:\n{1!r}"
        error_msg = template.format(type(ex).__name__, ex.args)
        message = "Failed to load all eloprogress records for a match " + match_id + "\n" + error_msg
        logger.info(message)
        return message
    else:
        message = "Elo progress records inserted.\n"

    time_to_write = str(round((_time.time() - t1), 3))
    logger.info(f"Time to process ELOs is {time_to_write} s")
    return message

def convert_stats_to_dict(stats):
    if len(stats) == 2 and len(stats[0]) > 1: #stats grouped in teams in a list of 2 teams , each team over 1 player
        logger.info("Number of stats entries 2, trying to merge teams")
        stats_tmp = stats[0].copy()
        stats_tmp.update(stats[1])
    else:
        logger.info("Merging list into dict.")
        stats_tmp = {}
        for player in stats:
            stats_tmp.update(player)
    logger.info("New statsall has " + str(len(stats_tmp)) + " players in a " + str(type(stats_tmp)))
    return stats_tmp

def wstat(new_wstats, guid, weapon, metric):
    """Safely get a number from a deeply nested dict."""
    if guid not in new_wstats:
        value = 0
    elif weapon not in new_wstats[guid]:
        value = 0
    elif metric not in new_wstats[guid][weapon]:
        value = 0
    else:
        value = new_wstats[guid][weapon][metric]
    return value

def process_elos(player_scores, elo_dict):
    """Given the players perforance and previous record, calculate new ELOs."""
    duration = player_scores[list(player_scores.keys())[0]].duration

    scores = {}
    alivetimes = {}
    games = {}

    for guid, p in player_scores.items():
        scores[guid] = p.score
        alivetimes[guid] = p.duration  #quke clan arena only, keeping for code wholesomeness
        games[guid] = p.games

    player_ids = scores.keys()

    elos = {}
    for p in player_ids:
        if p in elo_dict:
            elos[p] = PlayerElo(p)
            elos[p].elo = elo_dict[p]
            elos[p].games = player_scores[p].games

    # ensure that all player_ids have an elo record
    for pid in player_ids:
        if pid not in elos.keys():
            elos[pid] = PlayerElo(pid, None, ELOPARMS.initial)

    for pid in list(player_ids):
        elos[pid].k = KREDUCTION.eval(games[pid], alivetimes[pid], duration)
        if elos[pid].k == 0:
            del(elos[pid])
            del(scores[pid])
            del(alivetimes[pid])

    elos = update_elos(elos, scores, ELOPARMS)

    return elos

def update_elos(elos, scores, ep):
    if len(elos) < 2:
        return elos

    pids = list(elos.keys())

    eloadjust = {}
    for pid in pids:
        eloadjust[pid] = 0.0

    for i in range(0, len(pids)):
        ei = elos[pids[i]]
        for j in range(i+1, len(pids)):
            ej = elos[pids[j]]
            si = scores[ei.player_id]
            sj = scores[ej.player_id]

            # normalize scores
            ofs = min(0, si, sj)
            si -= ofs
            sj -= ofs
            if si + sj == 0:
                si, sj = 1, 1 # a draw

            # real score factor
            scorefactor_real = si / float(si + sj)

            # duels are done traditionally - a win nets
            # full points, not the score factor

            #if game.game_type_cd == 'duel':
            if False:
                # player i won
                if scorefactor_real > 0.5:
                    scorefactor_real = 1.0
                # player j won
                elif scorefactor_real < 0.5:
                    scorefactor_real = 0.0
                # nothing to do here for draws

            # expected score factor by elo
            elodiff = min(ep.maxlogdistance, max(-ep.maxlogdistance,
                (float(ei.elo) - float(ej.elo)) * ep.logdistancefactor))
            scorefactor_elo = 1 / (1 + math.exp(-elodiff))

            # initial adjustment values, which we may modify with additional rules
            adjustmenti = scorefactor_real - scorefactor_elo
            adjustmentj = scorefactor_elo - scorefactor_real

            if debug:
                print("Player i: {0}".format(ei.player_id))
                print("Player i's K: {0}".format(ei.k))
                print("Player j: {0}".format(ej.player_id))
                print("Player j's K: {0}".format(ej.k))
                print("Scorefactor real: {0}".format(scorefactor_real))
                print("Scorefactor elo: {0}".format(scorefactor_elo))
                print("adjustment i: {0}".format(adjustmenti))
                print("adjustment j: {0}".format(adjustmentj))

            if scorefactor_elo > 0.5:
            # player i is expected to win
                if scorefactor_real > 0.5:
                # he DID win, so he should never lose points.
                    adjustmenti = max(0, adjustmenti)
                else:
                # he lost, but let's make it continuous (making him lose less points in the result)
                    adjustmenti = (2 * scorefactor_real - 1) * scorefactor_elo
            else:
            # player j is expected to win
                if scorefactor_real > 0.5:
                # he lost, but let's make it continuous (making him lose less points in the result)
                    adjustmentj = (1 - 2 * scorefactor_real) * (1 - scorefactor_elo)
                else:
                # he DID win, so he should never lose points.
                    adjustmentj = max(0, adjustmentj)

            eloadjust[ei.player_id] += adjustmenti
            eloadjust[ej.player_id] += adjustmentj

    elo_deltas = {}
    for pid in pids:
        old_elo = float(elos[pid].elo)
        new_elo = max(float(elos[pid].elo) + eloadjust[pid] * elos[pid].k * ep.global_K / float(len(elos) - 1), ep.floor)
        elo_deltas[pid] = new_elo - old_elo

        debug_player = "kek"
        if debug_player in elo_deltas:
            print(debug_player + " elo delta: " + str(elo_deltas[debug_player]))

        elos[pid].elo = new_elo
        elos[pid].games += 1
        #elos[pid].update_dt = datetime.utcnow()

        #log.debug("Setting Player {0}'s Elo delta to {1}. Elo is now {2} (was {3}).".format(pid, elo_deltas[pid], new_elo, old_elo))
        if debug:
            print("Setting Player {0}'s Elo delta to {1}. Elo is now {2} (was {3}).".format(pid, elo_deltas[pid], new_elo, old_elo))

    return (elo_deltas, elos)

def make_error_dict(message, item_info):
    """Make an error message for API gateway."""
    return {"error": message + " " + item_info}

def get_item(pk, sk, table, log_stream_name):
    """Get one dynamodb item."""
    item_info = pk + ":" + sk + ". Logstream: " + log_stream_name
    try:
        response = table.get_item(Key={'pk': pk, 'sk': sk})
    except ClientError as e:
        logger.warning("Exception occurred: " + e.response['Error']['Message'])
        result = make_error_dict("[x] Client error calling database: ", item_info)
    else:
        if "Item" in response:
            result = response['Item']
        else:
            result = make_error_dict("[x] Item does not exist: ", item_info)
    return result

def prepare_player_elo_list(stats, match_region_type):
    """Make a list of guids to retrieve from ddb."""
    item_list = []
    for guid, player_stats in stats.items():
        item_list.append({"pk": "player#"+guid, "sk": "elo#" + match_region_type})
    return item_list

def prepare_player_name_list(stats):
    """Make a list of guids to retrieve from ddb."""
    item_list = []
    for guid, player_stats in stats.items():
        item_list.append({"pk": "player#"+guid, "sk": "realname"})
    return item_list

def get_batch_items(item_list, ddb_table, log_stream_name):
    """Get items in a batch."""
    dynamodb = boto3.resource('dynamodb')
    item_info = "get_batch_items. Logstream: " + log_stream_name
    try:
        response = dynamodb.batch_get_item(RequestItems={ddb_table.name: {'Keys': item_list, 'ProjectionExpression': 'pk, sk, #data_value, real_name, games', 'ExpressionAttributeNames': {'#data_value': 'data'}}})
    except ClientError as e:
        logger.warning("Exception occurred: " + e.response['Error']['Message'])
        result = make_error_dict("[x] Client error calling database: ", item_info)
    else:
        if len(response["Responses"][ddb_table.name]) > 0:
            result = response["Responses"][ddb_table.name]
        else:
            result = make_error_dict("[x] Item does not exist: ", item_info)
    return result

def create_batch_write_structure(table_name, items, start_num, batch_size):
    """
    Create item structure for passing to batch_write_item
    :param table_name: DynamoDB table name
    :param items: large collection of items
    :param start_num: Start index
    :param num_items: Number of items
    :return: dictionary of tables to write to
    """

    serializer = boto3.dynamodb.types.TypeSerializer()
    item_batch = { table_name: []}
    item_batch_list = items[start_num : start_num + batch_size]
    if len(item_batch_list) < 1:
        return None
    for item in item_batch_list:
        item_serialized = {k: serializer.serialize(v) for k,v in item.items()}
        item_batch[table_name].append({'PutRequest': {'Item': item_serialized}})

    return item_batch

def ddb_batch_write(client, table_name, items):
        messages = ""
        num_items = len(items)
        logger.info(f'Performing ddb_batch_write to dynamo with {num_items} items.')
        start = 0
        batch_size = 25
        while True:
            # Loop adding 25 items to dynamo at a time
            request_items = create_batch_write_structure(table_name,items, start, batch_size)
            if not request_items:
                break
            try:
                response = client.batch_write_item(RequestItems=request_items)
            except ClientError as err:
                logger.error(err.response['Error']['Message'])
                logger.error("Failed to run full batch_write_item")
                raise
            if len(response['UnprocessedItems']) == 0:
                logger.info(f'Wrote a batch of about {batch_size} items to dynamo')
            else:
                # Hit the provisioned write limit
                logger.warning('Hit write limit, backing off then retrying')
                sleep_time = 5 #seconds
                logger.warning(f"Sleeping for {sleep_time} seconds")
                _time.sleep(sleep_time)

                # Items left over that haven't been inserted
                unprocessed_items = response['UnprocessedItems']
                logger.warning('Resubmitting items')
                # Loop until unprocessed items are written
                while len(unprocessed_items) > 0:
                    response = client.batch_write_item(RequestItems=unprocessed_items)
                    # If any items are still left over, add them to the
                    # list to be written
                    unprocessed_items = response['UnprocessedItems']

                    # If there are items left over, we could do with
                    # sleeping some more
                    if len(unprocessed_items) > 0:
                        sleep_time = 5 #seconds
                        logger.warning(f"Sleeping for {sleep_time} seconds")
                        _time.sleep(sleep_time)

                # Inserted all the unprocessed items, exit loop
                logger.warning('Unprocessed items successfully inserted')
                break
            if response["ResponseMetadata"]['HTTPStatusCode'] != 200:
                messages += f"\nBatch {start} returned non 200 code"
            start += 25

def ddb_prepare_eloprogress_items(player_scores, elos, elo_deltas, match_id, match_region_type, real_names):
    elo_delta_items = []

    for guid in elo_deltas:
        if guid in player_scores:
            performance_score = player_scores[guid].score
        if guid in elos:
            elo = elos[guid].elo
        elo_item = {
            'pk'    : "eloprogress#" + guid,
            'sk'    : match_region_type + "#" + match_id,
            'data'  : int(round(elo_deltas[guid],0)),
            'gsi1pk': "eloprogressmatch",
            'gsi1sk': match_id,
            'real_name': real_names.get(guid,""),
            'elo': int(round(elo, 0)),
            'performance_score': performance_score
            }
        elo_delta_items.append(elo_item)
    return elo_delta_items

def ddb_prepare_player_elo_items(elos, match_region_type, real_names):
    player_elo_items = []
    ts = datetime.now().isoformat()
    for guid, elo in elos.items():
        elo_item = ddb_prepare_elo_item(guid, match_region_type, elo.elo, elo.games, ts, real_names.get(guid,""))
        player_elo_items.append(elo_item)
    return player_elo_items

def ddb_prepare_elo_item(guid, match_region_type, elo, games, ts, real_name):
    elo = int(round(elo, 0))
    elo_item = {
            'pk'            : "player"+ "#" + guid,
            'sk'            : "elo#" + match_region_type,
            # 'lsipk'         : "",
            'gsi1pk'        : "leaderelo#" + match_region_type,
            'gsi1sk'        : str(elo).zfill(3),
            'data'          : str(elo),
            'games'         : games,
            'updated'       : ts,
            "real_name"     : real_name
        }
    logger.info("Setting   " + guid + " " + real_name + " elo:" + str(elo) + " games " + str(games))
    return elo_item


''' Inference for the DreamBooth model. '''

import os
import zipfile
import predictor

import runpod
from runpod.serverless.utils import download, upload, validator, rp_cleanup

MODEL = predictor.Predictor()
MODEL.setup()

def run(job):
    '''
    Run inference on the model.

    input format:
    {
        "instance_prompt": str,
        "class_prompt": str,
        "instance_data": str,
        "samples":[
            {
                "save_sample_prompt": str,
                "save_sample_negative_prompt": str,
                "guidance_scale": float,
                "save_inference_steps": int,
                "n_save_sample": int,
            },
             {
                "save_sample_prompt": str,
                "save_sample_negative_prompt": str,
                "save_guidance_scale": float,
                "save_infer_steps": int,
                "n_save_sample": int,
            },
        ]
    }

    '''
    job_input = job['input']

    job_input['seed'] = job_input.get('seed', int.from_bytes(os.urandom(2), "big"))

    job_input["instance_data"] = download.download_input_objects(job_input["instance_data"])[0]

    MODEL.samples = job_input.get("samples", None)

    job_results = MODEL.predict(
        instance_prompt=job_input["instance_prompt"],
        class_prompt=job_input["class_prompt"],
        instance_data=job_input["instance_data"],
        class_data=job_input.get("class_data", None),
        num_class_images=job_input.get("num_class_images", 50),
        pad_tokens=job_input.get("pad_tokens", False),
        with_prior_preservation=job_input.get("with_prior_preservation", True),
        prior_loss_weight=job_input.get("prior_loss_weight", 1.0),
        seed=job_input.get("seed", 512),
        resolution=job_input.get("resolution", 512),
        center_crop=job_input.get("center_crop", False),
        train_text_encoder=job_input.get("train_text_encoder", True),
        train_batch_size=job_input.get("train_batch_size", 1),
        sample_batch_size=job_input.get("sample_batch_size", 2),
        num_train_epochs=job_input.get("num_train_epochs", 1),
        max_train_steps=job_input.get("max_train_steps", 2000),
        gradient_accumulation_steps=job_input.get("gradient_accumulation_steps", 1),
        gradient_checkpointing=job_input.get("gradient_checkpointing", False),
        learning_rate=job_input.get("learning_rate", 1e-6),
        scale_lr=job_input.get("scale_lr", False),
        lr_scheduler=job_input.get("lr_scheduler", "constant"),
        lr_warmup_steps=job_input.get("lr_warmup_steps", 0),
        use_8bit_adam=job_input.get("use_8bit_adam", True),
        adam_beta1=job_input.get("adam_beta1", 0.9),
        adam_beta2=job_input.get("adam_beta2", 0.999),
        adam_weight_decay=job_input.get("adam_weight_decay", 1e-2),
        adam_epsilon=job_input.get("adam_epsilon", 1e-8),
        max_grad_norm=job_input.get("max_grad_norm", 1.0),

        # samples=job_input.get("samples", None),
        save_guidance_scale=job_input.get("save_guidance_scale", 7.5),
        save_sample_prompt=job_input.get("save_sample_prompt", None),
        save_sample_negative_prompt=job_input.get("save_sample_negative_prompt", None),
        n_save_sample=job_input.get("n_save_sample", 1),
        save_infer_steps=job_input.get("save_infer_steps", 50),
    )

    os.makedirs("output_objects", exist_ok=True)

    with zipfile.ZipFile(job_results, 'r') as zip_ref:
        zip_ref.extractall("output_objects")

    img_paths = []
    for file in os.listdir("output_objects/samples"):
        img_paths.append(os.path.join("output_objects/samples", file))

    job_output = []

    image_urls = upload.files(job['id'], img_paths)

    for index, image_url in enumerate(image_urls):
        job_output.append({
            "image": image_url,
            "seed": job_input['seed'] + index
        })

    rp_cleanup.clean(['cog_class_data', 'cog_instance_data', 'checkpoints'])

    return job_output

runpod.serverless.start({"handler": run})


#!/usr/bin/env python3
import csv
import glob
import os
import json

us_states_and_territories = [
    "Alabama",
    "Alaska",
    "Arizona",
    "Arkansas",
    "California",
    "Colorado",
    "Connecticut",
    "Delaware",
    "Florida",
    "Georgia",
    "Hawaii",
    "Idaho",
    "Illinois",
    "Indiana",
    "Iowa",
    "Kansas",
    "Kentucky",
    "Louisiana",
    "Maine",
    "Maryland",
    "Massachusetts",
    "Michigan",
    "Minnesota",
    "Mississippi",
    "Missouri",
    "Montana",
    "Nebraska",
    "Nevada",
    "New Hampshire",
    "New Jersey",
    "New Mexico",
    "New York",
    "North Carolina",
    "North Dakota",
    "Ohio",
    "Oklahoma",
    "Oregon",
    "Pennsylvania",
    "Rhode Island",
    "South Carolina",
    "South Dakota",
    "Tennessee",
    "Texas",
    "Utah",
    "Vermont",
    "Virginia",
    "Washington",
    "West Virginia",
    "Wisconsin",
    "Wyoming",
    "District of Columbia",
    "Puerto Rico",
    "Virgin Islands",
    "Guam",
    "American Samoa",
    "Northern Mariana Islands",
]

def country(row):
    if "Country/Region" in row:
        return row["Country/Region"]
    return row["Country_Region"]

def setcountry(row, val):
    if "Country/Region" in row:
        row["Country/Region"] = val
    else:
        row["Country_Region"] = val

def state(row):
    if "Province/State" in row:
        return row["Province/State"]
    return row["Province_State"]

def toint(n):
    try:
        return int(n)
    except ValueError:
        return 0

# process modifies the data table data in-place, given data file daily
def process(data, daily, date):
    # the daily file (at least recently) has the data broken down very
    # granularly, so the first thing we'll do is sum it up before we add it to
    # the total data
    dailydata = {}
    for row in daily:
        if country(row) == "Mainland China":
            setcountry(row, "China")

        if state(row) and country(row) == "US":
            # skip all regions that are not states or territories
            if state(row) not in us_states_and_territories:
                continue
            displayname = f"{state(row)}, {country(row)}"
        else:
            displayname = f"{country(row)}"

        dailydata.setdefault(displayname, {}).setdefault("confirmed", []).append(
            toint(row["Confirmed"])
        )
        dailydata[displayname].setdefault("deaths", []).append(toint(row["Deaths"]))
        dailydata[displayname].setdefault("recovered", []).append(
            toint(row["Recovered"])
        )

        # Because we treat the US specially, we also need to sum up our national rows:
        if country(row) == "US":
            displayname = "United States"
            dailydata.setdefault(displayname, {}).setdefault("confirmed", []).append(
                toint(row["Confirmed"])
            )
            dailydata[displayname].setdefault("deaths", []).append(toint(row["Deaths"]))
            dailydata[displayname].setdefault("recovered", []).append(
                toint(row["Recovered"])
            )

    for name, today in dailydata.items():
        try:
            data.setdefault(name, {}).setdefault("confirmed", []).append(
                sum(today["confirmed"])
            )
            data[name].setdefault("deaths", []).append(sum(today["deaths"]))
            data[name].setdefault("recovered", []).append(sum(today["recovered"]))
        except:
            print("failed summing row", name, today)
            raise

def main():
    datadir = "jhudata/csse_covid_19_data/csse_covid_19_daily_reports/"
    data = {}
    dates = []
    # it's important to read this in sorted order
    for fname in list(sorted(glob.glob(datadir + "*.csv"))):
        with open(fname, encoding="utf-8-sig") as dailyfile:
            # Grab the date from the filename, like "03-24-2020", and append it
            # to the dates array
            date = fname.split("/")[-1].split(".")[0]
            dates.append(date)

            daily = csv.DictReader(dailyfile)
            process(data, daily, date)

    for key in data:
        data[key]["displayName"] = key

    envelope = {"dates": dates, "data": data}
    for region in data.values():
        # if there are dates where we have data, but this region does not yet,
        # prepend zeros so we have a data point for every date
        region["confirmed"] = [0] * (len(dates) - len(region["confirmed"])) + region[
            "confirmed"
        ]
        region["deaths"] = [0] * (len(dates) - len(region["deaths"])) + region["deaths"]
        region["recovered"] = [0] * (len(dates) - len(region["recovered"])) + region[
            "recovered"
        ]

    # json.dump(envelope, open("data.json", "w"), indent=2)
    json.dump(envelope, open("data.json", "w"))

if __name__ == "__main__":
    main()


import uuid
import os
from installed_clients.DataFileUtilClient import DataFileUtil
from installed_clients.KBaseReportClient import KBaseReport
from installed_clients.WorkspaceClient import Workspace
from shutil import copyfile

class htmlreportutils:
    def __init__(self):
        pass

    def create_html_report(self, callback_url, output_dir , workspace_name):
        '''
        '''
        dfu = DataFileUtil(callback_url)
        report_name = 'kb_gsea_report_' + str(uuid.uuid4())
        report = KBaseReport(callback_url)
        #copyfile(os.path.join(os.path.dirname(__file__), 'index.html'),
        #         os.path.join(output_dir, 'index.html'))

        report_shock_id = dfu.file_to_shock({'file_path': output_dir,
                                            'pack': 'zip'})['shock_id']

        html_file = {
            'shock_id': report_shock_id,
            'name': 'index.html',
            'label': 'index.html',
            'description': 'HTMLL report for GSEA'
            }

        report_info = report.create_extended_report({
                        'direct_html_link_index': 0,
                        'html_links': [html_file],
                        'report_object_name': report_name,
                        'workspace_name': workspace_name
                    })
        return {
            'report_name': report_info['name'],
            'report_ref': report_info['ref']
        }


import math

import overpy.exception
import holoviews
import datashader.geo
import sonar.lowrance_log_parser

import logging
logger = logging.getLogger('sonar_map')

def AdjustToIncludeNearestLandmark(lon_min, lon_max, lat_min, lat_max):
    orig_lon_min = lon_min
    orig_lon_max = lon_max

    orig_lat_min = lat_min
    orig_lat_max = lat_max

    lon_dist = orig_lon_max - orig_lon_min
    lat_dist = orig_lat_max - orig_lat_min

    # Find the nearest suburb to the middle of the bounding box
    middle_lon = orig_lon_min + lon_dist / 2.0
    middle_lat = orig_lat_min + lat_dist / 2.0

    # Overpass seems to take degrees not web mercator metres, so convert the middle to use in overpass
    middle_lon_degrees, middle_lat_degrees = sonar.lowrance_log_parser.meters_to_lnglat(middle_lon, middle_lat)

    api = overpy.Overpass()
    radius = 100.0

    # Permit path area to be as small as 1/10 of overall map to provide context
    # So if we travel 10km, will look upto 100km
    max_radius = 10 * max((orig_lat_max - orig_lat_min), (orig_lon_max - orig_lon_min))

    # If only travell 1 cm, still permit the graph to search context up to 5km
    max_radius = max(max_radius, 5000)

    # @todo Add the nearest landmark to the cache meta-data and use it if already exists
    nearest_landmark_debug = ''
    node = None
    node_lon = None
    node_lat = None
    while node is None and radius <= max_radius:
        logger.debug('Trying radius: %s', radius)

        query = '<query type="node"><has-kv k="place" /><around lon="%s" lat="%s" radius="%s"/></query><print/>' % (middle_lon_degrees, middle_lat_degrees, radius)
        logger.debug('Query: %s', query)

        try: result = api.query(query)
        except overpy.exception.OverpassTooManyRequests as e:
            logger.warning('Failed to query nearby suburb: %s', e)
            break
        except overpy.exception.OverpassGatewayTimeout as e:
            logger.warning('Failed to query nearby suburb: %s', e)
            break
        logger.debug('Looking at result: %s nodes:%s ways:%s relations:%s areas:%s',
            result,
            result.get_node_ids(),
            result.get_way_ids(),
            result.get_relation_ids(),
            result.get_area_ids())

        radius *= 5

        min_dist_index = None
        distances = []
        for i in range(0, len(result.nodes)):
            logger.debug('Looking at result node: %s of %s', i, len(result.nodes))
            n = result.nodes[i]
            n_lon, n_lat = sonar.lowrance_log_parser.lnglat_to_meters(float(n.lon), float(n.lat))
            lat_dist = middle_lat - n_lat
            lon_dist = middle_lon - n_lon
            dist = math.sqrt((lat_dist * lat_dist) + (lon_dist * lon_dist))
            logger.debug ('Distance: %s from middle of travelled area to node: %s %s', dist, n, n.tags)
            distances.append(dist)

            if min_dist_index is None or dist < distances[min_dist_index]:
                min_dist_index = i

        if min_dist_index is not None:
            node = result.nodes[min_dist_index]
            node_lon, node_lat = sonar.lowrance_log_parser.lnglat_to_meters(float(node.lon), float(node.lat))

            logger.debug('Selecting the nearest landmark: %s to include in graph with lat:%s lon:%s to the center of the travelled area lat:%s lon:%s with boundary: %s, %s, %s, %s', node.tags, node_lat, node_lon, middle_lat, middle_lon, lon_min, lat_min, lon_max, lat_max)
            nearest_landmark_debug = str(node.tags['name'])

    logger.debug('Adjusting map edges using node: %s', node)

    if node is None:
        logger.warning('Failed to locate the nearest landmark to (%s, %s) within radius: %s', middle_lon_degrees, middle_lat_degrees, radius)
        # You can debug this using the overpass API : https://overpass-turbo.eu/#
        # The query might look something like:
        # [out:json][timeout:30];
        # (
        #  node(around:<radius>,<latitude>,<longitude>) ["place"];
        # );
        # out body;

    else:
        # So lets first adjust bbox to fit in the new node
        node_lat = float(node_lat)
        node_lon = float(node_lon)

        if node_lat < lat_min:
            diff = lat_min - node_lat
            lat_min -= (diff + (diff / 4))
            lat_max += (diff / 4)

        if node_lat > lat_max:
            diff = node_lat - lat_max
            lat_min -= (diff / 4)
            lat_max += (diff + (diff / 4))

        if node_lon < lon_min:
            diff = lon_min - node_lon
            lon_min -= (diff + (diff / 4))
            lon_max += (diff / 4)

        if node_lon > lon_max:
            diff = node_lon - lon_max
            lon_min -= (diff / 4)
            lon_max += (diff + (diff / 4))

    return (lon_min, lon_max, lat_min, lat_max, nearest_landmark_debug)

def GenMap(sonar_data, include_nearest_landmark=True):
    gps_longitude_range = sonar_data.longitude.min().values.item(), sonar_data.longitude.max().values.item()
    gps_latitude_range = sonar_data.latitude.min().values.item(), sonar_data.latitude.max().values.item()
    lon_min = gps_longitude_range[0]
    lon_max = gps_longitude_range[1]
    lat_min = gps_latitude_range[0]
    lat_max = gps_latitude_range[1]

    nearest_landmark_debug = ''
    if include_nearest_landmark:
        lon_min, lon_max, lat_min, lat_max, nearest_landmark_debug = AdjustToIncludeNearestLandmark(lon_min, lon_max, lat_min, lat_max)
        if nearest_landmark_debug != '':
            nearest_landmark_debug = '. With nearest landmark: ' + nearest_landmark_debug

    logger.debug('Generating graph inside area: (%s, %s) - (%s, %s)%s', lon_min, lat_min, lon_max, lat_max, nearest_landmark_debug)

    tiles = holoviews.Tiles('https://maps.wikimedia.org/osm-intl/{Z}/{X}/{Y}@2x.png', name="Wikipedia")
    tiles = tiles.opts(width=600, height=600)

    # Adjust the framing of the tiles to show the path area and not the entire world
    # From: https://examples.pyviz.org/nyc_taxi/nyc_taxi.html
    #left, bottom = datashader.geo.lnglat_to_meters(lon_min, lat_min)
    #right, top = datashader.geo.lnglat_to_meters(lon_max, lat_max)
    # @todo cleanup, we are now using web mercator metres everywhere
    left, bottom = (lon_min, lat_min)
    right, top = (lon_max, lat_max)

    # http://holoviews.org/_modules/holoviews/core/dimension.html
    tiles = tiles.redim(
        x=holoviews.Dimension('x', range=(left, right)),
        y=holoviews.Dimension('y', range=(bottom, top))
        )

    logger.debug('left:%s right:%s bottom:%s top:%s', left, right, bottom, top)
    #import code
    #code.interact(local=dict(globals(), **locals()))

    channel = sonar_data.sel(channel=sonar.lowrance_log_parser.PRIMARY)
    latitude_arr = channel.latitude.values
    longitude_arr = channel.longitude.values
    frame_index_arr = channel.frame_index.values
    depth_arr = channel.water_depth.values

    #logger.info('Creating vector field for the path')
    #x_arr = []
    #y_arr = []
    #angle_arr = []
    #magnitude_arr = []
    #for i in range(0, len(frame_index_arr)):
    #    x,y = datashader.geo.lnglat_to_meters(longitude_arr[i], latitude_arr[i])
    #    angle = channel.heading.values[i]
    #    magnitude = channel.speed_gps.values[i]
    #    x_arr.append(x)
    #    y_arr.append(y)
    #    angle_arr.append(angle)
    #    magnitude_arr.append(magnitude)
    ## http://holoviews.org/reference/elements/bokeh/VectorField.html
    #path = holoviews.VectorField((x_arr, y_arr, angle_arr, magnitude_arr))
    #path = path.opts(magnitude='Magnitude', color='Magnitude')
    #logger.info('Done')

    path_points = []
    for i in range(0, len(frame_index_arr)):
        # @todo cleanup
        #path_points.append(datashader.geo.lnglat_to_meters(longitude_arr[i], latitude_arr[i]))
        path_points.append((longitude_arr[i], latitude_arr[i], depth_arr[i]))
    path = holoviews.Path(path_points, vdims='depth', kdims=[
        holoviews.Dimension('easting'),
        holoviews.Dimension('northing')])
    path = path.opts(color='depth', width=600, height=600, cmap='viridis')

    # See https://holoviz.org/tutorial/Composing_Plots.html for the easting/northing plot
    # @todo Note: Sometimes the x/y axis in overlay changes to use the path values not the tiles values which is not what we want we want to displat lon/lat not northings eastings
    overlay = tiles * path
    return overlay


from itertools import combinations

class Solution:
    def readBinaryWatch(self, num: int) -> list:
        hours = [8, 4, 2, 1]
        mins = [32, 16, 8, 4, 2, 1]
        ans = []

        for i in range(num+1):
            if num - i > 6: continue

            hour = list(combinations(hours, i))
            minute = list(combinations(mins, num - i))

            for h in hour:
                h = sum(h)
                if h >= 12: continue
                for m in minute:
                    m = sum(m)
                    if m >= 60: continue
                    ans.append(str(h)+":"+str(m).zfill(2))

        return ans


# !/usr/bin/env python3
# @file example2_drawing_circle.py
# SCRP: Example 2 - Drawing Circle
# Daryl Dang

"""
Example 2 - Drawing Circle
--------------------------
This example goes over a simple way to draw a circle with the pygame.draw library.
"""

import pygame

# GLOBALS
WHITE = (255, 255, 255)

# Initialize pygame and display window
pygame.init()
display = pygame.display.set_mode((500, 500))
pygame.display.set_caption("Drawing Circle")

# Game loop
running = True
while running:
    for event in pygame.event.get():
        # Quit when the user presses the X at the top right
        if event.type == pygame.QUIT:
            running = False

    # Draw circle
    surface     = display       # what surface to draw on
    colour      = WHITE         # the colour of the circle
    center      = [250, 250]    # the center position of the circle
    radius      = 100           # radius of the circle
    width       = 0             # used for line thickness or to just fill the circle
    pygame.draw.circle(surface, colour, center, radius, width)

    # Update the pygame window
    pygame.display.update()

# Close and clean up
pygame.quit()


import math
import random

class Dataset:

    def __init__(self, samples, batch_size, multim=False, audiovid=False, audiotext=False, textvid=False, exp3=False,
                 decision_level=False):
        self.samples = samples
        self.batch_size = batch_size
        self.num_batches = math.ceil(len(self.samples) / batch_size)
        self.speaker_to_idx = {'M': 0, 'F': 1}
        self.multim = multim
        self.size = 100 if exp3 else 100 if decision_level else 712 if multim else 200 if audiotext else 612 if textvid else 612 if audiovid else 100
        self.audiovid = audiovid
        self.audiotext = audiotext
        self.textvid = textvid
        self.exp3 = exp3
        self.decision_level = decision_level

    def __len__(self):
        return self.num_batches

    def __getitem__(self, index):
        batch = self.raw_batch(index)
        return self.padding(batch)

    def raw_batch(self, index):
        assert index < self.num_batches, "batch_idx %d > %d" % (index, self.num_batches)
        batch = self.samples[index * self.batch_size: (index + 1) * self.batch_size]

        return batch

    def padding(self, samples):
        batch_size = len(samples)
        text_len_tensor = torch.tensor([len(s.text) for s in samples]).long()
        mx = torch.max(text_len_tensor).item()
        text_tensor = torch.zeros((batch_size, mx, self.size))
        a_tensor = torch.zeros((batch_size, mx, 100))
        v_tensor = torch.zeros((batch_size, mx, 512))
        audiovid_tensor = torch.zeros((batch_size, mx, 100)) if self.audiotext else torch.zeros(
            (batch_size, mx, 512)) if self.textvid else torch.zeros((batch_size, mx, 612))

        speaker_tensor = torch.zeros((batch_size, mx)).long()
        labels = []
        n_utterances = 0
        for i, s in enumerate(samples):
            cur_len = len(s.text)
            n_utterances += cur_len
            tmp = [torch.from_numpy(t).float() for t in s.text]
            tmp_vid = [torch.from_numpy(t).float() for t in s.visual]
            tmp_aud = [torch.from_numpy(t).float() for t in s.audio]
            tmp = torch.stack(tmp)
            tmp_vid = torch.stack(tmp_vid)
            tmp_aud = torch.stack(tmp_aud)
            if i == 0:
                video_tensor = tmp_vid
                audio_tensor = tmp_aud
            else:
                video_tensor = torch.cat((video_tensor, tmp_vid), 0)
                audio_tensor = torch.cat((audio_tensor, tmp_aud), 0)

            if self.multim:
                aud = torch.tensor(s.audio)
                vid = torch.tensor(s.visual)
                tmp = torch.cat((tmp, vid, aud), 1)
            elif self.audiovid and not self.exp3 and not self.decision_level:
                aud = torch.tensor(s.audio)
                vid = torch.tensor(s.visual)
                tmp = torch.cat((vid, aud), 1)
            elif self.audiotext and not self.exp3 and not self.decision_level:
                aud = torch.tensor(s.audio)
                tmp = torch.cat((tmp, aud), 1)
            elif self.textvid and not self.exp3 and not self.decision_level:
                vid = torch.tensor(s.visual)
                tmp = torch.cat((tmp, vid), 1)

            audvid = torch.cat((tmp_vid, tmp_aud), 1)
            if self.textvid:
                audiovid_tensor[i, :cur_len, :] = tmp_vid
            elif self.audiotext:
                audiovid_tensor[i, :cur_len, :] = tmp_aud
            else:
                audiovid_tensor[i, :cur_len, :] = audvid
            text_tensor[i, :cur_len, :] = tmp
            a_tensor[i, :cur_len, :] = tmp_aud
            v_tensor[i, :cur_len, :] = tmp_vid
            speaker_tensor[i, :cur_len] = torch.tensor([self.speaker_to_idx[c] for c in s.speaker])
            labels.extend(s.label)
        # video_tensor = torch.zeros((n_utterances, 512)) #added <- should be size(num_utterances in batch, vec_dim)
        # audio_tensor = torch.zeros((n_utterances, 100)) #added <- same as above

        label_tensor = torch.tensor(labels).long()
        data = {
            "text_len_tensor": text_len_tensor,
            "text_tensor": text_tensor,
            "speaker_tensor": speaker_tensor,
            "label_tensor": label_tensor,
            "video_tensor": video_tensor,
            "audio_tensor": audio_tensor,
            "audiovid_tensor": audiovid_tensor,
            "audio": a_tensor,
            "video": v_tensor

        }

        return data

    def shuffle(self):
        random.shuffle(self.samples)

import numpy as np
import torch

import utils

log = utils.get_logger()

def batch_graphify(features, lengths, speaker_tensor, wp, wf, edge_type_to_idx, att_model, device):
    node_features, edge_index, edge_norm, edge_type = [], [], [], []
    batch_size = features.size(0)
    length_sum = 0
    edge_ind = []
    edge_index_lengths = []

    for j in range(batch_size):
        edge_ind.append(edge_perms(lengths[j].cpu().item(), wp, wf))

    edge_weights = att_model(features, lengths, edge_ind)

    for j in range(batch_size):
        cur_len = lengths[j].item()
        node_features.append(features[j, :cur_len, :])
        perms = edge_perms(cur_len, wp, wf)
        perms_rec = [(item[0] + length_sum, item[1] + length_sum) for item in perms]
        length_sum += cur_len
        edge_index_lengths.append(len(perms))

        for item, item_rec in zip(perms, perms_rec):
            edge_index.append(torch.tensor([item_rec[0], item_rec[1]]))
            edge_norm.append(edge_weights[j][item[0], item[1]])
            # edge_norm.append(edge_weights[j, item[0], item[1]])

            speaker1 = speaker_tensor[j, item[0]].item()
            speaker2 = speaker_tensor[j, item[1]].item()
            if item[0] < item[1]:
                c = '0'
            else:
                c = '1'
            edge_type.append(edge_type_to_idx[str(speaker1) + str(speaker2) + c])

    node_features = torch.cat(node_features, dim=0).to(device)  # [E, D_g]
    edge_index = torch.stack(edge_index).t().contiguous().to(device)  # [2, E]
    edge_norm = torch.stack(edge_norm).to(device)  # [E]
    edge_type = torch.tensor(edge_type).long().to(device)  # [E]
    edge_index_lengths = torch.tensor(edge_index_lengths).long().to(device)  # [B]

    return node_features, edge_index, edge_norm, edge_type, edge_index_lengths

def edge_perms(length, window_past, window_future):
    """
    Method to construct the edges of a graph (a utterance) considering the past and future window.
    return: list of tuples. tuple -> (vertice(int), neighbor(int))
    """

    all_perms = set()
    array = np.arange(length)
    for j in range(length):
        perms = set()

        if window_past == -1 and window_future == -1:
            eff_array = array
        elif window_past == -1:  # use all past context
            eff_array = array[:min(length, j + window_future + 1)]
        elif window_future == -1:  # use all future context
            eff_array = array[max(0, j - window_past):]
        else:
            eff_array = array[max(0, j - window_past):min(length, j + window_future + 1)]

        for item in eff_array:
            perms.add((j, item))
        all_perms = all_perms.union(perms)
    return list(all_perms)


import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
from highlight_text import ax_text

def get_rolling_data(team, window, data):

    df_team = data[data['Team'] == team]
    df_team_for = df_team.groupby('Round_ID')['Score', 'xScore'].sum().rename(columns={'Score':"For",
                                                                                "xScore":"xFor"})
    df_opp = data[((data['Home_Team'] == team) | (data['Away_Team'] == team)) & (data['Team'] != team)]
    df_team_agg = df_opp.groupby('Round_ID')['Score', 'xScore'].sum().rename(columns={'Score':"Against",
                                                                                "xScore":"xAgainst"})

    df_rolling = pd.merge(df_team_for, df_team_agg, left_index=True, right_index=True)
    df_rolling['rollingFor'] = df_rolling['For'].rolling(window=window, min_periods=0).mean()
    df_rolling['rollingxFor'] = df_rolling['xFor'].rolling(window=window, min_periods=0).mean()
    df_rolling['rollingAgainst'] = df_rolling['Against'].rolling(window=window, min_periods=0).mean()
    df_rolling['rollingxAgainst'] = df_rolling['xAgainst'].rolling(window=window, min_periods=0).mean()

    df_rolling['rollingDiff'] = df_rolling['rollingFor'] - df_rolling['rollingAgainst']
    df_rolling['rollingxDiff'] = df_rolling['rollingxFor'] - df_rolling['rollingxAgainst']

    # Filling in missing Finals games with previous round (no change)
    finals_2021 = ['2021F1', '2021F2', '2021F3', '2021F4']
    finals_2021_prev = ['202123', '2021F1', '2021F2', '2021F3' ]
    finals_2022 = ['2022F1', '2022F2', '2022F3', '2022F4']
    finals_2022_prev = ['202223', '2022F1', '2022F2', '2022F3']
    for index in range(len(finals_2021)):
        if finals_2021[index] not in df_rolling.index:
            # print(finals_2021[index])
            df_rolling.loc[finals_2021[index]] = df_rolling.loc[finals_2021_prev[index]]

    for index in range(len(finals_2022)):
        if finals_2022[index] not in df_rolling.index:
            # print(finals_2022[index])
            df_rolling.loc[finals_2022[index]] = df_rolling.loc[finals_2022_prev[index]]

    # Filling in 202101 if necessary
    if '202101' not in df_rolling.index:
        df_rolling.loc['202101'] = df_rolling.loc['202102']

    # Relative performance compared to Expected
    df_rolling['rollingForRelative'] = df_rolling['rollingFor'] - df_rolling['rollingxFor']
    df_rolling['rollingAgainstRelative'] = df_rolling['rollingAgainst'] - df_rolling['rollingxAgainst']
    df_rolling['rollingDiffRelative'] = df_rolling['rollingDiff'] - df_rolling['rollingxDiff']

    df_rolling = df_rolling.sort_index()

    return df_rolling

def plot_rolling_average(ax, data, positive_line, negative_line, color_for, color_against):

    x = data.index
    y_for = data[positive_line]
    y_against = data[negative_line]

    ax.set_facecolor("#121212")
    ax.spines[['top', 'right']].set_visible(False)
    ax.spines[["left", "bottom"]].set_color("white")

    line_for = ax.plot(x, y_for, label = positive_line, color = color_for, lw=1.5)
    line_against = ax.plot(x, y_against, label = negative_line, color= color_against, lw=1.5)
    ax.set_ylim(0, 125)
    ax.plot([22, 22], [ax.get_ylim()[0], ax.get_ylim()[1]], ls=":", lw=1.25, color="white", zorder=2)
    ax.plot([25, 25], [ax.get_ylim()[0], ax.get_ylim()[1]], ls=":", lw=1.25, color="white", zorder=2)
    plt.axvspan(22, 25, color=line_for[0].get_color(), alpha=0.3)

    ax.plot([48, 48], [ax.get_ylim()[0], ax.get_ylim()[1]], ls=":", lw=1.25, color="white", zorder=2)
    ax.plot([51, 51], [ax.get_ylim()[0], ax.get_ylim()[1]], ls=":", lw=1.25, color="white", zorder=2)
    plt.axvspan(48, 51, color=line_for[0].get_color(), alpha=0.3)

    ax.fill_between(x, y_against, y_for, where = y_for > y_against, interpolate=True, alpha=0.85, zorder=3, color=line_for[0].get_color())
    ax.fill_between(x, y_against, y_for, where = y_against >= y_for, interpolate=True, alpha=0.85, zorder=3, color=line_against[0].get_color())

    ax.tick_params(color="white", length=5, which="major", labelsize=6, labelcolor="white")
    ax.xaxis.set_major_locator(ticker.MultipleLocator(10))
    ax.yaxis.set_major_locator(ticker.MultipleLocator(10))

    plt.axvspan(0, 10, color="grey", alpha=0.3)

    ax_text(
        x=7, y=20,
        s = "2021",
        color="white"
    )
    ax_text(
        x=35, y=20,
        s = "2022",
        color="white"
    )

    return ax

## Expected Score
def get_expected_rolling_plot(ax, team, data, window=10, color_for="#0057B8", color_against = "#989898"):

    df_rolling = get_rolling_data(team=team, window=window, data = data)

    ax = plot_rolling_average(ax, data=df_rolling, positive_line="rollingxFor", negative_line="rollingxAgainst", color_for = color_for, color_against = color_against)

    for_number = df_rolling['rollingxFor'].iloc[-1]
    against_number = df_rolling['rollingxAgainst'].iloc[-1]
    text_colour_for = "white"
    text_colour_against = "white"
    if color_for == "white":
        text_colour_for = "black"
    if color_against == "white":
        text_colour_against = "black"

    ax_text(
        x=0, y=140,
        s=f'<{team}>\n<xS for: {for_number:.1f}>  <xS against: {against_number:.1f}>',
        highlight_textprops=[
            {'color':'white', 'weight':'bold', 'font':'DM Sans'},
            {'size':'10', 'bbox':{'edgecolor':color_for, 'facecolor':color_for, 'pad':1}, 'color':text_colour_for},
            {'size':'10', 'bbox':{'edgecolor':color_against, 'facecolor':color_against, 'pad':1}, 'color':text_colour_against},
        ],
        font="Karla",
        ha="left",
        size=14,
    )

    return ax

## Actual Scores
def get_actual_rolling_plot(ax, team, data, window=10, color_for="#0057B8", color_against = "#989898"):

    df_rolling = get_rolling_data(team=team, window=window, data = data)

    ax = plot_rolling_average(ax, data=df_rolling, positive_line="rollingFor", negative_line="rollingAgainst", color_for = color_for, color_against = color_against)

    for_number = df_rolling['rollingFor'].iloc[-1]
    against_number = df_rolling['rollingAgainst'].iloc[-1]
    text_colour_for = "white"
    text_colour_against = "white"
    if color_for == "white":
        text_colour_for = "black"
    if color_against == "white":
        text_colour_against = "black"

    ax_text(
        x=0, y=140,
        s=f'<{team}>\n<For: {for_number:.1f}>  <Against: {against_number:.1f}>',
        highlight_textprops=[
            {'color':'white', 'weight':'bold', 'font':'DM Sans'},
            {'size':'10', 'bbox':{'edgecolor':color_for, 'facecolor':color_for, 'pad':1}, 'color':text_colour_for},
            {'size':'10', 'bbox':{'edgecolor':color_against, 'facecolor':color_against, 'pad':1}, 'color':text_colour_against},
        ],
        font="Karla",
        ha="left",
        size=14,
    )

    return ax

# Expected Score Differences
def plot_rolling_difference(ax, data, difference_line, color_for, color_against, reverse=False):

    x = data.index
    y_diff = data[difference_line]
    y_neutral = [0]*len(x)

    ax.set_facecolor("#121212")
    ax.spines[['top', 'right']].set_visible(False)
    ax.spines[["left", "bottom"]].set_color("white")

    line_diff = ax.plot(x, y_diff, label = difference_line, color = color_for, lw=1.5)
    line_neutral = ax.plot(x, y_neutral, color= "white", lw=1.5)
    ax.set_ylim(-20, 20)
    ax.plot([22, 22], [ax.get_ylim()[0], ax.get_ylim()[1]], ls=":", lw=1.25, color="white", zorder=2)
    ax.plot([25, 25], [ax.get_ylim()[0], ax.get_ylim()[1]], ls=":", lw=1.25, color="white", zorder=2)
    plt.axvspan(22, 25, color=line_diff[0].get_color(), alpha=0.3)

    ax.plot([48, 48], [ax.get_ylim()[0], ax.get_ylim()[1]], ls=":", lw=1.25, color="white", zorder=2)
    ax.plot([51, 51], [ax.get_ylim()[0], ax.get_ylim()[1]], ls=":", lw=1.25, color="white", zorder=2)
    plt.axvspan(48, 51, color=line_diff[0].get_color(), alpha=0.3)

    positive_colour = color_for
    negative_colour = "grey"
    if reverse:
        positive_colour = "grey"
        negative_colour = color_for
    ax.fill_between(x, y_neutral, y_diff, where = y_diff > y_neutral, interpolate=True, alpha=0.85, zorder=3, color=positive_colour)
    ax.fill_between(x, y_neutral, y_diff, where = y_neutral >= y_diff, interpolate=True, alpha=0.85, zorder=3, color=negative_colour)

    ax.tick_params(color="white", length=5, which="major", labelsize=6, labelcolor="white")
    ax.xaxis.set_major_locator(ticker.MultipleLocator(10))
    ax.yaxis.set_major_locator(ticker.MultipleLocator(10))

    plt.axvspan(0, 10, color="grey", alpha=0.3)

    ax_text(
        x=7, y=-15,
        s = "2021",
        color="white"
    )
    ax_text(
        x=35, y=-15,
        s = "2022",
        color="white"
    )

def get_difference_rolling_plot(ax, team, data, window=10, color_for="#0057B8", color_against = "#989898", reverse=False):

    df_rolling = get_rolling_data(team=team, window=window, data = data)

    ax = plot_rolling_difference(ax, data=df_rolling, difference_line="rollingDiffRelative", color_for = color_for, color_against = color_against, reverse=reverse)

    diff_number = df_rolling['rollingDiffRelative'].mean()
    text_colour_for = "white"
    # text_colour_against = "white"
    if color_for == "white":
        text_colour_for = "black"
    # if color_against == "white":
        # text_colour_against = "black"

    ax_text(
        x=0, y=23,
        s=f'<{team}>\n<Score Difference Relative to Expected: {diff_number:.1f}>',
        highlight_textprops=[
            {'color':'white', 'weight':'bold', 'font':'DM Sans'},
            {'size':'10', 'bbox':{'edgecolor':color_for, 'facecolor':color_for, 'pad':1}, 'color':text_colour_for},
            # {'size':'10', 'bbox':{'edgecolor':color_against, 'facecolor':color_against, 'pad':1}, 'color':text_colour_against},
        ],
        font="Karla",
        ha="left",
        size=14,
    )

    return ax

def get_for_difference_rolling_plot(ax, team, data, window=10, color_for="#0057B8", color_against = "#989898"):

    df_rolling = get_rolling_data(team=team, window=window, data = data)

    ax = plot_rolling_difference(ax, data=df_rolling, difference_line="rollingForRelative", color_for = color_for, color_against = color_against)

    diff_number = df_rolling['rollingForRelative'].mean()
    text_colour_for = "white"
    # text_colour_against = "white"
    if color_for == "white":
        text_colour_for = "black"
    # if color_against == "white":
        # text_colour_against = "black"

    ax_text(
        x=0, y=23,
        s=f'<{team}>\n<Score For Relative to Expected: {diff_number:.1f}>',
        highlight_textprops=[
            {'color':'white', 'weight':'bold', 'font':'DM Sans'},
            {'size':'10', 'bbox':{'edgecolor':color_for, 'facecolor':color_for, 'pad':1}, 'color':text_colour_for},
            # {'size':'10', 'bbox':{'edgecolor':color_against, 'facecolor':color_against, 'pad':1}, 'color':text_colour_against},
        ],
        font="Karla",
        ha="left",
        size=14,
    )

    return ax

def get_against_difference_rolling_plot(ax, team, data, window=10, color_for="#0057B8", color_against = "#989898"):

    df_rolling = get_rolling_data(team=team, window=window, data = data)

    ax = plot_rolling_difference(ax, data=df_rolling, difference_line="rollingAgainstRelative", color_for = color_for, color_against = color_against, reverse=True)

    diff_number = df_rolling['rollingAgainstRelative'].mean()
    text_colour_for = "white"
    # text_colour_against = "white"
    if color_for == "white":
        text_colour_for = "black"
    # if color_against == "white":
        # text_colour_against = "black"

    ax_text(
        x=0, y=23,
        s=f'<{team}>\n<Score Against Relative to Expected: {diff_number:.1f}>',
        highlight_textprops=[
            {'color':'white', 'weight':'bold', 'font':'DM Sans'},
            {'size':'10', 'bbox':{'edgecolor':color_for, 'facecolor':color_for, 'pad':1}, 'color':text_colour_for},
            # {'size':'10', 'bbox':{'edgecolor':color_against, 'facecolor':color_against, 'pad':1}, 'color':text_colour_against},
        ],
        font="Karla",
        ha="left",
        size=14,
    )

    return ax


from django.db import models
from django.contrib.auth.models import AbstractUser

class User(AbstractUser):
    GUEST = 0
    OPERATOR = 1
    LAWYER = 2
    ADMIN = 99
    ROLE_CHOICES = (
        (GUEST, "guest"),
        (LAWYER, "lawyer"),
        (OPERATOR, "operator"),
        (ADMIN, "admin"),
    )

    phone = models.CharField(help_text="手机号", max_length=16, blank=True, null=True)
    role = models.IntegerField(help_text="角色", choices=ROLE_CHOICES, default=GUEST)
    created_at = models.DateTimeField(help_text="创建时间", auto_now_add=True)
    updated_at = models.DateTimeField(help_text="更新时间", auto_now=True)

    @property
    def name(self):
        return self.last_name + self.first_name


import tornado.web
import mallory

class HeartbeatHandler(tornado.web.RequestHandler):
    def initialize(self, circuit_breaker):
        self.circuit_breaker = circuit_breaker

    @tornado.web.asynchronous
    @tornado.gen.engine
    def get(self):
        if self.circuit_breaker.is_tripped():
            self.set_status(503)
            status_message = "Circuit Breaker Tripped"
        else:
            self.set_status(200)
            status_message = "OK"

        self.write("Mallory " + mallory.Version + "\n")
        self.write(status_message)
        self.finish()


import json, pytz
from dotenv import load_dotenv
from datetime import datetime
from dagster import sensor, RunRequest, SkipReason, get_dagster_logger
from models.cancel_queue import CancelQueue
from ops.helpers.db_config import db_conn

conn = db_conn()
load_dotenv()

my_logger = get_dagster_logger()

@sensor(job_name="amz_submit_cancel_orders", minimum_interval_seconds=60)
def process_submit_cancel_orders():
    pacific_zone = pytz.timezone('US/Pacific')
    orders_list = (
        conn.query(CancelQueue.order_number)
        .where(
            CancelQueue.desktopshipper_cancel == 1,
            CancelQueue.skubana_cancel == 1,
            CancelQueue.amazon_cancel == 0,
            CancelQueue.last_processed_at <= datetime.now(pacific_zone),
            CancelQueue.processing_status.in_(tuple(['PENDING', 'FAILED']))
        )
        .distinct()
        .all()
    )

    if len(orders_list) < 1:
        yield SkipReason("No New Order ")
        return
    result = []
    for order in orders_list:
        print('\n\n', 'order.order_number', order.order_number)
        item_list = (
            conn.query(
                CancelQueue.cancel_id,
                CancelQueue.order_item_id,
                CancelQueue.buyer_cancellation_reason
            )
            .where(
                CancelQueue.order_number == order.order_number)
            .all()
        )
        order_items = []
        for item in item_list:
            order_items.append({
                "cancel_id": item.cancel_id,
                "order_item_id": item.order_item_id,
                "buyer_cancellation_reason": item.buyer_cancellation_reason
            })
        result.append({
            order.order_number: order_items
        })
    record_timestamp = datetime.now().timestamp()
    run_key = str(record_timestamp) + "_orders_" + str(len(orders_list))
    yield RunRequest(
        run_key,
        run_config={
            "resources": {
                "report_details": {"config": {"id": json.dumps({"cqs": result})}}
            },
        },
        tags={"job_type": 'amz_submit_cancel_orders'},
    )


import logging

def get_logger(name, path):

    logger = logging.getLogger(name)

    if len(logger.handlers) > 0:
        return logger # Logger already exists

    logger.setLevel(logging.DEBUG)

    formatter = logging.Formatter("%(asctime)s - %(message)s")

    console = logging.StreamHandler()
    file_handler = logging.FileHandler(filename=path)

    console.setLevel(logging.DEBUG)
    file_handler.setLevel(logging.INFO)

    console.setFormatter(formatter)
    file_handler.setFormatter(formatter)

    logger.addHandler(console)
    logger.addHandler(file_handler)

    return logger

from easydict import EasyDict
import yaml

def get_args_from_yaml(yaml_path):

    with open(yaml_path) as f:
        cfgs = yaml.load(f, Loader=yaml.FullLoader)

    data_cfg = cfgs['dataset']
    model_cfg = cfgs['model']
    train_cfg = cfgs['train']

    args = EasyDict(
        {
            'key': cfgs['key'],

            'dataset': data_cfg['name'],
            'dataset_filename': data_cfg['file_name'],
            'efeat_path': data_cfg['efeat'],

            # model configs
            'model_type': model_cfg['type'],
            'hop': model_cfg['hop'],
            'in_nfeats': model_cfg.get('in_nfeats'),
            'out_nfeats': model_cfg.get('out_nfeats'),
            'in_efeats': model_cfg.get('in_efeats'),
            'out_efeats': model_cfg.get('out_efeats'),
            'num_heads': model_cfg.get('num_heads'),
            'node_features': model_cfg.get('node_features'),
            'parameters': model_cfg.get('parameters'),

            'mpnn_type': model_cfg.get('mpnn_type'),
            'edge_dropout': model_cfg['edge_dropout'],
            'review':model_cfg.get('review'),
            'rating':model_cfg.get('rating'),
            'timestamp':model_cfg.get('timestamp'),

            'latent_dims': model_cfg.get('latent_dims'), # baseline model

            #train configs
            'device':train_cfg['device'],
            'log_dir': train_cfg['log_dir'],
            'log_interval': train_cfg.get('log_interval'),
            'train_lrs': [ float(lr) for lr in train_cfg.get('learning_rates') ],
            'train_epochs': train_cfg.get('epochs'),
            'batch_size': train_cfg['batch_size'],
            'weight_decay': train_cfg.get('weight_decay', 0),
            'lr_decay_step': train_cfg.get('lr_decay_step'),
            'lr_decay_factor': train_cfg.get('lr_decay_factor'),

        }
    )

    return args

import numpy as np
import torch as th

def evaluate(model, loader, device):
    # Evaluate RMSE
    model.eval()
    mse = 0.
    for batch in loader:
        with th.no_grad():
            preds = model(batch[0].to(device))
        labels = batch[1].to(device)
        mse += ((preds - labels) ** 2).sum().item()
    mse /= len(loader.dataset)
    return np.sqrt(mse)


from selenium import webdriver
from time import sleep
import csv
from selenium.webdriver.chrome.options import Options
import threading
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from MyClasses import LinkPDF

browsers = []
threads = []
lock = threading.Lock()
chrome_options = Options()
chrome_options.add_argument("--headless")
sleep(2)
links =[]
for i in range(1, 2):
    links.append(LinkPDF(f"https://www.centralbankbahamas.com/bank-supervision/warning-notices"))
def thread_task(url) :
    browser = webdriver.Chrome()
    browser.get(url)
    sleep(4)
    for i in range(1,1000) :
        try :
            linkElement = WebDriverWait(browser, 10).until(
                EC.presence_of_element_located((By.XPATH,f'/html/body/div[5]/section[2]/div/div/div[2]/div[2]/div/div[2]/div/div/div[2]/table/tbody/tr[{i}]/td[1]/div/span/a')))
        except :
            print("cant find element")
            break
        link = linkElement.get_attribute("href")
        print(link)
    file_path = f'links_{i}.csv'
    #with lock:
        #with open(file_path, 'a', encoding="utf-8") as txt:
            #txt.write(f'{link}\n')
    browser.quit()

for link in links:
    thread = threading.Thread(target=thread_task, args=(link.URL,))
    thread.start()
    threads.append(thread)

# Chờ cho tất cả các luồng hoàn thành
for thread in threads:
    thread.join()


from pos.kernal.models import Product
from pos.kernal.models import InStockBatch, InStockRecord
from pos.kernal.models import OutStockRecord
from pos.kernal.models import Product
from pos.kernal.models import Product

# import the logging library
import logging

# Get an instance of a logger
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
"""
Below function for ajax use
"""
#def ajaxProductDetailView(request):
   # if request.method == 'GET':

class StockKeeper:
    def __init__(self, product):
        if not product:
            logger.error("StockKeeper build fail!!")
            return
        logger.info("StockKeeper '%s' build !!", product.name)
        self.product = product

class ProductBuilder:
    def build(self, dict, inStockBatch):
        inventoryDict = __buildProductsDict(dict)
        __buildProducts(inventoryDict, inStockBatch)
        logger.debug("product build")
    def __buildProductsDict(self, dict):
        logger.debug("instock items: %s", sales_item)
        for key,  value in sales_item:
            if key == "do_no":
                continue
            if key == "inv_no":
                continue
            if key == "do_date":
                continue
            if key.find("_") == -1:
                continue
            pk = key.split("_")[0]
            attr = key.split("_")[1]
            if pk not in inventoryDict :
                inventoryDict [pk] ={}
            inventoryDict [pk] [attr]= value
        logger.debug("inventory dict: %s", inventoryDict)
        return inventoryDict

    def __buildProducts(self, inventoryDict, inStockBatch):
        # build OutStockRecord to save data
        for pk in inventoryDict :
            inStockRecord = InStockRecord()
            inStockRecord.inStockBatch = inStockBatch
            inStockRecord.barcode = pk
            product = Product.objects.get(pk=pk)
            inStockRecord.product = product
            inStockRecord.cost = inventoryDict [pk]['cost'][0]
            inStockRecord.quantity = inventoryDict [pk]['quantity'] [0]
            inStockRecord.save()
            for value in inventoryDict[pk] :
                if 'serial-' in value:
                    serialNO = request.GET.get(pk+"_"+value, '')
                    if serialNO == '':
                        continue
                    serial = SerialNo()
                    serial.inStockRecord = inStockRecord
                    serial.serial_no = serialNO
                    serial.save()