from pathlib import Path

from . import data
from . import plot
from . import excel
from . import config
from . import latex
import subprocess

class Document:
    def __init__(self, prefix, title, subtitle, author, date):
        self.prefix = prefix
        self.title = title
        self.subtitle = subtitle
        self.author = author
        self.date = date
        self.captions = []
        self.tmp_dir = Path('/tmp')
        self.output_dir = Path('.')
        self.tables = {}

    def make_table(self, col_format, print_lines, rows):
        ident = len(self.captions)
        self.tables[ident] = (col_format, print_lines, rows)
        caption = 'To be specified...'
        self.captions.append(caption)

    def make_pie(self, rows, attr, legend = True):
        filename = 'fig-{:06d}.pdf'.format(len(self.captions))
        plot_data, total = data.pie_data(rows, attr)
        plot_data['legend'] = legend
        caption = '{} elements, gathered by {}'.format(total, attr)
        self.captions.append(caption)
        plot.pie(self.tmp_dir / filename, plot_data)
        return total

    def make_histo(self, rows, category_attr, legend = True):
        filename = 'fig-{:06d}.pdf'.format(len(self.captions))
        plot_data, total = data.histo_data(rows, category_attr)
        plot_data['legend'] = legend
        caption = '{} elements, categorized by {}.'.format(total, category_attr)
        self.captions.append(caption)
        plot.histo(self.tmp_dir / filename, plot_data)
        return total

    def make_bars(self, rows, category_attr, sort_attr, legend = True, display_values = False):
        filename = 'fig-{:06d}.pdf'.format(len(self.captions))
        plot_data, total = data.bar_data(rows, category_attr, sort_attr)
        plot_data['legend'] = legend
        caption = '{} elements, categorized by {}, colored by {}.'.format(total, category_attr, sort_attr)
        self.captions.append(caption)
        plot.bar(self.tmp_dir / filename, plot_data, display_values)
        return total

    def caption(self, caption):
        self.captions[-1] = caption

    def generate(self):
        texname = self.prefix + '.tex'
        pdfname = self.prefix + '.pdf'
        path = self.tmp_dir / texname
        f = path.open('w')
        latex.begin_document(f)
        latex.title(f, self.title, self.subtitle, self.author, self.date)
        for idx, caption in enumerate(self.captions):
            if idx in self.tables:
                col_format, print_lines, rows = self.tables[idx]
                latex.table(f, col_format, print_lines, rows, caption, idx)
            else:
                latex.fig(f, self.tmp_dir / 'fig-{:06d}.pdf'.format(idx), caption, idx)
        latex.end_document(f)
        f.close()
        print('File "{}" generated.'.format(str(path.absolute())))
        cmd = 'pdflatex {}'.format(texname)
        print(cmd)
        subprocess.run(cmd.split(), cwd = self.tmp_dir)
        cmd = 'cp {} {}'.format(pdfname, self.output_dir.absolute())
        subprocess.run(cmd.split(), cwd = self.tmp_dir)
        print()
        print()
        print()
        print('File "{}" generated.'.format((self.output_dir / pdfname).absolute()))


import collections
import glob
import logging
import os
from typing import List

import torch
from torch import nn
from torch.optim.lr_scheduler import LambdaLR
from torch.serialization import default_restore_location

logger = logging.getLogger()

CheckpointState = collections.namedtuple(
    "CheckpointState",
    [
        "model_dict",
        "optimizer_dict",
        "scheduler_dict",
        "offset",
        "epoch",
        "encoder_params",
    ],
)

def setup_for_distributed_mode(
    model: nn.Module,
    optimizer: torch.optim.Optimizer,
    device: object,
    n_gpu: int = 1,
    local_rank: int = -1,
    fp16: bool = False,
    fp16_opt_level: str = "O1",
) -> (nn.Module, torch.optim.Optimizer):
    model.to(device)
    if fp16:
        try:
            import apex
            from apex import amp

            apex.amp.register_half_function(torch, "einsum")
        except ImportError:
            raise ImportError(
                "Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
            )

        model, optimizer = amp.initialize(model, optimizer, opt_level=fp16_opt_level)

    if n_gpu > 1:
        model = torch.nn.DataParallel(model)

    if local_rank != -1:
        model = torch.nn.parallel.DistributedDataParallel(
            model,
            device_ids=[device if device else local_rank],
            output_device=local_rank,
            find_unused_parameters=True,
        )
    return model, optimizer

def move_to_cuda(sample):
    if len(sample) == 0:
        return {}

    def _move_to_cuda(maybe_tensor):
        if torch.is_tensor(maybe_tensor):
            return maybe_tensor.cuda()
        elif isinstance(maybe_tensor, dict):
            return {key: _move_to_cuda(value) for key, value in maybe_tensor.items()}
        elif isinstance(maybe_tensor, list):
            return [_move_to_cuda(x) for x in maybe_tensor]
        elif isinstance(maybe_tensor, tuple):
            return [_move_to_cuda(x) for x in maybe_tensor]
        else:
            return maybe_tensor

    return _move_to_cuda(sample)

def move_to_device(sample, device):
    if len(sample) == 0:
        return {}

    def _move_to_device(maybe_tensor, device):
        if torch.is_tensor(maybe_tensor):
            return maybe_tensor.to(device)
        elif isinstance(maybe_tensor, dict):
            return {
                key: _move_to_device(value, device)
                for key, value in maybe_tensor.items()
            }
        elif isinstance(maybe_tensor, list):
            return [_move_to_device(x, device) for x in maybe_tensor]
        elif isinstance(maybe_tensor, tuple):
            return [_move_to_device(x, device) for x in maybe_tensor]
        else:
            return maybe_tensor

    return _move_to_device(sample, device)

def get_schedule_linear(
    optimizer,
    warmup_steps,
    total_training_steps,
    steps_shift=0,
    last_epoch=-1,
):

    """Create a schedule with a learning rate that decreases linearly after
    linearly increasing during a warmup period.
    """

    def lr_lambda(current_step):
        current_step += steps_shift
        if current_step < warmup_steps:
            return float(current_step) / float(max(1, warmup_steps))
        return max(
            1e-7,
            float(total_training_steps - current_step)
            / float(max(1, total_training_steps - warmup_steps)),
        )

    return LambdaLR(optimizer, lr_lambda, last_epoch)

def init_weights(modules: List):
    for module in modules:
        if isinstance(module, (nn.Linear, nn.Embedding)):
            module.weight.data.normal_(mean=0.0, std=0.02)
        elif isinstance(module, nn.LayerNorm):
            module.bias.data.zero_()
            module.weight.data.fill_(1.0)
        if isinstance(module, nn.Linear) and module.bias is not None:
            module.bias.data.zero_()

def get_model_obj(model: nn.Module):
    return model.module if hasattr(model, "module") else model

def get_model_file(args, file_prefix) -> str:
    if args.model_file and os.path.exists(args.model_file):
        return args.model_file

    out_cp_files = (
        glob.glob(os.path.join(args.output_dir, file_prefix + "*"))
        if args.output_dir
        else []
    )
    logger.info("Checkpoint files %s", out_cp_files)
    model_file = None

    if len(out_cp_files) > 0:
        model_file = max(out_cp_files, key=os.path.getctime)
    return model_file

def load_states_from_checkpoint(model_file: str) -> CheckpointState:
    logger.info("Reading saved model from %s", model_file)
    state_dict = torch.load(
        model_file, map_location=lambda s, l: default_restore_location(s, "cpu")
    )
    logger.info("model_state_dict keys %s", state_dict.keys())
    return CheckpointState(**state_dict)


from databases import Database
from sqlalchemy import select, func
from typing import Dict

from db import models
from schemas import companies_members as schema_cm
from utils.exceptions import MyExceptions

class GenericDatabase:

    def __init__(self, db: Database):
        self.db = db
        self.company_members_table = models.company_members
        self.company_table = models.companies
        self.users_table = models.users
        self.quiz_table = models.quiz
        self.question_table = models.question
        self.avarage_mark_table=models.avarage_mark
        self.uiz_result_table = models.companies
        self.quiz_result_table=models.quiz_result
        self.exception = MyExceptions

    async def insert_values(self, table, values: Dict):
        return await self.db.execute(table.insert().values(values))

    async def company_members_join_company(self):
        query_joins = self.company_members_table.join(self.company_table)
        query = select(
            self.company_table.c.id.label('company_id'),
            self.company_table.c.name.label('company_name'),
            self.company_members_table.c.is_company_admin,
            self.company_members_table.c.active_member.label('active_from'),
            ).select_from(query_joins)

        return query


# Title: 감소하는 수
# Link: https://www.acmicpc.net/problem/1038
import itertools
import sys

sys.setrecursionlimit(10 ** 6)

def read_list_int():
    return list(map(int, sys.stdin.readline().strip().split(' ')))

def read_single_int():
    return int(sys.stdin.readline().strip())

def add_to_one(d_number):
    if d_number == 9876543210:
        return -1
    r_str = []
    d_str = list(reversed(str(d_number)))
    carry = 0
    num = '1' + '0'*(len(d_str)-1)
    for i, d in enumerate(d_str):
        if i+1 == len(d_str):
            r = int(d) + int(num[i]) + carry
            if r >= 10:
                r_str.append(str(len(r_str)))
                r_str.append(str(len(r_str)))
            else:
                r_str.append(str(r))
        else:
            r = int(d) + int(num[i]) + carry
            if r < int(d_str[i+1]):
                r_str.append(str(r))
                carry = 0
            else:
                carry = 1
                if len(r_str) == 0:
                    r_str.append('0')
                else:
                    r_str.append(str(int(r_str[-1])+1))
    return int(''.join(reversed(r_str)))

def get_nth_number(N):
    d_number = 0
    for _ in range(N):
        d_number = add_to_one(d_number)
        if d_number == -1:
            return -1

    return d_number

def get_nth_decreasing_numbers(N):
    decreasing_numbers = []
    for n in range(1, 11):
        decreasing_numbers += sorted(map(lambda l: int(''.join(l)), itertools.combinations('9876543210', n)))
        if len(decreasing_numbers) > N:
            return decreasing_numbers[N]
    return -1

if __name__ == '__main__':
    N = read_single_int()
    # print(get_nth_number(N))
    print(get_nth_decreasing_numbers(N))


#!/usr/bin/env python3
"""Contains function sum_list"""
from typing import Union, List

def sum_mixed_list(mxd_list: List[Union[int, float]]) -> float:
    """Calculates the sum of floats in the given list"""
    sum: float = 0.00
    for num in mxd_list:
        sum += num

    return sum


#bfs1 단지번호 붙이기
from collections import deque
n = int(input())
maps = []
for _ in range(n):
    tmp = [int(i) for i in input()]
    maps.append(tmp)

def bfs(coord,maps,visited):
    dirs = [(1,0),(-1,0),(0,1),(0,-1)]
    queue = deque()
    queue.append((coord))
    count = 0
    while queue:
        y,x = queue.popleft()
        #방문하지 않은 지점을 경우
        if visited[y][x] == 0:
            visited[y][x] = 1
            count += 1
            #현재 위치에서 상하좌우를 탐색하며 같은 단지의 집인지 확인한다.
            for _dir in dirs:
                new_y,new_x = y + _dir[0], x + _dir[1]
                if new_x >= 0 and new_x < n and \
                new_y >= 0 and new_y < n and visited[new_y][new_x] == 0 and maps[new_y][new_x] ==1:
                    new_coord = (new_y, new_x)
                    queue.append(new_coord)
    return count

visited = [[0 for _ in range(n)] for _ in range(n)]
count = 0
answer = []
for y in range(n):
    for x in range(n):
        if visited[y][x] == 0 and maps[y][x] == 1:
            #단지가 총 몇개있는지
            count +=1
            coord = (y,x)
            answer.append(bfs(coord,maps,visited))
answer.sort()
print(count)
print("\n".join([str(i) for i in answer]))

김수빈/BFS/단지번호붙이기.py


import requests
from get_access_token_by_refresh_token import get_access_token

def get_modules():
    url = 'https://www.zohoapis.in/crm/v2/settings/modules'
    access_token = get_access_token()

    headers = {
        'Authorization': 'Zoho-oauthtoken {}'.format(access_token)
    }

    response = requests.get(url=url, headers=headers)

    if response is not None:
        print("HTTP Status Code : " + str(response.status_code))

        print(response.json())

get_modules()


import flet as ft
import views as views_handle

def main(page: ft.Page):

    #defining the fonts
    page.fonts = {
        "SF Pro": "https://raw.githubusercontent.com/google/fonts/master/ofl/sfprodisplay/SFProDisplay-Bold.ttf",
    }

    #defining the window size
    page.window_min_width = 425
    page.window_width = 425
    page.window_min_height = 820
    page.window_height = 820

    page.title = "Project 334 Appication"
    page.horizontal_alignment = ft.CrossAxisAlignment.CENTER
    #page.scroll = ft.ScrollMode.HIDDEN
    page.padding = 0
    page.bgcolor = "#ddf7f1"

    page.theme = ft.Theme(font_family="SF Pro")
    page.theme_mode = ft.ThemeMode.LIGHT

    def route_change(route):
        print(page.route)
        # CLEAR ALL PAGE
        page.views.clear()
        page.views.append(
            views_handle.views(page)[page.route],
        )
        page.update()

    page.on_route_change = route_change
    page.go('/')

ft.app(target = main)


# import eventlet
# eventlet.monkey_patch()

from flask import Flask, request, jsonify, render_template
from flask_socketio import SocketIO, emit
from flask_cors import CORS
import time
import pandas as pd
import numpy as np
from math import sqrt
import heapq
from copy import deepcopy
import json

app = Flask(__name__, static_url_path='', static_folder='front-end/build')
app.config['SECRET_KEY'] = 'secret!'
CORS(app, resources={r"/*": {"origins": "*"}})
socketio = SocketIO(app, cors_allowed_origins="*")

@app.route("/http-call")
def http_call():
    """return JSON with string data as the value"""
    data = {'data': 'This text was fetched using an HTTP call to server on render'}
    return jsonify(data)

@app.route("/")
def index():
    return app.send_static_file('index.html')

@socketio.on("connect")
def connected():
    """event listener when client connects to the server"""
    print(request.sid)
    print("client has connected")
    emit("connect", {"data": f"id: {request.sid} is connected"})

# @socketio.on('dimension')
# def get_dimensions(data):
#     print("data for dimensions from the front end: ",str(data))

@socketio.on('data')
def handle_message(data):
    """event listener when client types a message"""
    print("data from the front end: ", (data))
    wjdata = json.loads(data)
    print("data from the front end wjdata: ", (wjdata))
    # emit("data",{'data':data,'id':request.sid},broadcast=True)
    # recursively call and emit data
    # for i in range(20):
    #     socketio.sleep(0.5)
    #     emit("data",{'data':i,'id':request.sid},broadcast=False)
    #     print(i)

    def create_grid(p, dim):
        '''

    :param p: probability with which a node is blocked
    :param dim: dimension of the matrix
    :return: a 2d list denoting grid where 1 = traversable node, 0 = non traversable

    This function generates a random grid by taking into account probability 'p' and dimension 'dim'
    '''

    # initialise a grid with all 0. 0 = cell blocked, 1 =  cell unblocked
        grid = [[0 for i in range(dim)] for j in range(dim)]

        # Loop over inputted dimension
        for i in range(dim):
            for j in range(dim):
                actual_prob = np.random.random_sample()  # generating a random number
                # if the generated random number > p, assign it 1 (meaning it is
                if actual_prob > p:
                    grid[i][j] = 1  # traversable.
                else:
                    grid[i][j] = 0

        grid[0][0] = 1  # start node and end node is always traversable.
        grid[dim - 1][dim - 1] = 1
        return grid

    class Node:
        '''
        A node class that stores 5 things for a node - position, parent, g(n), h(n), f(n)
        '''

        def __init__(self, parent=None, position=None):
            '''
            This function initalises a node by setting parent , position and heuristic values as 0
            :param parent: parent of the current code
            :param position: position of the current code
            '''
            self.parent = parent
            self.position = position
            self.g = 0
            self.f = 0
            self.h = 0

        def __eq__(self, node):
            '''
            This function is overload for == operator in python. It is used to compare if two nodes are equal.

            :param node: node to compare with
            :return: 1 if two self and node is equal, otherwise 0

            '''
            if (self.position[0] == node.position[0] and self.position[1] == node.position[1]):
                return True
            else:
                return False

        def __lt__(self, other):
            '''
            This function is overload for < operator in python. It is used to compare if one node is less than other.

            :param other: node to compare with
            :return: 1 if self's f value is less than other's f value, otherwise 0

            '''
            return self.f < other.f

    def generate_children(grid, knowledge_grid, fringe, visited_list, current, all_moves, end_node, is_gridknown):
        '''
        This function uses a grid (be it grid or knowledge) and generates all valid children of the current node.

        :param grid: the original actual grid
        :param knowledge_grid: the knowledge grid
        :param fringe: list of nodes in the priority queue
        :param visited_list: a dictionary of nodes already visited
        :param current: current node in the queue
        :param all_moves: array of all valid moves
        :param end_node: end position/node in the grid
        :param is_gridknown: parameter to switch between grid and knowledge grid
        :return: array of relevant children
        '''
        current_x, current_y = current.position
        relevant_children = []
        dim = len(grid)

        for a_move in all_moves:  # looping over all valid moves
            child_x = current_x + a_move[0]
            child_y = current_y + a_move[1]
            # condition to check if node is in within
            if child_x > dim-1 or child_x < 0 or child_y > dim-1 or child_y < 0:
                # boundaries of the grid
                continue
            # initalising children node with current
            children_node = Node(current, (child_x, child_y))
            # as parent and child_x, child_y as position

            if (is_gridknown == "No"):  # isgridknown checks whether to we have grid
                # loaded in the memory, if not we use knowledge
                # grid
                grid_for_pathcalculation = knowledge_grid
            else:
                grid_for_pathcalculation = grid
            # condition to check is current node
            if (grid_for_pathcalculation[child_x][child_y] != 0) and (visited_list.get(children_node.position) != "Added"):
                # is not blocked and current node is
                # not in visited list

                # assigining current g = g(parent) + 1
                children_node.g = current.g + 1
                children_node.h = abs(children_node.position[0] - end_node.position[0]) + abs(  # using manhattan distance as our heuristic
                    children_node.position[1] - end_node.position[1])

                children_node.f = children_node.g + \
                    children_node.h  # f(n) = g(n) + f(n)
                relevant_children.append(children_node)
        return relevant_children

    def search(grid, fringe, knowledge_grid, start_position, end_position, is_gridknown):
        '''

        :param grid: the original actual grid
        :param fringe: list of all processed nodes
        :param knowledge_grid: the knowledge grid
        :param start_position: start position in grid
        :param end_position: end position in grid
        :param is_gridknown: parameter to switch between grid and knowledge grid
        :return: the path from start node to end node
        '''
        startNode = Node(None, start_position)
        endNode = Node(None, end_position)

        fringe = []
        visited_nodes = {}
        already_fringed = {}  # a hashmap to keep track of fringed nodes and its lowest cost
        already_fringed[startNode.position] = startNode.f
        # pushing start node in fringe
        heapq.heappush(fringe, (startNode.f, startNode))
        all_moves = [[1, 0],  # defined all moves -
                     [0, 1],  # [1,0] - move right
                     [-1, 0],  # [0,1] - move down
                     [0, -1]]  # [0,-1] - move up
        # [-1,0] - move left

        path = []
        while fringe:  # while fringe is not empty
            current = heapq.heappop(fringe)  # popping node from fringe
            current = current[1]
            # assigning current node to visited
            visited_nodes[current.position] = "Added"

            if current.position == endNode.position:

                i = current
                while (i is not None):  # traversing path if current=goal to get the path from start to goal
                    path.append(i.position)
                    i = i.parent

                return "Solvable", path
            children = generate_children(  # otherwise generate children
                grid, knowledge_grid, fringe, visited_nodes, current, all_moves, endNode, is_gridknown)
            if children:
                for node in children:
                    if node.position in already_fringed:  # checking if the children is already fringed,
                        # if yes update and push the moinimum cost one
                        if already_fringed[node.position] > node.f:
                            # otherwise ignore the child
                            already_fringed[node.position] = node.f
                            heapq.heappush(fringe, (node.f, node))
                    else:
                        # if the child is not already fringed, push it
                        heapq.heappush(fringe, (node.f, node))
                        # to priority queue and assign in the hashmap
                        already_fringed[node.position] = node.f

        return "Unsolvable", path

    fringe = []
    dim = int(wjdata["dim"])
    is_gridknown = "No"
    density = float(wjdata["density"])

    # create a grid with entered density and dim values.
    grid = create_grid(density, dim)

    # assuming unblocked for all cells
    # intialise knowledge grid to all 1's
    knowledge_grid = [[1 for _ in range(dim)] for _ in range(dim)]
    im = None
    pltGrid = deepcopy(grid)
    pltGrid[0][0] = 3
    prevPos = [0, 0]
    start = (0, 0)
    end = (dim-1, dim-1)
    all_moves = [[1, 0],
                 [0, 1],
                 [-1, 0],
                 [0, -1]]
    for a_move in all_moves:
        child_x = start[0] + a_move[0]
        child_y = start[1] + a_move[1]
        if (child_x > dim-1 or child_x < 0 or child_y > dim-1 or child_y < 0):
            continue
        else:
            if (grid[child_x][child_y] == 0):
                # update the knowledge grid with field of view
                knowledge_grid[child_x][child_y] = 0
    ll, path = search(grid, fringe, knowledge_grid, start, end, is_gridknown)
    final_path = []
    if (ll != "Unsolvable" and is_gridknown == "No"):
        while (len(path) > 1 and ll != "Unsolvable"):
            count = 0
            flag = 0

            # traverse the path obtained from search function to see if blocked cell exists or not.
            # If blocked cell exists, run search function again to calculate the path
            #  Continue in this while loop -1) either path returned is 0 that means nothing left in fringe and no path to reach goal 2) or path exists to reach goal

            for i in path[::-1]:
                count += 1
                for a_move in all_moves:
                    child_x = i[0] + a_move[0]
                    child_y = i[1] + a_move[1]
                    if (child_x > dim-1 or child_x < 0 or child_y > dim-1 or child_y < 0):
                        continue
                    else:
                        if (grid[child_x][child_y] == 0):
                            knowledge_grid[child_x][child_y] = 0
                final_path.append((i[0], i[1]))

                if (grid[i[0]][i[1]] == 0):
                    pltGrid[i[0]][i[1]] = 4  # blocked in grid
                    final_path.pop()
                    knowledge_grid[i[0]][i[1]] = 0  # updating knowledge_grid
                    new_start_position = path[path.index(
                        i)+1][0], path[path.index(i)+1][1]
                    ll, path = search(grid, fringe, knowledge_grid,
                                      new_start_position, end, is_gridknown)
                    finalresult = ll
                    break

                pltGrid[prevPos[0]][prevPos[1]] = 2
                pltGrid[i[0]][i[1]] = 3
                prevPos = [i[0], i[1]]
                socketio.sleep(0.5)
                # print("pltGrid", pltGrid)
                emit("data", {'data': pltGrid,
                     'id': request.sid, 'status': "Solving..."}, broadcast=False)

                if (count == len(path)):
                    emit("data", {'data': pltGrid,
                                  'id': request.sid, 'status': "Reached the destination!"}, broadcast=False)
                    print("Solved")
                    flag = 1
                    break
            if (flag == 1):

                return final_path, knowledge_grid
                break
        if (ll == "Unsolvable"):

            emit("data", {'data': pltGrid, 'status': "Unsolvable"
                          }, broadcast=False)
            print("Unsolvable")
            # return [], knowledge_grid
        if (flag != 1):
            print("finalresult", finalresult)
            # return [], knowledge_grid

    elif (is_gridknown == "Yes"):
        print(ll)
        print("path", path)

    else:
        emit("data", {'data': pltGrid, 'status': "Unsolvable"
                      }, broadcast=False)
        print("Unsolvable")
        for (i, j) in final_path:
            grid[i][j] = 2

        # return [], knowledge_grid

@socketio.on("disconnect")
def disconnected():
    """event listener when client disconnects to the server"""
    print("user disconnected")
    emit("disconnect", f"user {request.sid} disconnected", broadcast=True)

if __name__ == '__main__':
    socketio.run(app, debug=True, port=5001)


__all__ = ["load", "save", "load_single_particle_model", "load_multi_particle_model"]

import os
import numpy as np
import tensorflow as tf
import deeptrack as dt
from tqdm import tqdm
import glob
import pandas as pd

def load(filename: str):
    *_, ext = filename.split(os.path.extsep)

    if os.path.isdir(filename):
        return load_dir(filename)

    if ext == "py":
        return load_python(filename)

    return load_video(filename)

def load_python(filename):
    pass

def load_video(filename):
    import cv2

    cap = cv2.VideoCapture(filename)

    n_frames, width, height = (
        int(cap.get(cv2.CAP_PROP_FRAME_COUNT)),
        int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)),
        int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)),
    )

    frames = np.empty((n_frames, width, height, 3))

    print("Reading video...")
    for t in tqdm(range(n_frames)):
        _, frames[t] = cap.read()

    return frames

def load_dir(filename):
    npys = glob.glob(os.path.join(filename, "*.npy"))
    csvs = glob.glob(os.path.join(filename, "*.csv"))

    npy_files = [np.load(f) for f in npys]
    csv_files = [pd.read_csv(f) for f in csvs]

    npy_files = np.stack(npy_files, axis=0)

    csv_files = pd.concat(csv_files)
    # csv_files = np.stack(csv_files, axis=0)

    npy_files = np.reshape(npy_files, (-1, *npy_files.shape[-3:]))

    return npy_files, csv_files

def load_image(filename):
    pass

def save(model, args):

    _out_path = f"checkpoints/{args.prefix}{os.path.split(args.filename)[-1]}"
    out_path = _out_path
    idx = 0
    while os.path.exists(out_path):
        idx += 1
        out_path = f"{_out_path}_{idx}"
    out_path = os.path.normcase(out_path)
    os.makedirs(out_path, exist_ok=True)

    model.save(out_path)

def load_single_particle_model(path):
    return tf.keras.models.load_model(
        path,
        custom_objects={
            "AutoTrackerModel": dt.models.autotrack.AutoTracker.AutoTrackerModel
        },
    )

def load_multi_particle_model(path):
    return tf.keras.models.load_model(
        path,
        custom_objects={
            "AutoTrackerModel": dt.models.autotrack.AutoTracker.AutoTrackerModel
        },
    )


import networkx as nx
import numpy as np
import torch
from datanetAPI import DatanetAPI

POLICIES = np.array(['WFQ', 'SP', 'DRR'])

def sample_to_dependency_graph(sample):
    G = nx.DiGraph(sample.get_topology_object())
    R = sample.get_routing_matrix()
    T = sample.get_traffic_matrix()
    P = sample.get_performance_matrix()

    D_G = nx.DiGraph()
    for src in range(G.number_of_nodes()):
        for dst in range(G.number_of_nodes()):
            if src != dst:
                D_G.add_node('p_{}_{}'.format(src, dst),
                             traffic=T[src, dst]['Flows'][0]['AvgBw'],
                             packets=T[src, dst]['Flows'][0]['PktsGen'],
                             tos=int(T[src, dst]['Flows'][0]['ToS']),
                             source=src,
                             destination=dst,
                             drops=float(P[src, dst]['AggInfo']['PktsDrop']) / float(T[src, dst]['Flows'][0]['PktsGen']),
                             delay=float(P[src, dst]['AggInfo']['AvgDelay']),
                             jitter=float(P[src, dst]['AggInfo']['Jitter']))

                if G.has_edge(src, dst):
                    D_G.add_node('l_{}_{}'.format(src, dst),
                                 capacity=G.edges[src, dst]['bandwidth'],
                                 policy=np.where(G.nodes[src]['schedulingPolicy'] == POLICIES)[0][0])
                for h_1, h_2 in [R[src, dst][i:i + 2] for i in range(0, len(R[src, dst]) - 1)]:
                    D_G.add_edge('p_{}_{}'.format(src, dst), 'l_{}_{}'.format(h_1, h_2))
                    if 'schedulingWeights' in G.nodes[h_1]:
                        q_w = str(G.nodes[h_1]['schedulingWeights']).split(',')
                    else:
                        q_w = ['-']
                    if 'tosToQoSqueue' in G.nodes[h_1]:
                        map = [m.split(',') for m in str(G.nodes[h_1]['tosToQoSqueue']).split(';')]
                    else:
                        map = [['0'], ['1'], ['2']]
                    q_n = 0
                    for q in range(G.nodes[h_1]['levelsQoS']):
                        D_G.add_node('q_{}_{}_{}'.format(h_1, h_2, q),
                                     priority=q_n,
                                     weight=float(q_w[q]) if q_w[0] != '-' else 0)
                        D_G.add_edge('l_{}_{}'.format(h_1, h_2), 'q_{}_{}_{}'.format(h_1, h_2, q))
                        if str(int(T[src, dst]['Flows'][0]['ToS'])) in map[q]:
                            D_G.add_edge('p_{}_{}'.format(src, dst), 'q_{}_{}_{}'.format(h_1, h_2, q))
                            D_G.add_edge('q_{}_{}_{}'.format(h_1, h_2, q), 'p_{}_{}'.format(src, dst))
                        q_n += 1

    D_G.remove_nodes_from([node for node, out_degree in D_G.out_degree() if out_degree == 0])

    n_q = 0
    n_p = 0
    n_l = 0
    mapping = {}
    for entity in list(D_G.nodes()):
        if entity.startswith('q'):
            mapping[entity] = ('q_{}'.format(n_q))
            n_q += 1
        elif entity.startswith('p'):
            mapping[entity] = ('p_{}'.format(n_p))
            n_p += 1
        elif entity.startswith('l'):
            mapping[entity] = ('l_{}'.format(n_l))
            n_l += 1

    D_G = nx.relabel_nodes(D_G, mapping)
    return D_G, n_q, n_p, n_l

def generator(data_dir, label, shuffle=False):
    tool = DatanetAPI(data_dir, [], shuffle)
    it = iter(tool)
    for sample in it:

        D_G, n_q, n_p, n_l = sample_to_dependency_graph(sample)

        link_to_path = np.array([], dtype='int32')
        queue_to_path = np.array([], dtype='int32')
        l_p_s = np.array([], dtype='int32')
        l_q_p = np.array([], dtype='int32')
        path_ids = np.array([], dtype='int32')
        for i in range(n_p):
            l_s_l = 0
            q_s_l = 0
            for elem in D_G['p_{}'.format(i)]:
                if elem.startswith('l_'):
                    link_to_path = np.append(link_to_path, int(elem.replace('l_', '')))
                    l_s_l += 1
                elif elem.startswith('q_'):
                    queue_to_path = np.append(queue_to_path, int(elem.replace('q_', '')))
                    q_s_l += 1
            path_ids = np.append(path_ids, [i] * q_s_l)
            l_p_s = np.append(l_p_s, range(l_s_l))
            l_q_p = np.append(l_q_p, range(q_s_l))

        path_to_queue = np.array([], dtype='int32')
        sequence_queues = np.array([], dtype='int32')
        for i in range(n_q):
            seq_len = 0
            for elem in D_G['q_{}'.format(i)]:
                path_to_queue = np.append(path_to_queue, int(elem.replace('p_', '')))
                seq_len += 1
            sequence_queues = np.append(sequence_queues, [i] * seq_len)

        queue_to_link = np.array([], dtype='int32')
        sequence_links = np.array([], dtype='int32')
        l_q_l = np.array([], dtype='int32')
        for i in range(n_l):
            seq_len = 0
            for elem in D_G['l_{}'.format(i)]:
                queue_to_link = np.append(queue_to_link, int(elem.replace('q_', '')))
                seq_len += 1
            sequence_links = np.append(sequence_links, [i] * seq_len)
            l_q_l = np.append(l_q_l, range(seq_len))

        if 0 in list(nx.get_node_attributes(D_G, 'jitter').values()) or 0 in list(nx.get_node_attributes(D_G, 'delay').values()):
            continue

        yield {"traffic": list(nx.get_node_attributes(D_G, 'traffic').values()),
               "packets": list(nx.get_node_attributes(D_G, 'packets').values()),
               "capacity": list(nx.get_node_attributes(D_G, 'capacity').values()),
               "policy": list(nx.get_node_attributes(D_G, 'policy').values()),
               "priority": list(nx.get_node_attributes(D_G, 'priority').values()),
               "weight": [w / 100 for w in list(nx.get_node_attributes(D_G, 'weight').values())],
               "link_to_path": link_to_path,
               "queue_to_path": queue_to_path,
               "path_to_queue": path_to_queue,
               "queue_to_link": queue_to_link,
               "sequence_queues": sequence_queues,
               "sequence_links": sequence_links,
               "path_ids": path_ids,
               "l_p_s": l_p_s,
               "l_q_p": l_q_p,
               "l_q_l": l_q_l,
               "n_queues": n_q,
               "n_links": n_l,
               "n_paths": n_p,
               }, list(nx.get_node_attributes(D_G, label).values())

# torch dataset
class GenDataset(torch.utils.data.Dataset):
    def __init__(self, data_dir, label, shuffle, sample=4000, transform=None):
        super(GenDataset).__init__()
        self.data_dir = data_dir
        self.label = label
        self.shuffle = shuffle
        self.transform = transform
        self.len = sample
        self.data_gen = generator(data_dir, label, shuffle)
    def __getitem__(self, idx):
        tempx, tempy = next(self.data_gen)
        # for name in tempx:
        #     tempx[name] = torch.tensor(tempx[name])
        tempx['traffic'] = torch.tensor(tempx['traffic'],dtype=torch.float32) # list of float
        tempx['packets'] = torch.tensor(tempx['packets'],dtype=torch.float32)
        tempx['capacity'] = torch.tensor(tempx['capacity'],dtype=torch.float32)
        tempx['policy'] = torch.tensor(tempx['policy'],dtype=torch.int64)# index
        tempx['priority'] = torch.tensor(tempx['priority'],dtype=torch.int64)
        tempx['weight'] = torch.tensor(tempx['weight'],dtype=torch.float32)

        tempx['link_to_path'] = torch.tensor(tempx['link_to_path'],dtype=torch.int64)
        tempx['queue_to_path'] = torch.tensor(tempx['queue_to_path'],dtype=torch.int64)
        tempx['path_to_queue'] = torch.tensor(tempx['path_to_queue'],dtype=torch.int64)
        tempx['queue_to_link'] = torch.tensor(tempx['queue_to_link'],dtype=torch.int64)

        tempx['sequence_links'] = torch.tensor(tempx['sequence_links'],dtype=torch.int32)
        tempx['sequence_queues'] = torch.tensor(tempx['sequence_queues'],dtype=torch.int64)
        tempx['path_ids'] = torch.tensor(tempx['path_ids'],dtype=torch.int32)
        tempx['l_p_s'] = torch.tensor(tempx['l_p_s'],dtype=torch.int64)
        tempx['l_q_p'] = torch.tensor(tempx['l_q_p'],dtype=torch.int64)
        tempx['l_q_l'] = torch.tensor(tempx['l_q_l'],dtype=torch.int64)
        tempx['n_queues'] = torch.tensor(tempx['n_queues'],dtype=torch.int32)
        tempx['n_links'] = torch.tensor(tempx['n_links'],dtype=torch.int32)
        tempx['n_paths'] = torch.tensor(tempx['n_paths'],dtype=torch.int32)
        tempy = torch.tensor(tempy)
        if self.transform:
            tempx, tempy = self.transform(tempx, tempy)
        return tempx, tempy
    def map(self, map_func):
        x, y = next(self.data_gen)
        return map_func(x, y)
    def __len__(self):
        return self.len


#길이제한 해주는 프로그램 GPT3 최대 2048개의 토큰 처리 가능
import pandas as pd
from transformers import GPT2Tokenizer

def trim_to_combined_token_limit(csv_file, max_tokens=2048):
    tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
    df = pd.read_csv(csv_file)

    for idx, row in df.iterrows():
        # convert 'nan' or empty inputs to ""
        row['Title'] = row['Title'] if row['Title'] and str(row['Title']).lower() != 'nan' else ""
        row['Content'] = row['Content'] if row['Content'] and str(row['Content']).lower() != 'nan' else ""
        row['Comments'] = row['Comments'] if row['Comments'] and str(row['Comments']).lower() != 'nan' else ""

        tokenized_title = tokenizer.encode(row['Title'], truncation=False)
        tokenized_content = tokenizer.encode(row['Content'], truncation=False)
        tokenized_comments = tokenizer.encode(row['Comments'], truncation=False)

        # If the combined tokens exceed the limit, trim the 'content' from the end
        # and the 'comments' from the beginning until they fit
        while len(tokenized_title) + len(tokenized_content) + len(tokenized_comments) > max_tokens:
            if len(tokenized_content) > len(tokenized_comments):
                tokenized_content = tokenized_content[:-1]
            else:
                tokenized_comments = tokenized_comments[1:]

        df.loc[idx, 'Title'] = tokenizer.decode(tokenized_title)
        df.loc[idx, 'Content'] = tokenizer.decode(tokenized_content)
        df.loc[idx, 'Comments'] = tokenizer.decode(tokenized_comments)

    df.to_csv('trimmed_dataset.csv', index=False)

trim_to_combined_token_limit('everytime_hotarticle_merged.csv')


# coding: utf-8
from pycparser import c_parser, c_ast
from cflags_loader import read_compiler_dflags_lib, get_cpp_dflags
from c_preprocessing import preprocess

def parse_text(text):
    parser = c_parser.CParser()
    ast = parser.parse(text, filename="<none>")

    return ast

def get_ast(grouprep, filename, include_paths, cpp_flags_file):
    dflags = read_compiler_dflags_lib(cpp_flags_file)

    grouprep_dflags = get_cpp_dflags(grouprep, dflags)
    text = preprocess(filename, grouprep_dflags, include_paths)
    return parse_text(text)

def get_asts_for_all_files(grouprep, include_paths, source_files, cpp_flags_file):
    return dict(
        print(f"Processing {filename}")
        or (filename, get_ast(grouprep, filename, include_paths, cpp_flags_file))
        for filename in source_files
    )

def ast_rec_iterator(node, tag="", level=0):
    yield tag, node
    if hasattr(node, "children"):
        for tag, child in node.children():
            yield from ast_rec_iterator(child, tag, level + 1)

def find_function_definitions(ast):
    # Function definitions should be unique.
    decl_list = [(node.decl.name, node.decl.storage, node)
        for _, node in ast_rec_iterator(ast)
        if isinstance(node, c_ast.FuncDef)]

    names =  [ name for name,storage,node in decl_list ]
    for name in set(names):
        count = names.count(name)
        assert count == 1, f"Function {name} is declared {count} times!"
    return decl_list

def find_function_call_names(ast):

    fcalls = set(
        node.name.name
        for _, node in ast_rec_iterator(ast)
        if isinstance(node, c_ast.FuncCall)
    )

    ids =  set(
        node.name
        for _, node in ast_rec_iterator(ast)
        if isinstance(node, c_ast.ID)
    )

    return set.union(ids,fcalls)


import numpy as np
from pbcore.io import BamReader, IndexedBamReader, IndexedFastaReader, AlignmentSet
import pickle
import sys
import itertools
import pandas as pd
import gzip
import pysam
from array import array
from tqdm import tqdm, trange

# Kinetics tools can be found https://github.com/PacificBiosciences/kineticsTools
# and can be installed in to a conda environment using setup.py
#from kineticsTools.KineticWorker import KineticWorkerProcess
#from kineticsTools.ResultWriter import KineticsWriter
from kineticsTools.ipdModel import IpdModel, GbmContextModel
#from kineticsTools import ReferenceUtils, loader
from kineticsTools.sharedArray import SharedArray

bamIn = pysam.AlignmentFile(sys.argv[1], check_sq=False)
bamOut = pysam.AlignmentFile(sys.argv[2], "wb", header=bamIn.header)

def snippetFunc(refId, pre, post):
    """
    Return a function that returns a snippet of the reference sequence around a given position
    """

    refArray = refId.getNumpyWrapper()

    def f(tplPos, tplStrand):
        """Closure for returning a reference snippet. The reference is padded with N's for bases falling outside the extents of the reference"""
        # skip over the padding
        tplPos += pad

        # Forward strand
        if tplStrand == 0:
            slc = refArray[(tplPos - pre):(tplPos + 1 + post)]
            slc = np.right_shift(slc, 4)
            return "".join(c for c in seqMapNp[slc])

        # Reverse strand
        else:
            slc = refArray[(tplPos + pre):(tplPos - post - 1):-1]
            slc = np.right_shift(slc, 4)
            return "".join(c for c in seqMapComplementNp[slc])

    return f

def _makeFramepoints():
    B = 2
    t = 6
    T = 2**t

    framepoints = []
    next = 0
    for i in range(256//T):
        grain = B**i
        nextOnes = next + grain * np.arange(0, T)
        next = nextOnes[-1] + grain
        framepoints = framepoints + list(nextOnes)
    return np.array(framepoints, dtype=np.uint16)

def _makeLookup(framepoints):
    # (frame -> code) involves some kind of rounding
    # basic round-to-nearest
    frameToCode = np.empty(shape=max(framepoints)+1, dtype=int)
    for i, (fl, fu) in enumerate(zip(framepoints, framepoints[1:])):
        if (fu > fl + 1):
            m = (fl + fu)//2
            for f in range(fl, m):
                frameToCode[f] = i
                frameToCode[f] = i + 1
        else:
            frameToCode[fl] = i
    # Extra entry for last:
    frameToCode[fu] = i + 1
    return frameToCode, fu

_framepoints = _makeFramepoints()
_frameToCode, _maxFramepoint = _makeLookup(_framepoints)

def framesToCode(nframes):
    nframes = np.minimum(_maxFramepoint, nframes)
    return _frameToCode[nframes]

def codeToFrames(code):
    return _framepoints[code]

byte = np.dtype('byte')
uint8 = np.dtype('uint8')

# Map for ascii encoded bases to integers 0-3 -- will be used to define a 24-bit lookup code
# for fetching predicted IPDs from the kinetic LUT.

# We start everything at 0, so anything will map to 'A' unless it appears
# in this table
lutCodeMap = np.zeros(256, dtype=uint8)
maps = {'a': 0, 'A': 0, 'c': 1, 'C': 1, 'g': 2, 'G': 2, 't': 3, 'T': 3}
for k in maps:
    lutCodeMap[ord(k)] = maps[k]
lutReverseMap = {0: 'A', 1: 'C', 2: 'G', 3: 'T'}

seqCodeMap = np.ones(256, dtype=uint8) * 4
for k in maps:
    seqCodeMap[ord(k)] = maps[k]
seqMap = {0: 'A', 1: 'C', 2: 'G', 3: 'T', 4: 'N'}
seqMapNp = np.array(['A', 'C', 'G', 'T', 'N'])

seqMapComplement = {0: 'T', 1: 'G', 2: 'C', 3: 'A', 4: 'N'}
seqMapComplementNp = np.array(['T', 'G', 'C', 'A', 'N'])

# Base letters for modification calling
# 'H' : m6A, 'I' : m5C, 'J' : m4C, 'K' : m5C/TET
baseToCode = {'N': 0, 'A': 0, 'C': 1, 'G': 2,
              'T': 3, 'H': 4, 'I': 5, 'J': 6, 'K': 7}
baseToCanonicalCode = {'N': 0, 'A': 0, 'C': 1,
                       'G': 2, 'T': 3, 'H': 0, 'I': 1, 'J': 1, 'K': 1}

codeToBase = dict([(y, x) for (x, y) in baseToCode.items()])

pre = 10
post = 4
pad = 30
base4 = 4 ** np.array(range(pre + post + 1))
refDict = {}
refLengthDict = {}

refid = 0

lutPath = "./kineticsTools/resources/SP3-C3.npz.gz"

with gzip.open(lutPath, "rb") as npz_in:
    gbmModelData = np.load(npz_in, allow_pickle=True)
    gbmModel = GbmContextModel(gbmModelData, -1)

for read in tqdm(bamIn, desc='Reads'):
    rawSeq = read.seq
    refSeq = np.frombuffer(rawSeq.encode("utf-8"), dtype=byte)

    # Store the reference length

    length = len(rawSeq)

    # Make a shared array
    sa = SharedArray(dtype='B', shape=len(rawSeq) + pad * 2)
    saWrap = sa.getNumpyWrapper()

    # Lut Codes convert Ns to As so that we don't put Ns into the Gbm Model
    # Seq Codes leaves Ns as Ns for getting reference snippets out
    innerLutCodes = lutCodeMap[refSeq]
    innerSeqCodes = seqCodeMap[refSeq]
    innerCodes = np.bitwise_or(innerLutCodes, np.left_shift(innerSeqCodes, 4))

    saWrap[pad:(len(rawSeq) + pad)] = innerCodes

    # Padding codes -- the lut array is padded with 0s the sequence
    # array is padded with N's (4)
    outerCodes = np.left_shift(np.ones(pad, dtype=uint8) * 4, 4)
    saWrap[0:pad] = outerCodes
    saWrap[(len(rawSeq) + pad):(len(rawSeq) + 2 * pad)] = outerCodes

    snipFunction = snippetFunc(sa, post, pre)
    sites = range(0,length)
    contexts = [snipFunction(sites[x], 1) for x in sites]

    control = gbmModel.getPredictions(contexts)
    ip = read.get_tag('ip')
    ipFrames = codeToFrames(read.get_tag('ip'))
    ipFramesNorm = ipFrames / np.mean(ipFrames)
    ipr = ipFramesNorm / control

    read.set_tag('in', array('f', ipFramesNorm))
    read.set_tag('ir', array('f', ipr))
    bamOut.write(read)

bamOut.close()


#!/usr/bin/env python
# -*- encoding:UTF-8 -*-
import doctest
from typing import Generator, MutableSequence

from common import CT

def rank(mseq: MutableSequence[CT], k: CT) -> CT:
    """Return the `k` rank element of `seq`

    Args:
        seq (MutableSequence[CT]): input sequence
        k (CT): element index

    Returns:
        CT: element in sequence

    >>> import random
    >>> seq = [i for i in range(10)]
    >>> random.shuffle(seq)
    >>> [rank(seq, i) for i in range(10)]
    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
    """
    def partition(seq: MutableSequence[CT], low: int, high: int) -> int:
        """Quick sort partition process, return pivot's changed position
        after partition.

        Args:
            seq (MutableSequence[CT]): input array
            low (int): start index
            high (int): end index

        Returns:
            int: index of pivot
        """
        i, j = low + 1, high
        val = seq[low]
        while 1:
            while i < high and seq[i] <= val:
                i += 1
            while j > low and seq[j] >= val:
                j -= 1
            if i >= j:
                break
            seq[i], seq[j] = seq[j], seq[i]

        seq[low], seq[j] = seq[j], seq[low]
        return j

    low, high = 0, len(mseq) - 1
    while high > low:
        j = partition(mseq, low, high)
        if j == k:
            return mseq[k]
        elif j > k:
            high = j - 1
        elif j < k:
            low = j + 1
    return mseq[k]

# 2.5.4 practice, return a sorted and non-duplicated-item list
def dedup(seq: MutableSequence[CT]) -> Generator[CT, None, None]:
    """Return a new sorted and deduplicated sequence from `seq`.

    Args:
        seq (MutableSequence[CT]): input sequence

    Yields:
        Generator[CT]: generator with unique element

    >>> lst = [i for i in dedup([2, 1, 3, 1, 1, 3, 2, 3, 4, 7])]
    >>> lst
    [1, 2, 3, 4, 7]
    >>> lst2 = [i for i in dedup([1, 1])]
    >>> lst2
    [1]
    >>> lst3 = [i for i in dedup([2, 1, 1, 4, 3, 5])]
    >>> lst3
    [1, 2, 3, 4, 5]
    """
    assert seq and len(seq) >= 2

    new_list = sorted(seq)
    val, count, length = new_list[0], 1, len(new_list)
    for i in range(1, length):
        if new_list[i] == val:
            if i == length - 1:
                yield new_list[i]
            count += 1
        else:
            count = 1
            val = new_list[i]
            yield new_list[i - count]
    if count == 1:
        yield new_list[length - 1]

# 2.5.10 practice, implement a version class with __cmp__
class Version(object):

    """
    >>> lst = [Version(i) for i in ['115.1.1', '115.10.1', '115.10.2']]
    >>> lst.sort()
    >>> lst
    [Version(115.1.1), Version(115.10.1), Version(115.10.2)]
    """

    def __init__(self, version):
        self._version = version

    def __eq__(self, other):
        return self._version == other._version

    def __lt__(self, other):
        return self._version < other._version

    def __repr__(self):
        return 'Version({})'.format(self._version)

    @property
    def version(self):
        return self._version

    @version.setter
    def version(self, val):
        assert all(i.isdigit() for i in val.split('.'))
        self._version = val

# 2.5.14 practice, implement a domain class with __cmp__,
# compare the reversed order domain.
class Domain(object):

    """
    >>> test_list = ['cs.princeton.edu', 'cs.harvard.edu', 'mail.python.org', 'cs.mit.edu']
    >>> lst = [Domain(i) for i in test_list]
    >>> lst.sort()
    >>> lst
    [Domain(cs.harvard.edu), Domain(cs.mit.edu), Domain(cs.princeton.edu), Domain(mail.python.org)]
    """

    def __init__(self, domain):
        self._domain = domain
        self._cmp_domain = '.'.join(reversed(self._domain.split('.')))

    def __eq__(self, other):
        return self._cmp_domain == other._cmp_domain

    def __lt__(self, other):
        return self._cmp_domain < other._cmp_domain

    def __repr__(self):
        return 'Domain({})'.format(self._domain)

    @property
    def domain(self):
        return self._domain

    @domain.setter
    def domain(self, val):
        self._domain = val
        self._cmp_domain = '.'.join(reversed(self._domain.split('.')))

# 2.5.16 practice, construct object which
# order by the name with a new alphabet order
class California(object):

    """
    >>> seq = [California(name) for name
    ... in ('RISBY', 'PATRICK', 'DAMIEN', 'GEORGE')]
    >>> seq.sort()
    >>> seq
    [California(RISBY), California(GEORGE), California(PATRICK), California(DAMIEN)]
    """
    alphabet = ('R', 'W', 'Q', 'O', 'J', 'M', 'V', 'A',
                'H', 'B', 'S', 'G', 'Z', 'X', 'N',
                'T', 'C', 'I', 'E', 'K', 'U', 'P', 'D', 'Y', 'F', 'L')

    def __init__(self, name):
        self._name = name
        self._cmp_tuple = tuple(California.alphabet.index(i)
                                for i in self._name)

    def __eq__(self, other):
        return self._cmp_tuple == other._cmp_tuple

    def __lt__(self, other):
        return self._cmp_tuple < other._cmp_tuple

    def __repr__(self):
        return 'California({})'.format(self._name)

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, val):
        self._name = val
        self._cmp_tuple = tuple(California.alphabet.index(i)
                                for i in self._name)

# 2.5.19 practice, kendall tau algorithm implementation
class KendallTau(object):

    """
    >>> klt = KendallTau()
    >>> klt.kendall_tau_count((0, 3, 1, 6, 2, 5, 4), (1, 0, 3, 6, 4, 2, 5))
    4
    """

    def kendall_tau_count(self, origin_list, count_list):
        lst = [origin_list.index(count_list[i])
               for i in range(len(count_list))]
        aux = lst[:]
        return self.count(lst, aux, 0, len(lst) - 1)

    def count(self, lst, aux, low, high):
        if low >= high:
            return 0
        mid = (low + high) // 2
        lc = self.count(lst, aux, low, mid)
        rc = self.count(lst, aux, mid + 1, high)
        mc = self.merge_count(lst, aux, low, mid, high)
        return lc + rc + mc

    def merge_count(self, lst, aux, low, mid, high):
        aux[low:high + 1] = lst[low:high + 1]
        count, left, right = 0, low, mid + 1
        for j in range(low, high + 1):
            if left > mid:
                lst[j] = aux[right]
                right += 1
            elif right > high:
                lst[j] = aux[left]
                left += 1
            elif aux[left] < aux[right]:
                lst[j] = aux[left]
                left += 1
            else:
                lst[j] = aux[right]
                right += 1
                count += mid - left + 1
        return count

if __name__ == '__main__':
    doctest.testmod()


# test_dbmanager.py

import unittest
import os
from binance import enums as k_binance

from src.pp_order import Order, OrderStatus
from polaris_old.pp_dbmanager import DBManager

TEST_DATABASE = 'test.db'

class TestDBManager(unittest.TestCase):
    def setUp(self) -> None:
        try:
            os.remove(TEST_DATABASE)
        except IOError as e:
            print(e)
        print('cwd: ', os.getcwd())

        self.dbm = DBManager(db_name=TEST_DATABASE, order_tables=['orders'])

        self.order = Order(
            session_id='S_20210501_2008',
            order_id='ORDER_ID',
            pt_id='PT_ID',
            k_side=k_binance.SIDE_BUY,
            price=50_000.0,
            amount=1.0,
        )
        self.order_2 = Order(
            session_id='S_20210501_2008',
            order_id='OR_000001',
            pt_id='PT_000001',
            k_side=k_binance.SIDE_SELL,
            price=60_000.88,
            amount=1.0876548765,
            uid='0123456789abcdef'
        )

    def test_get_table_creation_query(self):
        print(self.dbm.get_table_creation_query(table='orders'))

    def test_add_order(self):
        table = 'orders'
        self.dbm.add_order(table=table, order=self.order)
        c = self.dbm.conn.cursor()
        rows = c.execute(f'SELECT * FROM {table};').fetchall()
        self.assertEqual(1, len(rows))
        row = rows[0]
        self.assertEqual(OrderStatus.MONITOR.name, row[8])

    def test_delete_order(self):
        table = 'orders'
        self.dbm.add_order(table=table, order=self.order)
        self.dbm.add_order(table=table, order=self.order_2)
        self.dbm.delete_order(table=table, order=self.order)
        c = self.dbm.conn.cursor()
        rows = c.execute(f'SELECT * FROM {table};').fetchall()
        self.assertEqual(1, len(rows))
        self.assertEqual('0123456789abcdef', rows[0][0])

    def tearDown(self) -> None:
        self.dbm.conn.close()


import logging
from tvm import topi, te
from tvm.target import Target
from .. import tag

def schedule_pool(outs, layout):
    """Create schedule for avgpool/maxpool"""

    if layout != "NHWC":
        logger = logging.getLogger("topi")
        logger.warning(
            """We currently only support NHWC target specific pools on arm_cpu,
               falling back on generic pool scheduling"""
        )
        return topi.generic.schedule_pool(outs, layout)

    return schedule_pool_2d(outs)

def schedule_pool_2d(outs):
    """Create arm_cpu specific 2D schedule for avgpool/maxpool"""

    outs = [outs] if isinstance(outs, te.tensor.Tensor) else outs
    schedule_ops = [x.op for x in outs]
    schedule = te.create_schedule(schedule_ops)
    scheduled_ops = []

    def traverse(op):
        # Recursively inline any injective operation that isn't the pooling
        # operation or hasn't already been scheduled.
        if tag.is_injective(op.tag):
            if op not in schedule.outputs:
                schedule[op].compute_inline()
            for tensor in op.input_tensors:
                if isinstance(tensor.op, te.tensor.ComputeOp) and tensor.op not in scheduled_ops:
                    traverse(tensor.op)
        # schedule the actual pooling operation
        elif op.tag.startswith("pool"):
            n, height, width, channel = schedule[op].op.axis
            # Average pool consists of two parts; a sum then a division.
            # We can schedule the division loop to parallelize across height and
            # vectorize across width.
            enable_explicit_vectorization = not Target.current(allow_none=False).features.has_sve
            if op != outs[0].op:
                output = outs[0]
                output_fused = schedule[output].fuse(output.op.axis[1], output.op.axis[2])
                schedule[output].parallel(output_fused)
                vectorization_factor = (
                    8 if enable_explicit_vectorization else output.op.axis[3].dom.extent
                )
                _, inner = schedule[output].split(output.op.axis[3], vectorization_factor)
                schedule[output].vectorize(inner)

            padded_input = op.input_tensors[0]
            if isinstance(padded_input.op, te.tensor.ComputeOp):
                schedule[padded_input].compute_inline()

            # For targets without SVE try explicitly vectorizing the channel
            # loop, For SVE targets leave the loop in place for LLVM to convert
            # into a scalable vector loop.
            vectorization_factor = 8 if enable_explicit_vectorization else channel.dom.extent
            channel_outer, channel_inner = schedule[op].split(channel, vectorization_factor)
            schedule[op].vectorize(channel_inner)
            schedule[op].parallel(height)
            if len(schedule[op].op.reduce_axis) > 0:
                filter_height, filter_width = schedule[op].op.reduce_axis
                # We consider any filter of area < 10 to be small enough to
                # unroll; 3x3 filters have shown better performance when
                # unrolled.
                if filter_height.dom.extent * filter_width.dom.extent <= 9:
                    # For small filters, unrolling the filter loops allows us to
                    # vectorize over channels without reordering anything.
                    schedule[op].unroll(filter_width)
                    schedule[op].unroll(filter_height)
                else:
                    # Reordering so that channels is the fastest moving axis allows
                    # LLVM to vectorize across contiguous memory in the NHWC
                    # ordering.
                    schedule[op].reorder(
                        n, height, width, filter_height, filter_width, channel_outer, channel_inner
                    )
            else:
                schedule[op].reorder(n, height, width, channel_outer, channel_inner)
        else:
            raise RuntimeError("Unsupported operator: %s" % op.tag)

        scheduled_ops.append(op)

    traverse(outs[0].op)
    return schedule


import random

from astral import Astral
from datetime import datetime, timedelta, time
from timedevent import TimedEvent
from scenes import VACATION_SCENES

import logging
logger = logging.getLogger(__name__)

#
#   This event builds the script of events for vacation mode for a single day.  Once it executes, it will rescue
#
class VacationBuildEvent(TimedEvent):
    def __init__(self, event_time, parent):
        TimedEvent.__init__(self, event_time)
        self.parent = parent

    def execute(self):
        date = datetime.now().date()
        events = self.parent.create_scene_script(date)
        for event in events:
            if event.event_time >= datetime.now():
                self.parent.home.timed_event_queue.add_event(event)
        date += timedelta(days=1)
        next_build_time = datetime.combine(date, time(10))      # Build the next day's script at 10am
        self.parent.home.timed_event_queue.add_event(VacationBuildEvent(next_build_time, self.parent))

class VacationLightEvent(TimedEvent):
    def __init__(self, event_time, home, scene_off, scene_on):
        TimedEvent.__init__(self, event_time)
        self.home = home
        self.scene_off = scene_off
        self.scene_on = scene_on

    def execute(self):
        if self.scene_off is not None:
            logger.info("Vacation turning off {0}".format(self.scene_off.names[0]))
            self.scene_off.off(self.home.lights)
        if self.scene_on is not None:
            logger.info("Vacation turning on {0}".format(self.scene_on.names[0]))
            self.scene_on.on(self.home.lights)

class VacationMode:
    def __init__(self, home, start=timedelta(minutes=-30), end_time=time(23, 0, 0), interval=timedelta(minutes=30)):
        self.start = start          # Can be either a timedelta or a datetime.time
        self.end_time = end_time
        self.interval = interval
        self.home = home
        self.enabled = False

    # After all the services have been started this method is called.  If the persistent store shows that we are
    # in vacation mode then we probably just had a power failure.
    def init_from_persistent_store(self):
        if self.home.persistent_store.get_value('vacation_mode', default=False):
            logger.info("Enabling vacation mode because enabled in persistent store")
            self.enable()

    def enable(self, delay=timedelta(minutes=5)):
        if not self.enabled:
            logger.info("Enabling vacation mode")
            self.enabled = True
            self.home.persistent_store['vacation_mode'] = True
            self.home.timed_event_queue.add_event(VacationBuildEvent(datetime.now() + delay, self))

    def disable(self):
        if self.enabled:
            logger.info("Disabling vacation mode")
            self.enabled = False
            self.home.persistent_store['vacation_mode'] = False
            self.home.timed_event_queue.remove_events(VacationBuildEvent)
            self.home.timed_event_queue.remove_events(VacationLightEvent)

    def create_scene_script(self, date):
        if isinstance(self.start, timedelta):
            a = Astral()
            start_time = (a['seattle'].sun(date=date, local=True)['sunset'] + self.start).time()
        else:
            start_time = self.start
        last_on = None
        events = []
        start_datetime = datetime.combine(date, start_time)
        end_datetime = datetime.combine(date, self.end_time)
        while start_datetime < end_datetime:
            next_on = random.choice(VACATION_SCENES.all_scenes)
            if next_on == last_on:
                next_on = None      # If we randomly choose the same scene twice in a row, don't turn anything on
            events.append(VacationLightEvent(start_datetime, self.home, last_on, next_on))
            last_on = next_on
            start_datetime += self.interval
        # always turn off the last_on light at the end of the script
        events.append(VacationLightEvent(start_datetime, self.home, last_on, None))
        return events


"""
    Search module for searching the web based on user queries.
"""

import asyncio
from search_engine_parser.core.engines.yahoo import Search as YahooSearch

def searcher(query, n_results, n_pages, only_description):
    """
    Search the web using Yahoo Search engine and retrieve search results.

    Parameters:
        query : str
            The search query.

        n_results : int
            The number of search results to retrieve.

        n_pages : int
            The number of search result pages to crawl.

        only_description : bool
            If True, only retrieve the descriptions of the search results.
            If False, retrieve the titles, links, and descriptions of the search results.

    Returns:
        The search results.
    """
    results = []
    for i in range(1, n_pages + 1):
        search_args = (query, i)
        ysearch = YahooSearch()

        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        yresults = ysearch.search(*search_args)

        for title, link, description in zip(
            yresults["titles"], yresults["links"], yresults["descriptions"]
        ):
            if only_description:
                results.append(f"|DESCRIPTIONS: {description}")
            else:
                results.append(
                    f"|TITLE: {title} LINK: {link} DESCRIPTIONS: {description}"
                )

    return results[:n_results]


from django.db import models
from django.contrib.auth.models import User
from django.db.models.signals import post_save

from datetime import datetime

##
## begin util functies
##
def nvl(value,default_value) :
  return default_value if value is None else value

##
## einde util functies
##

class AukArtikelRankVw(models.Model):
    artikel_id = models.IntegerField()
    artikel_titel = models.CharField(max_length=200)
    krant = models.TextField()
    cat_id = models.IntegerField()
    cat_volgorde = models.IntegerField()
    categorie = models.CharField(max_length=200)
    pubdate = models.DateTimeField()
    pubdate_yyyymmdd = models.TextField()
    krant_weight = models.DecimalField(decimal_places=6,max_digits=10)
    calculated_weight = models.DecimalField(decimal_places=6,max_digits=10)
    cloud_weight = models.DecimalField(decimal_places=6,max_digits=10)
    total_weight = models.DecimalField(decimal_places=6,max_digits=10)
    rank = models.DecimalField(decimal_places=6,max_digits=10)
#    class Meta:
#        abstract = True

class AukArtikel(models.Model):
    uniek = models.CharField(max_length=200, unique=True)
    titel = models.CharField(max_length=200)
    pubdate = models.DateTimeField()
    rssfeeddescription = models.CharField(max_length=4000)
    url = models.CharField(max_length=1000)
    body = models.CharField(max_length=64000)
    body_clean = models.CharField(max_length=64000)
#    body_stemmed = models.CharField(max_length=64000)
    class Meta:
        db_table = u'auk_artikel'

    def __unicode__(self):
        return u'%s' % (self.uniek)

#Alles van de krant
class Krant(models.Model):
#    id = models.IntegerField(primary_key=True)
    naam = models.CharField(unique=True, max_length=100)
    url = models.CharField(max_length=1000)
    class Meta:
        db_table = u'auk_krant'

    def __unicode__(self):
        return u'%s' % (self.naam)

class Rubriek(models.Model):
    krt = models.ForeignKey(Krant)
    naam = models.CharField(max_length=100)
    url = models.CharField(unique=True,max_length=1000)
    actief = models.CharField(max_length=1)
    class Meta:
        db_table = u'auk_rubriek'

    def __unicode__(self):
        return u'%s' % (self.naam)

class Categorie(models.Model):
#    id = models.IntegerField(primary_key=True)
    volgorde = models.IntegerField()
    naam = models.CharField(unique=True,max_length=100)
    cloud = models.CharField(max_length=4000,null=True,blank=True)
    cloud_factor = models.DecimalField(default=1,max_digits=4, decimal_places=2)
    aging_factor = models.DecimalField(default=1,max_digits=4, decimal_places=2)
    class Meta:
        db_table = u'auk_categorie'

    def __unicode__(self):
        return u'%s' % (self.naam)

class GebruikerCategorie(models.Model):
#    id = models.IntegerField(primary_key=True)
    usr = models.ForeignKey(User)
    cat = models.ForeignKey(Categorie)
    volgorde = models.IntegerField()
    aantal_artikelen = models.IntegerField()
    class Meta:
        db_table = u'auk_gebruiker_categorie'
        unique_together = ("usr","cat")

#
# Intersectie tussen rubriek en categorie
#

class AukRubriekCategorie(models.Model):
    rbk = models.ForeignKey(Rubriek)
    cat = models.ForeignKey(Categorie)
    class Meta:
        db_table = u'auk_rubriek_categorie'
        unique_together = ("rbk","cat")

#
# Intersectie tussen artikel en categorie
#

class AukArtikelCategorie(models.Model):
    art               = models.ForeignKey(AukArtikel)
    cat               = models.ForeignKey(Categorie)
    krant_weight      = models.FloatField(null=True)
    calculated_weight = models.FloatField(null=True)
    cloud_weight      = models.FloatField(null=True)
    total_weight      = models.FloatField(null=True)
    rank              = models.IntegerField(null=True)

    class Meta:
        db_table = u'auk_artikel_categorie'
        unique_together = ("art","cat")

#
# Intersectie tussen artikel en rubriek
#

class AukArtikelPublicatie(models.Model):
    art = models.ForeignKey(AukArtikel)
    rbk = models.ForeignKey(Rubriek)

    class Meta:
        db_table = u'auk_artikel_publicatie'
        unique_together = ("art", "rbk")

#
# Search and Rank tabellen
#
class SarIgnorewordlist(models.Model):
    word = models.CharField(max_length=4000)
    class Meta:
        db_table = u'sar_ignorewordlist'

class SarWordlist(models.Model):
    word = models.CharField(unique=True, max_length=4000)
    class Meta:
        db_table = u'sar_wordlist'

class SarWordlocation(models.Model):
    art = models.ForeignKey(AukArtikel)
    wrd = models.ForeignKey(SarWordlist)
    location = models.IntegerField()
    class Meta:
        db_table = u'sar_wordlocation'
        unique_together = ("art", "wrd", "location")

#
#
# Logging om hierna learning te bewerkstelligen
#
class LogAction(models.Model):
     usr = models.ForeignKey(User,null=True)
     art = models.ForeignKey(AukArtikel,null=True)
     session_key       = models.CharField(max_length=40)
     first_action_time = models.DateTimeField(default=datetime.now)
     last_action_time  = models.DateTimeField()
     action            = models.CharField(max_length=100)
     spent_time        = models.IntegerField(default=0)
     unique_together = ("usr","art","session_key")

class Vote(models.Model) :
     usr   = models.ForeignKey(User)
     art   = models.ForeignKey(AukArtikel)
     score = models.IntegerField()
     unique_together = ("usr","art")


from flask import Flask, request, jsonify
import json

# 将当前程序作为web app应用
# 配置static模式
# static_folder静态文件的路径
app = Flask(__name__,static_folder='static',static_url_path='/')

@app.route(rule='/sales', methods=['GET'])
def post_test():
    # 附加题: 从excel表中读取这些数据
    sales_data = [20, 20, 30, 10, 10, 15]
    r = jsonify(sales_data)
    return r

if __name__ == '__main__':
    app.run(port=80, debug=True)


import pysam, sys, getopt

argv = sys.argv[1:]
opts, args = getopt.getopt(argv, 'i:o:t:f:')
input_file = None
output_directory = None
tag = None
names = None
for opt, arg in opts:
    if opt == '-i':
        input_file = arg
    elif opt == '-o':
        output_directory = arg
    elif opt == '-t':
        tag = arg
    elif opt == '-f':
        names = arg
if input_file is None or output_directory is None or tag is None:
    print('arguments wrong')
    sys.exit(2)

pure_bam = pysam.AlignmentFile(input_file)
memory_read_arrays = {}
uniq_names = None
if names is not None:
    uniq_names = set()
    names = open(names, mode='r')
    for line in names.readlines():
        uniq_names.add(line[0:len(line)-1])

for read in pure_bam:
    try:
        key = read.get_tag(tag)
    except:
        continue
    if uniq_names is not None:
        if key not in uniq_names:
            continue
    if memory_read_arrays.get(key) is None:
        read_array = list()
    else:
        read_array = memory_read_arrays.get(key)
    read_array.append(read)
    memory_read_arrays[key] = read_array

for key in memory_read_arrays:
    f = pysam.AlignmentFile(output_directory + key + '.bam', 'wb', template=pure_bam)
    for read in memory_read_arrays.get(key):
        f.write(read)
    f.close()
pure_bam.close()


"""
PCBA dataset loader.
"""
import os
import logging
import deepchem
import gzip

logger = logging.getLogger(__name__)

DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir()

def load_pcba(featurizer='ECFP',
              split='random',
              reload=True,
              data_dir=None,
              save_dir=None,
              **kwargs):
  return load_pcba_dataset(
      featurizer=featurizer,
      split=split,
      reload=reload,
      assay_file_name="pcba.csv.gz",
      data_dir=data_dir,
      save_dir=save_dir,
      **kwargs)

def load_pcba_146(featurizer='ECFP',
                  split='random',
                  reload=True,
                  data_dir=None,
                  save_dir=None,
                  **kwargs):
  return load_pcba_dataset(
      featurizer=featurizer,
      split=split,
      reload=reload,
      assay_file_name="pcba_146.csv.gz",
      data_dir=data_dir,
      save_dir=save_dir,
      **kwargs)

def load_pcba_2475(featurizer='ECFP',
                   split='random',
                   reload=True,
                   data_dir=None,
                   save_dir=None,
                   **kwargs):
  return load_pcba_dataset(
      featurizer=featurizer,
      split=split,
      reload=reload,
      assay_file_name="pcba_2475.csv.gz",
      data_dir=data_dir,
      save_dir=save_dir,
      **kwargs)

def load_pcba_dataset(featurizer='ECFP',
                      split='random',
                      reload=True,
                      assay_file_name="pcba.csv.gz",
                      data_dir=None,
                      save_dir=None,
                      **kwargs):
  """Load PCBA dataset

  PubChem BioAssay (PCBA) is a database consisting of biological activities of
  small molecules generated by high-throughput screening. We use a subset of
  PCBA, containing 128 bioassays measured over 400 thousand compounds,
  used by previous work to benchmark machine learning methods.

  Random splitting is recommended for this dataset.

  The raw data csv file contains columns below:

  - "mol_id" - PubChem CID of the compound
  - "smiles" - SMILES representation of the molecular structure
  - "PCBA-XXX" - Measured results (Active/Inactive) for bioassays:
        search for the assay ID at
        https://pubchem.ncbi.nlm.nih.gov/search/#collection=bioassays
        for details

  References
  ----------
  .. [1] Wang, Yanli, et al. "PubChem's BioAssay database."
     Nucleic acids research 40.D1 (2011): D400-D412.
  """
  if data_dir is None:
    data_dir = DEFAULT_DIR
  if save_dir is None:
    save_dir = DEFAULT_DIR

  if reload:
    save_folder = os.path.join(save_dir,
                               assay_file_name.split(".")[0] + "-featurized",
                               featurizer)
    if featurizer == "smiles2img":
      img_spec = kwargs.get("img_spec", "std")
      save_folder = os.path.join(save_folder, img_spec)
    save_folder = os.path.join(save_folder, str(split))

  dataset_file = os.path.join(data_dir, assay_file_name)

  if not os.path.exists(dataset_file):
    deepchem.utils.data_utils.download_url(
        url="https://deepchemdata.s3-us-west-1.amazonaws.com/datasets/{0}".
        format(assay_file_name),
        dest_dir=data_dir)

  # Featurize PCBA dataset
  logger.info("About to featurize PCBA dataset.")
  if featurizer == 'ECFP':
    featurizer = deepchem.feat.CircularFingerprint(size=1024)
  elif featurizer == 'GraphConv':
    featurizer = deepchem.feat.ConvMolFeaturizer()
  elif featurizer == 'Weave':
    featurizer = deepchem.feat.WeaveFeaturizer()
  elif featurizer == 'Raw':
    featurizer = deepchem.feat.RawFeaturizer()
  elif featurizer == "smiles2img":
    img_spec = kwargs.get("img_spec", "std")
    img_size = kwargs.get("img_size", 80)
    featurizer = deepchem.feat.SmilesToImage(
        img_size=img_size, img_spec=img_spec)

  with gzip.GzipFile(dataset_file, "r") as fin:
    header = fin.readline().rstrip().decode("utf-8")
    columns = header.split(",")
    columns.remove("mol_id")
    columns.remove("smiles")
    PCBA_tasks = columns

  if reload:
    loaded, all_dataset, transformers = deepchem.utils.data_utils.load_dataset_from_disk(
        save_folder)
    if loaded:
      return PCBA_tasks, all_dataset, transformers

  loader = deepchem.data.CSVLoader(
      tasks=PCBA_tasks, smiles_field="smiles", featurizer=featurizer)

  dataset = loader.featurize(dataset_file)

  if split == None:
    transformers = [deepchem.trans.BalancingTransformer(dataset=dataset)]

    logger.info("Split is None, about to transform data")
    for transformer in transformers:
      dataset = transformer.transform(dataset)

    return PCBA_tasks, (dataset, None, None), transformers

  splitters = {
      'index': deepchem.splits.IndexSplitter(),
      'random': deepchem.splits.RandomSplitter(),
      'scaffold': deepchem.splits.ScaffoldSplitter(),
      'stratified': deepchem.splits.SingletaskStratifiedSplitter()
  }
  splitter = splitters[split]
  logger.info("About to split dataset using {} splitter.".format(split))
  frac_train = kwargs.get("frac_train", 0.8)
  frac_valid = kwargs.get('frac_valid', 0.1)
  frac_test = kwargs.get('frac_test', 0.1)

  train, valid, test = splitter.train_valid_test_split(
      dataset,
      frac_train=frac_train,
      frac_valid=frac_valid,
      frac_test=frac_test)

  transformers = [deepchem.trans.BalancingTransformer(dataset=train)]

  logger.info("About to transform dataset.")
  for transformer in transformers:
    train = transformer.transform(train)
    valid = transformer.transform(valid)
    test = transformer.transform(test)

  if reload:
    deepchem.utils.data_utils.save_dataset_to_disk(save_folder, train, valid,
                                                   test, transformers)

  return PCBA_tasks, (train, valid, test), transformers


import requests
from bs4 import BeautifulSoup

def music_Leaderboard(input_country):
    country = {"西洋": "3",
               "日韓": "2",
               "華語": "1"}
    req = requests.get(
        'https://www.kiss.com.tw/music/billboard.php?a=%s' % (country[input_country]))
    soup = BeautifulSoup(req.text, 'html.parser')
    music_list = soup.find('tbody')
    music_info = music_list.find_all('td')
    music_name = []
    for music in music_info:
        music_name.append(music.get_text())
    show = ''
    for i in range(8, len(music_name)+8, 8):
        show += "名次:%s\n歌名:%s\n歌手:%s\n專輯名稱:%s\n發行公司:%s\n\n" % (
            music_name[i-8], music_name[i-7], music_name[i-6], music_name[i-5], music_name[i-4])
    return show


from pylab import *
from matplotlib.patches import FancyArrow

# Foreman-Mackey's taste in figures
rc("font", size=20, family="serif", serif="Computer Sans")
rc("text", usetex=True)

import word
import parameters

t = linspace(-10., 10., 10001)
plot(t, word.TwoExp(t).model(parameters.TwoExpParameters(1.0, 0.5, 1., 3., log=False)), linewidth=2)
xlabel('Time (seconds)')
ylabel('Poisson Rate')
ylim([0., 1.1])
axvline(1., color='r', linestyle='--')
title('A Word')

# Build an arrow.
ar1 = FancyArrow(1., 1.01*exp(-1.), -0.5, 0., length_includes_head=True,
                color='k', head_width=0.01, head_length=0.2, width=0.001, linewidth=1)
ar2 = FancyArrow(1., 0.99*exp(-1.), 1.5, 0., length_includes_head=True,
                color='k', head_width=0.01, head_length=0.2, width=0.001, linewidth=1)
ax = gca()
# Add the arrow to the axes.
ax.add_artist(ar1)
ax.add_artist(ar2)

# Add text
text(-0.4, 1.*exp(-1.), r'$\tau$')
text(2.7, 1.*exp(-1.), r'$\tau S$')

savefig('documents/word.pdf', bbox_inches='tight')
show()


import training.dataset
import unittest
import model.rigid_body_model as rbm
import model.param as model_param
import numpy as np
from scipy.spatial.transform import Rotation as R

class TestDexGraspDataset(unittest.TestCase):
    def setUp(self) -> None:
        self.hand_plant = rbm.AllegroHandPlantDrake(meshcat_open_brower=False,
                                           num_viz_spheres=0)

    def test_make_dataset_from_same_point_cloud(self):
        rs = np.random.RandomState(0)
        copies = 3
        dut = training.dataset.make_dataset_from_same_point_cloud('003_cracker_box',
                                                                make_datset=False, random_xy=False,
                                                                random_yaw=False,
                                                                copies=copies,
                                                                data_idx_start=0, data_idx_end=1,
                                                                random_state=rs)
        # Theres should be exactly 10 data points
        self.assertEqual(len(dut), copies)
        # construct with random orientation
        dut_orn = training.dataset.make_dataset_from_same_point_cloud('003_cracker_box',
                                                                make_datset=False, random_xy=False,
                                                                random_yaw=True,
                                                                copies=copies,
                                                                data_idx_start=0, data_idx_end=1,
                                                                random_state=rs)

        def test_dataset_consistency(d):
            # Check that fingertip_normals is consistent with the fingertip positions
            # computed from (base_position, base_quaternion, finger_q)
            diagram_context, plant_context = self.hand_plant.create_context()
            gt_fingertip_angles_dict = {}
            gt_fingertip_angles_dict = model_param.finger_q_to_finger_angles_dict(d.finger_q)
            gt_fingertip_angles_dict[model_param.AllegroHandFinger.RING] = np.zeros(4)
            # Compute the corresponding fingeritp locations
            gt_drake_q = self.hand_plant.convert_hand_configuration_to_q(d.base_position,
                d.base_quaternion, gt_fingertip_angles_dict
            )
            gt_p_WF = self.hand_plant.compute_p_WF(gt_drake_q, plant_context)
            gt_fingertip_normals = d.fingertip_normals
            for fi, finger in enumerate(model_param.ActiveAllegroHandFingers):
                np.testing.assert_allclose(gt_fingertip_normals[fi,:3],
                np.squeeze(gt_p_WF[finger]),
                atol=1e-2 # Allow large tolerance as gt_p_WF is from IK solution
            )

        for idx in range(len(dut)):
            # Retrieve the random orientation
            original_entry = dut[idx]
            rotated_entry = dut_orn[idx]
            original_base_quaternion = R.from_quat(original_entry.base_quaternion[[1,2,3,0]])
            rotated_base_quaternion = R.from_quat(rotated_entry.base_quaternion[[1,2,3,0]])
            rot_diff = rotated_base_quaternion*original_base_quaternion.inv()
            # Rotate everything else back
            rot_diff_matrix = rot_diff.as_matrix()
            # Compare point clouds
            pc_original = original_entry.point_cloud
            pc_rot = rotated_entry.point_cloud
            np.testing.assert_allclose(pc_rot,
                                        (rot_diff_matrix @ (pc_original.T)).T)
            # Compare normals
            fingertip_normals_original = original_entry.fingertip_normals
            fingertip_normals_rot = rotated_entry.fingertip_normals
            # Check fingertip positions
            np.testing.assert_allclose(fingertip_normals_rot[:,:3],
                                        (rot_diff_matrix @ (fingertip_normals_original[:,:3].T)).T)
            # Check fingertip normal vectors
            np.testing.assert_allclose(fingertip_normals_rot[:,3:],
                            (rot_diff_matrix @ (fingertip_normals_original[:,3:].T)).T)
            # Check finger q
            np.testing.assert_allclose(original_entry.finger_q, rotated_entry.finger_q)

            # Check base position
            np.testing.assert_allclose(rotated_entry.base_position, rot_diff_matrix @ original_entry.base_position)

            # Test consistency
            test_dataset_consistency(original_entry)
            test_dataset_consistency(rotated_entry)

        # construct with random translation
        dut_trans = training.dataset.make_dataset_from_same_point_cloud('003_cracker_box',
                                                                make_datset=False, random_xy=True,
                                                                random_yaw=False,
                                                                copies=copies,
                                                                data_idx_start=0, data_idx_end=1,
                                                                random_state=rs)
        for idx in range(len(dut)):
            # Retrieve the random translation
            original_entry = dut[idx]
            translated_entry = dut_trans[idx]
            trans = translated_entry.base_position-original_entry.base_position
            # Compare point clouds
            pc_original = original_entry.point_cloud
            pc_trans = translated_entry.point_cloud
            pc_diff = pc_trans-pc_original
            self.assertTrue(np.allclose(pc_diff, trans))
            # Compare normals
            fingertip_normals_original = original_entry.fingertip_normals
            fingertip_normals_trans = translated_entry.fingertip_normals
            # Check fingertip positions
            self.assertTrue(np.allclose((fingertip_normals_trans-fingertip_normals_original)[:,:3],
                                        trans))
            # Check fingertip normal vectors
            np.testing.assert_allclose(fingertip_normals_trans[:,3:], fingertip_normals_original[:,3:])
            # Check finger q
            np.testing.assert_allclose(original_entry.finger_q, translated_entry.finger_q)
            np.testing.assert_allclose(original_entry.base_quaternion, translated_entry.base_quaternion)
            test_dataset_consistency(original_entry)
            test_dataset_consistency(translated_entry)

if __name__ == '__main__':
    unittest.main()


import cv2
import numpy as np
import matplotlib.pyplot as plt
def 垂直边缘提取():
    #差分求图像梯度,应用特殊卷积求差分.
    img = cv2.imread("opencv\\files\sudoku.jpg", 0)
    #卷积核
    kernel = np.array([[-1, 0, 1],
                        [-2, 0, 2],
                        [-1, 0, 1]], dtype=np.float32)
            #这样的核可以提取竖直边缘，转置后可提取水平边缘
    #提取竖直边缘
    dst_v = cv2.filter2D(img, -1, kernel)
    #提取水平边缘
    dst_h = cv2.filter2D(img, -1, kernel.T)
    cv2.imshow('img', np.hstack((img, dst_v, dst_h)))
    cv2.waitKey(0)
#垂直边缘提取()

def Sobel算子():
    #sobel算子是高斯平滑和微分操作的结合体,抗噪声能力强
    #先在垂直方向计算梯度，后再水平方向计算梯度，然后求总梯度
    img = cv2.imread("opencv\\files\sudoku.jpg", 0)
    sobelx = cv2.Sobel(img, -1, 1, 0, ksize=3)  #只计算x方向
    sobely = cv2.Sobel(img, -1, 0, 1, ksize=3)  #只计算y方向
    sobel = np.sqrt((np.square(sobelx) + np.square(sobely)))
    #Scharr算子
    scharrx = cv2.Scharr(img, -1, 1, 0)  # 只计算x方向
    scharry = cv2.Scharr(img, -1, 0, 1)  # 只计算y方向
    scharr = np.sqrt((np.square(scharrx) + np.square(scharry)))
    cv2.imshow("img", np.hstack((sobel.astype(int).astype(
        float), scharr.astype(int).astype(float))))
    cv2.waitKey(0)
    '''
    Prewitt算子k=[[-1,0,1],[-1,0,1],[-1,0,1]]
    Scharr算子(比Sobel更好用)k=[[-3,0,3],[-10,0,10],[-3,0,3]]
    '''
#Sobel算子()

#Laplacian算子是二姐边缘检测的典型代表
def Laplacian算子():
    img = cv2.imread('opencv\\files\sudoku.jpg', 0)
    laplacian = cv2.Laplacian(img, -1)
    cv2.imshow('img', np.hstack((img, laplacian)))
    cv2.waitKey(0)
#Laplacian算子()

def Canny边缘检测():
    img = cv2.imread('opencv\\files\\number13.jpg', 0)
    #阈值分割,(再检测边缘，效果更好)
    ret, th = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
    #30，70分别为低、高阈值
    edge = cv2.Canny(th, 30, 70)
    cv2.imshow('img', np.hstack((img, th, edge)))
    cv2.waitKey(0)
#Canny边缘检测()

def 图像金字塔():
    '''
    下采样：高斯内核卷积。去除偶数行列，压缩图像
    上采样：在每个方向放大两倍，新增的行列用0填充。再用内核卷积。
    '''
    img = cv2.imread('opencv\\files\dog.jpg', 1)
    #下采样,尺寸变小，分辨率降低
    lower_reso = cv2.pyrDown(img)
    #从下采样本再上采样，尺寸变大，但分辨率不会变高，因为信息丢失不会回来。
    higher_reso=cv2.pyrUp(lower_reso)
    cv2.imshow('img', img)
    cv2.imshow('lower', lower_reso)
    cv2.imshow('higher', higher_reso)
    cv2.waitKey(0)
#图像金字塔()

def 金字塔混合():
    '''
    图像金字塔的一个应用就是图像融合。
    例如，在图像缝合中，如果你需要将两幅图叠在一起，但是由于连接区域图像像素不连续性，政府图像效果看起来很差，这时就可以通过图像金字塔进行融合。
    步骤如下：
    1、读入两幅图像
    2、构建img1和img2的高斯金字塔
    3、根据高斯金字塔计算拉普拉斯金字塔(拉普拉斯金字塔适合用于重建)
    4、在拉普拉斯的每一层进行图像融合
    5、根据融合后的图像金字塔重建原始图像
    '''
    A = cv2.imread('opencv\\files\dog.jpg', 1)
    B = cv2.imread('opencv\\files\dog2.jpg', 1)
    A = cv2.resize(A, (256, 256), cv2.INTER_LINEAR)
    B = cv2.resize(B, (256, 256), cv2.INTER_LINEAR)
    print(A.shape, B.shape)
    #生成A的高斯金字塔
    G = A.copy()
    gpA = [G]
    for i in range(6):
        G = cv2.pyrDown(G)
        gpA.append(G)
        print(G.shape)
    #gpA=[A,1/4A,1/16A,...]
    #生成B的高斯金字塔
    G = B.copy()
    gpB = [G]
    for i in range(6):
        G = cv2.pyrDown(G)
        gpB.append(G)
        print(G.shape)
    lpA = [gpA[5]]  # 进行拉普拉斯金字塔处理，总共5级处理
    for i in range(5, 0, -1):
        GE = cv2.pyrUp(gpA[i])
        # print(GE.shape)
        # print(gpA[i].shape)
        L = cv2.subtract(gpA[i-1], GE)#subtract图像相减
        lpA.append(L)
    # generate Laplacian Pyramid for B
    lpB = [gpB[5]]  # 进行拉普拉斯金字塔处理，总共5级处理
    for i in range(5, 0, -1):
        GE = cv2.pyrUp(gpB[i])
        L = cv2.subtract(gpB[i-1], GE)
        lpB.append(L)
    #左右拼接，在每一层拉普拉斯金字塔进行左右拼接
    LS = []
    for la, lb in zip(lpA, lpB):
        rows, cols, dpt = la.shape
        print('la', la.shape)
        ls = np.hstack((la[:, 0:cols // 2,:], lb[:, cols // 2:,:]))#各取一半
        LS.append(ls)
    ls_ = LS[0]
    #利用拉普拉斯金字塔，复原图像。add是图像相加
    for i in range(1, 6):
        ls_ = cv2.pyrUp(ls_)
        ls_ = cv2.add(ls_, LS[i])
    real = np.hstack((A[:,:cols // 2,:], B[:, cols // 2:,:]))
    cv2.imshow('Direct_blending---Pyramid_blending',np.hstack((real,ls_)))
    k=cv2.waitKey(0)
金字塔混合()


from sqlalchemy.engine import Connection

from saltapi.repository.finder_chart_repository import FinderChartRepository
from saltapi.service.finder_chart_service import FinderChartService

def test_get_finder_chart(db_connection: Connection) -> None:
    finding_chart_id = 55345
    expected_proposal_code = "2015-2-SCI-028"
    expected_finder_chart_path = (
        expected_proposal_code + "/4/Included/FindingChart_1445723219288.pdf"
    )

    repository = FinderChartRepository(db_connection)
    service = FinderChartService(repository)

    proposal_code, finding_chart_path = service.get_finder_chart(
        f"{finding_chart_id}.pdf"
    )

    assert expected_proposal_code == proposal_code
    assert expected_finder_chart_path in str(finding_chart_path)


# The data set used in this example is from http://archive.ics.uci.edu/ml/datasets/Wine+Quality
# P. Cortez, A. Cerdeira, F. Almeida, T. Matos and J. Reis.
# Modeling wine preferences by data mining from physicochemical properties. In Decision Support Systems, Elsevier, 47(4):547-553, 2009.

"""
CREATE LOCAL mlruns
"""

import warnings
import sys

import pandas as pd
import numpy as np
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
from sklearn.model_selection import train_test_split
from sklearn.linear_model import ElasticNet
from urllib.parse import urlparse
import mlflow.sklearn
from mlflow.models.signature import infer_signature

from  tests import MODEL_NAME

import logging

from shippedbrain import shippedbrain

logging.basicConfig(level=logging.WARN)
logger = logging.getLogger(__name__)

def eval_metrics(actual, pred):
    rmse = np.sqrt(mean_squared_error(actual, pred))
    mae = mean_absolute_error(actual, pred)
    r2 = r2_score(actual, pred)
    return rmse, mae, r2

def train_and_eval(alpha, l1_ratio, train_x, train_y, test_x, test_y):
    lr = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, random_state=46)

    lr.fit(train_x, train_y)

    predicted_qualities = lr.predict(test_x)

    (rmse, mae, r2) = eval_metrics(test_y, predicted_qualities)

    # Infer model signature
    signature = infer_signature(test_x, predicted_qualities)

    return lr, predicted_qualities, signature, rmse, mae, r2

def build_train():
    # Read the wine-quality csv file from the URL
    # data = pd.read_csv("./tests/resources/data/winequality-red.csv", sep=",", header=True)
    csv_url = (
        "http://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv"
    )
    try:
        print("Downloading dataset...")
        data = pd.read_csv(csv_url, sep=";")
    except Exception as e:
        logger.exception(
            "Unable to download training & test CSV, check your internet connection. Error: %s", e
        )

    # Split the data into training and test sets. (0.75, 0.25) split.
    train, test = train_test_split(data)

    # The predicted column is "quality" which is a scalar from [3, 9]
    train_x = train.drop(["quality"], axis=1)
    test_x = test.drop(["quality"], axis=1)
    train_y = train[["quality"]]
    test_y = test[["quality"]]

    alpha = 0.5
    l1_ratio = 0.5

    return train_x, train_y, test_x, test_y, alpha, l1_ratio

def main(log_model_option: dict = {"flavor": "mlflow"}, run_inside_mlflow_context: bool = True):
    """ Train and log model

    :param log_model_option: Log model using options:
                             if {"flavor": "mlflow"}: log model using mlflow log_model method
                             else if {"flavor": "_log_model"} log model using shipped brain _log_model function
                             else ig {"flavor": "upload_run" | "upload_model", args...} log model using named function with args
                             NB: input_example and signature are not required
    :param run_inside_mlflow_context: if True run log method from mlflow run context,
                                      otherwise use shippedbrain.log_flavor outside without mlflow run context
    """
    warnings.filterwarnings("ignore")
    np.random.seed(46)

    print("MLflow Tracking URI:", mlflow.get_tracking_uri())

    train_x, train_y, test_x, test_y, alpha, l1_ratio = build_train()

    lr, predicted_qualities, signature, rmse, mae, r2 = train_and_eval(alpha,
                                                                       l1_ratio,
                                                                       train_x,
                                                                       train_y,
                                                                       test_x,
                                                                       test_y)

    log_model_option["signature"] = signature
    log_model_option["input_example"] = test_x.iloc[0:2]

    print(f"[INFO] RUN INSIDE MLFLOW RUN CONTEXT={run_inside_mlflow_context}")
    if run_inside_mlflow_context:
        with mlflow.start_run() as run:
            print("[INFO] Starting run with id:", run.info.run_id)

            print("[INFO]Elasticnet model (alpha=%f, l1_ratio=%f):" % (alpha, l1_ratio))
            print("[INFO]\tRMSE: %s" % rmse)
            print("[INFO]\tMAE: %s" % mae)
            print("[INFO]\tR2: %s" % r2)

            mlflow.log_param("alpha", alpha)
            mlflow.log_param("l1_ratio", l1_ratio)
            mlflow.log_metric("rmse", rmse)
            mlflow.log_metric("r2", r2)
            mlflow.log_metric("mae", mae)

            # Model registry does not work with file store
            print("[DEBUG] Log model option flavor:", log_model_option["flavor"])
            if log_model_option["flavor"] == "_log_flavor":
                _ = shippedbrain._log_flavor("sklearn", sk_model = lr, signature = signature, input_example = log_model_option["input_example"], artifact_path="model")
            # INTEGRATION
            elif log_model_option["flavor"] == "upload_model" or log_model_option["flavor"] == "upload_run":
                flavor = log_model_option["flavor"]
                log_model_option.pop("flavor")
                log_func = eval(f"shippedbrain.{flavor}")
                _ = log_func(**log_model_option)
            elif log_model_option["flavor"] == "mlflow":
                mlflow.sklearn.log_model(lr, "model", signature=signature, input_example=log_model_option["input_example"])

            print(f"[INFO] Model URI runs:/{run.info.run_id}/model\n")

        return run

    else:
        if log_model_option["flavor"] == "_log_flavor":
            run = shippedbrain._log_flavor("sklearn",
                                           sk_model=lr,
                                           signature=signature,
                                           input_example=log_model_option["input_example"],
                                           artifact_path="model")
        # INTEGRATION
        elif log_model_option["flavor"] == "upload_model" or log_model_option["flavor"] == "upload_run":
            flavor = log_model_option["flavor"]
            log_model_option.pop("flavor")
            log_func = eval(f"shippedbrain.{flavor}")
            run = log_func(**log_model_option)

        print(f"[INFO] Model URI runs:/{run.info.run_id}/model\n")

        return run