"""
CUDA_VISIBLE_DEVICES=0 python training_agents/test_a2c.py --num-per-apartment 3 --max-num-edges 10 \
--max-episode-length 250 --batch_size 32 --obs_type partial --gamma 0.95 \
--lr 1e-4 --nb_episodes 100000 --save-interval 200 --simulator-type unity \
--base_net TF --log-interval 1 --long-log 50 --logging --base-port 8681 \
--num-processes 5 --teleport --executable_file ../../executable/linux_exec_v2.x86_64 \
--agent_type hrl_mcts --num_steps_mcts 24

NO TELEPORT
CUDA_VISIBLE_DEVICES=0 python test_a2c.py --num-per-apartment 3 --max-num-edges 10 \
--max-episode-length 250 --batch_size 32 --obs_type mcts --gamma 0.95 --lr 1e-4 \
--task_type find  --nb_episodes 100000 --save-interval 200 --simulator-type unity \
--base_net TF --log-interval 1 --long-log 50 --logging --base-port 8681 --num-processes 5 --teleports \
--executable_file /data/vision/torralba/frames/data_acquisition/SyntheticStories/MultiAgent/challenge/executables/exec_linux.05.29.x86_64 \
--agent_type hrl_mcts --num_steps_mcts 24

CUDA_VISIBLE_DEVICES=5 python test_a2c.py --num-per-apartment 3 --max-num-edges 10 \
--max-episode-length 250 --batch_size 32 --obs_type mcts --gamma 0.95 --lr 1e-4 \
--task_type find  --nb_episodes 100000 --save-interval 200 --simulator-type unity \
--base_net TF --log-interval 1 --long-log 50 --logging --base-port 8681 --num-processes 5 \
--agent_type hrl_mcts --num_steps_mcts 50 \
--load-model trained_models/env.virtualhome/\
task.full-numproc.5-obstype.mcts-sim.unity/taskset.full/agent.hrl_mcts_alice.False/\
mode.RL-algo.a2c-base.TF-gamma.0.95-cclose.0.0-cgoal.0.0-lr0.0001-bs.32/\
stepmcts.24-lep.250-teleport.True-gtgraph/4200.pt

# WITH ALICE
CUDA_VISIBLE_DEVICES=6 python test_a2c.py --num-per-apartment 3 --max-num-edges 10 \
--max-episode-length 250 --batch_size 32 --obs_type mcts --gamma 0.95 --lr 1e-4 \
--task_type find  --nb_episodes 100000 --save-interval 200 --simulator-type unity \
--base_net TF --log-interval 1 --long-log 50 --logging --base-port 8781 --num-processes 5 \
--agent_type hrl_mcts --num_steps_mcts 20 --use-alice --max-number-steps 50 \
--load-model trained_models/env.virtualhome/\
task.full-numproc.5-obstype.mcts-sim.unity/taskset.full/agent.hrl_mcts_alice.False/\
mode.RL-algo.a2c-base.TF-gamma.0.95-cclose.0.0-cgoal.0.0-lr0.0001-bs.32_finetuned/\
stepmcts.50-lep.250-teleport.False-gtgraph-forcepred/2000.pt

"""
import sys
import os
curr_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(f'{curr_dir}/..')

from envs.unity_environment import UnityEnvironment
import pdb
import pickle
import random
import copy
from agents import MCTS_agent, HRL_agent
from arguments import get_args
from algos.arena_mp2 import ArenaMP
from algos.a2c import A2C
from algos.a2c_mp import A2C as A2C_MP
from utils import utils_goals, utils_rl_agent
import ray

if __name__ == '__main__':
    args = get_args()
    #args.task = 'setup_table'
    #args.num_per_apartment = '50'
    if args.num_processes > 1:
        ray.init()

    args.mode = 'full'
    num_agents = 1
    args.dataset_path = 'dataset/train_env_set_help.pik'

    data = pickle.load(open(args.dataset_path, 'rb'))

    executable_args = {
            'file_name': args.executable_file,
            'x_display': 0,
            'no_graphics': True
    }

    with open(args.dataset_path, 'rb') as f:
        env_task_set = pickle.load(f)

    for env in env_task_set:
        g = env['init_graph']
        id2node = {node['id']: node['class_name'] for node in g['nodes']}
        cloth_ids = [node['id'] for node in g['nodes'] if node['class_name'] in ["clothespile"]]
        g['nodes'] = [node for node in g['nodes'] if node['id'] not in cloth_ids]
        g['edges'] = [edge for edge in g['edges'] if edge['from_id'] not in cloth_ids and edge['to_id'] not in cloth_ids]

    #env_task_set = [env_task_set[220]]
    if args.debug:
        # # debug 1: 1 predicate, 1 room
        # env_task_set = env_task_set[0]
        # single_goal = [x for x,y in env_task_set['task_goal'][0].items() if y > 0 and x.split('_')[0] in ['on', 'inside']][0]

        # if args.obs_type == 'mcts':
        #     env_task_set['init_rooms'] = ['kitchen']
        # env_task_set['task_goal'] = {0: {single_goal: 1}, 1: {single_goal: 1}}
        # env_task_set = [env_task_set]

        # debug 2: multiple predicates, 1 room
        # env_task_set0 = copy.deepcopy(env_task_set)
        # env_task_set = []
        # env_task = env_task_set0[0]
        # single_goals = [x for x,y in env_task_set0[0]['task_goal'][0].items() if y > 0 and x.split('_')[0] in ['on', 'inside']]
        # # pdb.set_trace()
        # if args.obs_type == 'mcts':
        #     env_task['init_rooms'] = ['kitchen']
        #
        # for single_goal in single_goals:
        #     env_task_new = copy.deepcopy(env_task)
        #     env_task_new['task_goal'] = {0: {single_goal: 1}, 1: {single_goal: 1}}
        #     env_task_set.append(env_task_new)
        #
        # print('# env_task for debug:', len(env_task_set))
        #
        # for env_task in env_task_set:
        #     print(env_task['task_name'], env_task['task_goal'][0])

        # One room, multi preds in same task
        env_task_set = [env_task_set[0]]

        # # debug 3: 1 predicate, multiple rooms
        # env_task_set0 = copy.deepcopy(env_task_set)
        # env_task_set = []
        # for env_task in env_task_set0:
        #   if env_task['task_name'] == 'setup_table':
        #     single_goal = [x for x, y in env_task['task_goal'][0].items() if y > 0 and x.split('_')[1] == 'plate']
        #     if len(single_goal) == 1:
        #       env_task_new = copy.deepcopy(env_task)
        #       env_task_new['task_goal'] = {0: {single_goal[0]: 1}, 1: {single_goal[0]: 1}}
        #       env_task_set.append(env_task_new)
        # print('# env_task for debug:', len(env_task_set))
    else:
        if args.task_set != 'full':
            env_task_set = [env_task for env_task in env_task_set if env_task['task_name'] == args.task_set]

    # env_task_set = [[env for env in env_task_set if env['env_id'] == 2][0]]
    print('Number of episides: {}'.format(len(env_task_set)))

    agent_goal = 'full'
    args.task_type = 'full'
    # if args.task_type == 'put':
    #     agent_goal = 'put'

    agent_goals = [agent_goal]
    if args.use_alice:
        num_agents += 1
        observation_types = ['mcts', args.obs_type]
        agent_goals.append(agent_goal)
        rl_agent_id = 2
    else:
        rl_agent_id = 1
        observation_types = [args.obs_type]

    def env_fn(env_id):
        if args.simulator_type == 'unity':

            return UnityEnvironment(num_agents=num_agents,
                                    max_episode_length=args.max_episode_length,
                                    port_id=env_id,
                                    env_task_set=env_task_set,
                                    agent_goals=agent_goals,
                                    observation_types=observation_types,
                                    use_editor=args.use_editor,
                                    executable_args=executable_args,
                                    base_port=args.base_port,
                                    seed=env_id)
        else:
            return PythonEnvironment(num_agents=num_agents, max_episode_length=args.max_episode_length,
                                    env_task_set=env_task_set,
                                    agent_goals=agent_goals,
                                    observation_types=observation_types,
                                    seed=env_id)

    graph_helper = utils_rl_agent.GraphHelper(max_num_objects=args.max_num_objects,
                                              max_num_edges=args.max_num_edges, current_task=None,
                                              simulator_type=args.simulator_type)

    def MCTS_agent_fn(arena_id, env):
        args_mcts = dict(recursive=False,
                           max_episode_length=5,
                           num_simulation=100,
                           max_rollout_steps=5,
                           c_init=0.1,
                           c_base=1000000,
                           num_samples=1,
                           num_processes=1,
                           logging=False,
                           logging_graphs=False)

        args_mcts['agent_id'] = 1
        args_mcts['char_index'] = 0
        return MCTS_agent(**args_mcts)

    def HRL_agent_fn(arena_id, env):
        args_agent2 = {'agent_id': rl_agent_id, 'char_index': rl_agent_id - 1,
                       'args': args, 'graph_helper': graph_helper}
        args_agent2['seed'] = arena_id
        return HRL_agent(**args_agent2)

    agents = [HRL_agent_fn]

    if args.use_alice:
        agents = [MCTS_agent_fn] + agents
    if args.num_processes > 1:
        ArenaMP = ray.remote(ArenaMP) #, max_reconstructions=ray.ray_constants.INFINITE_RECONSTRUCTION)
        arenas = [ArenaMP.remote(args.max_number_steps, arena_id, env_fn, agents) for arena_id in range(args.num_processes)]
        a2c = A2C_MP(arenas, graph_helper, args)
    else:
        arenas = [ArenaMP(args.max_number_steps, arena_id, env_fn, agents) for arena_id in range(args.num_processes)]
        a2c = A2C_MP(arenas, graph_helper, args)
    a2c.train()
    pdb.set_trace()


from flask import Flask, render_template, url_for
from forms import RegistrationForm, LoginForm
app = Flask(__name__)
app.config['SECRET_KEY'] = '0e575292378a0ca28abc0f97df278444'

articles = [
    {
        "id": 1,
        "author": "Andrew Maclean",
        "title": "Article One",
        "body": "What a wonderful man Enzo Ferrari was. He created a brand of cars."
    }, {
        "id": 2,
        "author": "John Doe",
        "title": "Article Two",
        "body": "What a terrible man Henry Ford was. He created an assembly line that made unappealing cars."
    }
]

@app.route('/')
@app.route('/home')
def dispHome():
	return render_template('home.html')

@app.route('/intro')
def dispIntro():
	return render_template('intro.html', title="Intro")

@app.route('/articles')
def dispArticles():
	return render_template('articles.html', articles=articles, title="Articles")

@app.route('/register', methods=['GET', 'POST'])
def dispRegister():
	form = RegistrationForm()
	return render_template('register.html', title='Register', form=form)

@app.route('/login', methods=['GET', 'POST'])
def dispLogin():
	form = LoginForm()
	return render_template('login.html', title='Login', form=form)

if __name__ == '__main__':
	app.run(debug=True)


# %%
import pandas as pd
import re
import os
from pathlib import Path
import requests
import time

# %%
file_path = Path("Resources/10K_10Q_List.csv")
df = pd.read_csv(file_path)
df.head()
# %%
reqd_Index = df[df['Form Type'] == '10-K'].index.tolist()
print(type(reqd_Index[0]))
# %%
headers = {'user-agent': 'project_10k/0.0.1'}
for index,filing in enumerate(df.Filename[242910:]):
    requestURL = filing
    filing_path = Path('./Resources/Filings/' + str(index+242910) + '.txt')
    response_data = requests.get(requestURL, headers = headers)
    file = open(filing_path, 'w+')
    file.write(response_data.text)
    file.close()
    file = open(filing_path)
    line = file.read().replace("\n", " ")
    file.close()
    result = re.search('<TYPE>10-K(.*?)</TEXT>', line)
    if result == None:
        result = re.search('<TYPE>10-Q(.*?)</TEXT>', line)
    result = re.sub("((?i)<TYPE>).*?(?=<)", "", result.group(1))
    result = re.sub("((?i)<SEQUENCE>).*?(?=<)", "", result)
    result = re.sub("((?i)<FILENAME>).*?(?=<)", "", result)
    result = re.sub("((?i)<DESCRIPTION>).*?(?=<)", "", result)
    result = re.sub("(?s)(?i).*?", "", result)
    result = re.sub("(?s)(?i)<(table).*?(
)", "", result)
    result = re.sub("(?s)(?i)(?m)> +Item|>Item|^Item", ">°Item", result)
    result= re.sub("(?s)<.*?>", " ", result)
    result = re.sub("&(.{2,6});", " ", result)
    result = re.sub("(?s) +", " ", result)
    result = re.sub("^.*?UNITED STATES SECURITIES AND EXCHANGE COMMISSION", "UNITED STATES SECURITIES AND EXCHANGE COMMISSION", result)
    output_path = Path('./Resources/Cleaned Filings/' + str(index+242910) + '.txt')
    file = open(output_path, 'w+')
    file.write(result)
    file.close()
    os.remove(filing_path)
# %%


#p341.Lab
from itertools import combinations
from itertools import product

n, m = map(int, input().split())
maps = []

#0:Empty, 1:Wall, 2:Virus
for i in range(n):
    maps.append(list(map(int, input().split())))

dx = [-1, 0, 1, 0]
dy = [0, 1, 0, -1]

max_safty = 0
walls = []
for x in range(n):
    for y in range(m):
        if maps[x][y] == 0:
            walls.append((x, y))
walls_3 = combinations(walls, 3)

def spread_virus(x, y):
    global n, m
    for d in range(4):
        nx = x+dx[d]
        ny = y+dy[d]
        #print(nx, ny)
        if nx > -1 and ny > -1 and nx < n and ny < m and maps[nx][ny] == 0:
            maps[nx][ny] = 12
            spread_virus(nx, ny)

#build wall x 3
for ws in walls_3:
    for wx, wy in ws:
        maps[wx][wy] = 11
    #find_virus
    for x in range(n):
        for y in range(m):
            if maps[x][y] == 2:
                #spread_virus
                spread_virus(x,y)

    cnt_safty = 0

    #get Number of Safty
    for x in range(n):
        for y in range(m):
            if maps[x][y] == 0:
                cnt_safty += 1
    if cnt_safty > max_safty: max_safty = cnt_safty

    #Roll Back
    for x in range(n):
        for y in range(m):
            if maps[x][y] == 11 or maps[x][y] == 12:
                maps[x][y] = 0

print(max_safty)


import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "2,3,4,5"
os.environ["XLA_PYTHON_CLIENT_ALLOCATOR"] = "platform"
os.environ["XLA_PYTHON_CLIENT_PREALLOCATE"] = "false"
os.environ["TF_GPU_THREAD_MODE"] = "gpu_private"
os.environ["TF_ENABLE_ONEDNN_OPTS"] = "0"

from logger import make_logger
logger = make_logger(__name__)

import tensorflow as tf
#tf.config.experimental.set_visible_devices([], 'GPU')
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
    try:# Currently, memory growth needs to be the same across GPUs
        for gpu in gpus:
            tf.config.experimental.set_memory_growth(gpu, True)
        logical_gpus = tf.config.experimental.list_logical_devices('GPU')
        logger.info(f'TF: {len(gpus)} Physical GPUs, {len(logical_gpus)} Logical GPUs')
    except RuntimeError as e: # Memory growth must be set before GPUs have been initialized
        print(e)
tf.get_logger().setLevel('ERROR')

import argparse
from copy import deepcopy

import jax
import jax.numpy as jnp
jnp.set_printoptions(threshold=2)
jax.config.update("jax_debug_nans", True)

import mlflow
from tqdm.auto import tqdm

import utils
from config import Config
from model.utils import make_model, make_model_info
from dataset import dataset
from artifact import Artifact
from updater import Updater, RunState

def main(opts):
    # config

    config = Config(opts, jax.process_index(), **{'setup:model': opts.model, 'setup:seed': opts.seed, 'setup:model_name': opts.model_name})
    local_device_count = jax.local_device_count()

    # random seed
    rng = jax.random.PRNGKey(config.setup.seed)
    rng = jnp.broadcast_to(rng, (local_device_count,) + rng.shape)
    utils.set_seed(config.setup.seed, logger = logger)

    make_mp_policy, initial_loss_scale = utils.make_mixed_precision(config)
    mp_policy = make_mp_policy()

    forward = make_model(config, mp_policy)

    datasets = {}
    data_length = 0

    for phase in config.phases:
        datasets[phase] = dataset.make_dataset(phase = phase, **config.dataset, dtype = mp_policy.compute_dtype)
        datasets[phase].n_step = 20
        data_length += datasets[phase].n_step

    artifact = Artifact(config, data_length = data_length)

    optimizer = utils.make_optimizer(config, datasets['train'].n_step)()
    loss_f = utils.make_loss(config)

    updater = Updater(forward, loss_f, optimizer, mp_policy, config.mp_nonfinite, initial_loss_scale, config.workdir, config.setup.n_class)
    grad_update = jax.pmap(updater.update_nominal, axis_name='batch', donate_argnums=(0,))
    eval_step   = jax.pmap(updater.eval_step,      axis_name='batch')
    eval_output = jax.pmap(updater.eval_output,    axis_name='batch')

    if config.mode.restore():
        run_state = jax.pmap(updater.restore_from_file)(rng)
    else:
        run_state = jax.pmap(updater.initial_state)(rng, next(datasets['train'].iter))

    model_info = make_model_info(run_state.params)
    #summary = hk.experimental.tabulate(train_step, columns = ['module', 'params_size', 'params_bytes'], tabulate_kwargs={'tablefmt':'pipe'})(train_state, batch)
    summary = None
    if config.process_idx == 0:
        config.print_summary({'experiment':config.experiment_name, 'run':config.run_name}, summary, datasets, model_info, config.phases, artifact.hparams)

    if config.mode.training():
        logger.info(f'starting training')
        logger.info(f'monitor started')

        # mlflow
        if config.process_idx == 0:
            mlflow.set_tracking_uri(f'{config.exp_dir}/mlruns')
            experiment = mlflow.set_experiment(experiment_name = config.experiment_name)
            mlflow.start_run(run_name = config.run_name)
            mlflow.log_param('seed', config.seed)
            mlflow.log_params(artifact.hparams)

        for epoch in range(1, config.setup.n_epochs+1):
            with tqdm(**artifact.pbar_args) as pbar:
                pbar.set_description(artifact.pbar_desc(epoch))
                monitor = utils.Monitor(0.1)#sec
                monitor.start_record()

                with artifact.timer('train', epoch):
                    for step in range(datasets['train'].n_step):
                        data = next(datasets['train'].iter)
                        run_state = grad_update(run_state, data)
                        if step % config.info_step == 0:
                            metrics = eval_step(run_state, data)
                            artifact.set_metrics('train', metrics = jax.tree_map(lambda x: x[0], metrics))
                            pbar.set_postfix_str(artifact.pbar_update())
                        pbar.update(1)

                monitor.stop_record()
                artifact.set_monitor('train', monitor.get_gpu_usage())

                monitor = utils.Monitor(1)#sec
                monitor.start_record()

                for _ in range(datasets['test'].n_step):
                    metrics = eval_step(run_state, next(datasets[phase].iter))
                    artifact.set_metrics('test', metrics = jax.tree_map(lambda x: x[0], metrics))
                    pbar.update(1)
                monitor.stop_record()
                artifact.set_monitor('test', monitor.get_gpu_usage())

                if artifact.end_of_epoch(epoch,  jax.tree_map(lambda x: x[0], run_state.params)):
                    c_params, c_state, c_opt_state, c_loss_scale, c_rng = deepcopy(run_state)
                    c_opt_state = jax.tree_map(lambda x: x[0], c_opt_state)
                    c_params = jax.tree_map(lambda x: x[0], c_params)
                    checkpoint_name = artifact.save_checkpoint(RunState(c_params, c_state, c_opt_state, c_loss_scale, c_rng))
                    mlflow.log_artifact(checkpoint_name)

                pbar.set_postfix_str(artifact.postfix)
                pbar.update(1)
                if config.process_idx == 0:
                    mlflow.log_metrics(artifact.get_metrics(), step = epoch)

        mlflow.end_run()
    artifact.close()

    if config.mode.eval():
        logger.info(f'starting evaluating')
        for _ in range(datasets['test'].n_step):
            data = next(datasets['test'].iter)
            logit = eval_output(run_state, data)

            artifact.add_x(jax.device_get(data['prop']),  'prop')
            artifact.add_x(jax.device_get(data['label']), 'label', shape = [data['label'].shape[0]*data['label'].shape[-1]] )
            artifact.add_x(jax.device_get(logit),         'logit')

        artifact.save_outputs('eval-outputs')

if __name__ == "__main__":
    #from distutils.util import strtobool
    parser = argparse.ArgumentParser( description = 'This is a script for running for model training.')

    parser.add_argument('-c',  '--config',       action = 'store', dest = 'config',          type = str, default = 'config/config.yaml')
    parser.add_argument('-s',  '--seed',         action = 'store', dest = 'seed',            type = int, default = None)
    parser.add_argument('-m',  '--model',        action = 'store', dest = 'model',           type = str, default = None)
    parser.add_argument('-md', '--mode',         action = 'store', dest = 'mode',            type = str, default = 'nominal')
    parser.add_argument('-mn', '--model_name',   action = 'store', dest = 'model_name',      type = str, default = 'nominal')
    parser.add_argument('-e',  '--experiment',   action = 'store', dest = 'experiment_name', type = str, default = None)
    parser.add_argument('-r',  '--run',          action = 'store', dest = 'run_name',        type = str, default = None)

    opts = parser.parse_args()

    main(opts)

    #test_checkpoint()


import pathlib

from aiogram import types
from aiogram.dispatcher import FSMContext
from aiogram.types import InputMediaPhoto, InputFile
from aiogram_datepicker import Datepicker, DatepickerSettings

from config import dp, keyboards
from config import bot
from config import roles
from states.form import StateUsers
from core.api_requests import ApiRequests as api
from config import const
from aiogram.utils.markdown import quote_html
from src import images
from app import dir_path

@dp.message_handler(commands=['help'])
async def command_start(message: types.Message, state: FSMContext):
    if message.from_user.id in roles.admins:
        await bot.send_document(message.from_user.id, InputFile(str(pathlib.Path(dir_path, 'src', 'documents',
                                                                                'user_guide.docx'))))
    else:
        message_from_user = 'Активирование бота со стороны клиента осуществляется при отправке команды «/start». ' \
                            'При нажатии на одну из кнопок клавиатура происходит переход бота в машинное состояние, ' \
                            'ожидающее определенное ' \
                            'действие от пользователя. Необходимо следовать указаниям бота, иначе он просто не будет ' \
                            'отвечать на ' \
                            'непредусмотренные действия.\nЕсли в браузерной версии пропала клавиатура, то вернуть '\
                            'её можно при помощи команды «/keyboard»'
        await bot.send_photo(message.from_user.id, InputFile(str(pathlib.Path(dir_path, 'src', 'images',
                                                                              'bot_states.png'))),
                             caption=message_from_user)
        message_from_user = 'Для простого понимания, машинное состояние представляет из себя меню, где кнопки ' \
                            'отвечают за переход в следующее меню. Кнопка «Отмена» является выходом из меню ( ' \
                            'машинного состояния), ' \
                            'а кнопка «Назад» является переходом в прошлое меню. '
        await bot.send_photo(message.from_user.id, InputFile(str(pathlib.Path(dir_path, 'src', 'images',
                                                                              'bot_states_buttons.png'))),
                             caption=message_from_user)
        message_from_user = '«/Магазин» - вывод подробной информации о магазине\n' \
                            '«/Каталог» - просмотр каталога, заполненный продавцом, добавление товара в корзину ' \
                            'соответствующей кнопкой. \n«/Корзина» - просмотр корзины, изменение количества товара и ' \
                            'удаление товаров из ' \
                            'корзины, формирование заказа. \nИзменение количества товара осуществляется по кнопкам ' \
                            '«+» и «-».\n'
        await bot.send_photo(message.from_user.id, InputFile(str(pathlib.Path(dir_path, 'src', 'images',
                                                                              'edit_basket.png'))),
                             caption=message_from_user)
        message_from_user = '«Мои_Заказы» - просмотр сформированных заказов и их статусов. Заказ также можно ' \
                            'отменить по соответствующей кнопке, но удалить можно только тот заказ, который не ' \
                            'оформлен продавцом.'
        await bot.send_photo(message.from_user.id, InputFile(str(pathlib.Path(dir_path, 'src', 'images',
                                                                              'order.png'))),
                             caption=message_from_user)


import torch
import torch.nn as nn
import torch.nn.functional as F

def _get_padding(kernel_size, nd):
    if isinstance(kernel_size, int):
        kernel_size = (kernel_size, )*nd
    padding = []
    for i in range(nd):
        padding += [(kernel_size[i]-1) // 2]
    return tuple(padding)

class _ConvNdBlock(nn.Module):

    def __init__(self,
        in_channels,
        out_channels,
        nd,
        kernel_size=3,
        depth=2,
        padding_mode='zeros',
        res_connection=True,
        activation=None,
        last_activation=True
    ):

        assert depth >= 1

        if nd == 2:
            conv = nn.Conv2d
        elif nd == 3:
            conv = nn.Conv3d
        else:
            raise ValueError(f'Invalid convolution dimensionality {nd}.')

        super().__init__()

        self.res_connection = res_connection
        if not activation:
            self.act = nn.ReLU()
        else:
            self.act = activation

        if last_activation:
            self.acts = [self.act] * depth
        else:
            self.acts = [self.act] * (depth-1) + [self._identity]

        padding = _get_padding(kernel_size, nd)
        self.convs = nn.ModuleList([conv(in_channels, out_channels, kernel_size=kernel_size, padding=padding, padding_mode=padding_mode)])
        for i in range(depth-1):
            self.convs.append(conv(out_channels, out_channels, kernel_size=kernel_size, padding=padding, padding_mode=padding_mode))
        if res_connection and in_channels != out_channels:
            self.res_conv = conv(in_channels, out_channels, kernel_size=1)
        else:
            self.res_conv = None

    def _identity(self, x):
        return x

    def forward(self, x_in):
        x = x_in
        for conv, act in zip(self.convs, self.acts):
            x = act(conv(x))
        if self.res_connection:
            if self.res_conv:
                x = x + self.res_conv(x_in)
            else:
                x = x + x_in
        return x

class Conv3dBlock(_ConvNdBlock):
    '''
    Pytorch 3D convolution block module.

    Arguments:
        in_channels: int. Number of channels entering the first convolution layer.
        out_channels: int. Number of output channels in each layer of the block.
        kernel_size: int or tuple. Size of convolution kernel.
        depth: int >= 1. Number of convolution layers in the block.
        padding_mode: str. Type of padding in each convolution layer. 'zeros', 'reflect', 'replicate' or 'circular'.
        res_connection: Boolean. Whether to use residual connection over the block (f(x) = h(x) + x).
                        If in_channels != out_channels, a 1x1x1 convolution is applied to the res connection
                        to make the channel numbers match.
        activation: torch.nn.Module. Activation function to use after every layer in block. If None,
                    defaults to ReLU.
        last_activation: Bool. Whether to apply the activation after the last conv layer (before res connection).
    '''
    def __init__(self,
        in_channels,
        out_channels,
        kernel_size=3,
        depth=2,
        padding_mode='zeros',
        res_connection=True,
        activation=None,
        last_activation=True
    ):
        super().__init__(in_channels, out_channels, 3, kernel_size, depth, padding_mode, res_connection, activation, last_activation)

class Conv2dBlock(_ConvNdBlock):
    '''
    Pytorch 2D convolution block module.

    Arguments:
        in_channels: int. Number of channels entering the first convolution layer.
        out_channels: int. Number of output channels in each layer of the block.
        kernel_size: int or tuple. Size of convolution kernel.
        depth: int >= 1. Number of convolution layers in the block.
        padding_mode: str. Type of padding in each convolution layer. 'zeros', 'reflect', 'replicate' or 'circular'.
        res_connection: Boolean. Whether to use residual connection over the block (f(x) = h(x) + x).
                        If in_channels != out_channels, a 1x1 convolution is applied to the res connection
                        to make the channel numbers match.
        activation: torch.nn.Module. Activation function to use after every layer in block. If None,
                    defaults to ReLU.
        last_activation: Bool. Whether to apply the activation after the last conv layer (before res connection).
    '''
    def __init__(self,
        in_channels,
        out_channels,
        kernel_size=3,
        depth=2,
        padding_mode='zeros',
        res_connection=True,
        activation=None,
        last_activation=True
    ):
        super().__init__(in_channels, out_channels, 2, kernel_size, depth, padding_mode, res_connection, activation, last_activation)

class _AttentionConvQueryNd(nn.Module):

    def __init__(self, in_channels, global_channels, query_features, kernel_size,
            padding_mode, activation, upsample_mode, nd):
        super().__init__()
        self.nd = nd
        if nd == 2:
            conv = nn.Conv2d
            self.sum_dims = [2,3]
        elif nd == 3:
            conv = nn.Conv3d
            self.sum_dims = [2,3,4]
        else:
            raise ValueError(f'Invalid convolution dimensionality {nd}.')
        padding = _get_padding(kernel_size, nd)
        self.a_conv = conv(in_channels, query_features, kernel_size=kernel_size,
            padding=padding, padding_mode=padding_mode)
        self.g_conv = conv(global_channels, in_channels, kernel_size=kernel_size,
            padding=padding, padding_mode=padding_mode)
        self.softmax = nn.Softmax(dim=1)
        self.sigmoid = nn.Sigmoid()
        self.activation = activation
        self.upsample_mode = upsample_mode

    def _unsqueeze_query(self, q):
        if self.nd == 2:
            q = q[:,:,None,None]
        else:
            q = q[:,:,None,None,None]
        return q

    def forward(self, x, q, g=None):

        if g is not None:
            # Add global features to x
            g = F.interpolate(g, size=x.size()[2:], mode=self.upsample_mode, align_corners=False)
            g = F.relu(self.g_conv(g))
            a = x+g
        else:
            a = x

        # Get attention map from input and query vector
        a = self.a_conv(a)                      # (batch, query_features, width, height, (depth))
        q = self.softmax(q)                     # (batch, query_features)
        q = self._unsqueeze_query(q)            # (batch, query_features, 1, 1, (1))
        a = torch.sum(a*q, dim=1, keepdim=True) # (batch, 1, width, height, (depth))

        # Apply activation to normalize attention map
        if self.activation == 'softmax':
            size = a.size()
            a = self.softmax(a.reshape(a.size(0), -1)).reshape(size)
        elif self.activation == 'sigmoid':
            a = self.sigmoid(a)
        else:
            raise ValueError(f'Unrecognized attention map activation {self.activation}.')

        # Get attention-gated features from x
        x = torch.sum(a*x, dim=self.sum_dims)

        return x, a.squeeze(dim=1)

class AttentionConvQuery2d(_AttentionConvQueryNd):
    '''
    Pytorch attention layer for 2D convolution with a query 1D vector.

    Arguments:
        in_channels: int. Number of channels in the attended feature map.
        global_channels: int. Number of channels in the global feature map.
        query_features: int. Number of features in query vector.
        kernel_size: int. Size of convolution kernel.
        padding_mode: str. Type of padding in each convolution layer. 'zeros', 'reflect', 'replicate' or 'circular'.
        activation: str. Type of activation to use for attention map. 'sigmoid' or 'softmax'.
        upsample_mode: str. Algorithm for upsampling global feature map to the attended
                       feature map size. See torch.nn.functional.interpolate.
    '''
    def __init__(self, in_channels, global_channels, query_features, kernel_size,
            padding_mode='zeros', activation='softmax', upsample_mode='bilinear'):
        super().__init__(in_channels, global_channels, query_features, kernel_size,
            padding_mode, activation, upsample_mode, 2)

class AttentionConvQuery3d(_AttentionConvQueryNd):
    '''
    Pytorch attention layer for 3D convolution with a query 1D vector.

    Arguments:
        in_channels: int. Number of channels in the attended feature map.
        global_channels: int. Number of channels in the global feature map.
        query_features: int. Number of features in query vector.
        kernel_size: int. Size of convolution kernel.
        padding_mode: str. Type of padding in each convolution layer. 'zeros', 'reflect', 'replicate' or 'circular'.
        activation: str. Type of activation to use for attention map. 'sigmoid' or 'softmax'.
        upsample_mode: str. Algorithm for upsampling global feature map to the attended
                       feature map size. See torch.nn.functional.interpolate.
    '''
    def __init__(self, in_channels, global_channels, query_features, kernel_size,
            padding_mode='zeros', activation='softmax', upsample_mode='trilinear'):
        super().__init__(in_channels, global_channels, query_features, kernel_size,
            padding_mode, activation, upsample_mode, 3)

class UNetAttentionConv(nn.Module):
    '''
    Pytorch attention layer for U-net model upsampling stage.

    Arguments:
        in_channels: int. Number of channels in the attended feature map.
        query_channels: int. Number of channels in query feature map.
        attention_channels: int. Number of channels in hidden convolution layer before computing attention.
        kernel_size: int. Size of convolution kernel.
        padding_mode: str. Type of padding in each convolution layer. 'zeros', 'reflect', 'replicate' or 'circular'.
        conv_activation: nn.Module. Activation function to use after convolution layers
        attention_activation: str. Type of activation to use for attention map. 'sigmoid' or 'softmax'.
        upsample_mode: str. Algorithm for upsampling query feature map to the attended
            feature map size. See torch.nn.functional.interpolate.
        ndim: 2 or 3. Dimensionality of convolution.
    References:
        https://arxiv.org/abs/1804.03999
    '''
    def __init__(self,
        in_channels,
        query_channels,
        attention_channels,
        kernel_size,
        padding_mode='zeros',
        conv_activation=nn.ReLU(),
        attention_activation='softmax',
        upsample_mode='bilinear',
        ndim=2
    ):
        super().__init__()

        self.ndim = ndim
        if ndim == 2:
            conv = nn.Conv2d
        elif ndim == 3:
            conv = nn.Conv3d
        else:
            raise ValueError(f'Invalid convolution dimensionality {ndim}.')

        if attention_activation == 'softmax':
            self.attention_activation = self._softmax
        elif attention_activation == 'sigmoid':
            self.attention_activation = self._sigmoid
        else:
            raise ValueError(f'Unrecognized attention map activation {attention_activation}.')

        padding = _get_padding(kernel_size, ndim)
        self.x_conv = conv(in_channels, attention_channels, kernel_size=kernel_size,
            padding=padding, padding_mode=padding_mode)
        self.q_conv = conv(query_channels, attention_channels, kernel_size=kernel_size,
            padding=padding, padding_mode=padding_mode)
        self.a_conv = conv(attention_channels, 1, kernel_size=kernel_size,
            padding=padding, padding_mode=padding_mode)
        self.softmax = nn.Softmax(dim=1)
        self.sigmoid = nn.Sigmoid()
        self.upsample_mode = upsample_mode
        self.conv_activation = conv_activation

    def _softmax(self, a):
        shape = a.shape
        return self.softmax(a.reshape(shape[0], -1)).reshape(shape)

    def _sigmoid(self, a):
        return self.sigmoid(a)

    def forward(self, x, q):

        # Upsample query q to the size of input x and convolve
        q = F.interpolate(q, size=x.size()[2:], mode=self.upsample_mode, align_corners=False)
        q = self.conv_activation(self.q_conv(q))

        # Convolve input x and sum with q
        a = self.conv_activation(self.x_conv(x))
        a = self.conv_activation(a + q)

        # Get attention map and mix it with x
        a = self.attention_activation(self.a_conv(a))
        x = a * x

        return x, a.squeeze(dim=1)


# Import section
import requests
import argparse
import re
import os
import json
from jsonschema import validate, ValidationError, SchemaError

# Response schema
response_schema = {
    "type": "object",
    "properties":{
        "vendorDetails": {
            "type": "object",
            "properties": {
                "companyName": {
                    "type": ["string"],
                    "minLength": 1
                }
            },
            "required": ["companyName"]
        }
    },
    "required": ['vendorDetails']
}

# Init section
REQUEST_URL = "https://api.macaddress.io/v1?output=json&search={}"
API_TOKEN = os.getenv("API_TOKEN","")

# Initialize the parser
parser = argparse.ArgumentParser()
parser.add_argument("-a","--address",help="MAC Address",required=True)
parser.add_argument("-o","--output",choices=["string","json","xml"],default="string",help="Output format")

# Parsing the args
args = parser.parse_args()

# Checking the MAC address format using REGEX
p = re.compile(r'(?:[0-9a-fA-F]:?){12}')
matches = re.findall(p,args.address)
if not matches:
    print("\n  >> [ERROR]: Incorrect MAC address\n")
    exit()

# Sending request
headers = {"X-Authentication-Token": API_TOKEN}
response = requests.get(REQUEST_URL.format(args.address),headers=headers)
if response.ok == False:
    print("\n  >> [ERROR]: Request failed. Status code={}, Error={}\n".format(response.status_code,response.text))
    exit()

# Processing the response
result = response.json()

try:
    validate(result,schema=response_schema)
except (ValidationError, SchemaError) as e:
    print("\n  >> [ERROR]: Incorrect response: {}\n".format(e.message))
    exit()

if args.output == "json":
    print(json.dumps({
        "name": result['vendorDetails']['companyName']
    }))
elif args.output == "xml":
    print('<?xml version="1.0" encoding="UTF-8"?><company>{}</company>'.format(result['vendorDetails']['companyName']))
else:
    print(result['vendorDetails']['companyName'])


import numpy as np
import pandas as pd
import datetime as dt
from pandas.plotting import table

from flask import Flask, jsonify

# Python SQL toolkit and Object Relational Mapper
import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func

# create engine to hawaii.sqlite
engine = create_engine("sqlite:///hawaii.sqlite")

# reflect an existing database into a new model
Base = automap_base()

# reflect the tables
Base.prepare(engine, reflect=True)

# Save references to each table
measurementTable = Base.classes.measurement
stationTable = Base.classes.station

# Create our session (link) from Python to the DB
session = Session(engine)

app = Flask(__name__)

@app.route("/")
def indexRoute():
    routes = ['/', '/api/v1.0/precipitation', '/api/v1.0/stations', '/api/v1.0/tobs', '/api/v1.0/<start>', '/api/v1.0/<start>/<end>']
    return jsonify(routes = routes)

@app.route("/api/v1.0/precipitation")
def getPrecipitation():
    # Calculate the date one year from the last date in data set.
    timeDiff = dt.date(2017,8,23) - dt.timedelta(days = 365)

    # Perform a query to retrieve the data and precipitation scores
    datas = session.query(measurementTable.date, measurementTable.prcp).filter(measurementTable.date >= timeDiff).all()

    session.close()

    resDict = {}
    print(datas)
    for data in datas:
        date = data[0]
        precipitation = data[1]
        resDict[date] = precipitation
    return jsonify(resDict)

@app.route("/api/v1.0/stations")
def getStations():
    datas = session.query(stationTable.station).all()
    session.close()

    res = []
    for data in datas:
        res.append(data[0])
    return jsonify(res = res)

@app.route("/api/v1.0/tobs")
def getTobs():
    # Calculate the date one year from the last date in data set.
    timeDiff = dt.date(2017,8,23) - dt.timedelta(days = 365)

    # Using the most active station id
    # Query the last 12 months of temperature observation data for this station and plot the results as a histogram

    lastYearTempObservation = session.query(measurementTable.tobs).filter(measurementTable.date >= timeDiff).filter(measurementTable.station == 'USC00519281').all()

    session.close()

    res = []
    for data in lastYearTempObservation:
        res.append(data[0])
    return jsonify(res = res)

@app.route("/api/v1.0/<start>")
@app.route("/api/v1.0/<start>/<end>")
def getMinAvgMaxTemperatureList(start = None, end = None):

    if end is None:
        start = dt.datetime.strptime(start, "%m%d%Y")
        datas = session.query(func.min(measurementTable.tobs), func.max(measurementTable.tobs), func.avg(measurementTable.tobs)).filter(measurementTable.date >= start).all()
        session.close()
        res = []
        for data in datas:
            res.append(data[0])
        return jsonify(res = res)
    else:
        start = dt.datetime.strptime(start, "%m%d%Y")
        end = dt.datetime.strptime(end, "%m%d%Y")

        datas = session.query(func.min(measurementTable.tobs), func.max(measurementTable.tobs), func.avg(measurementTable.tobs)).filter(measurementTable.date >= start).filter(measurementTable.date <= end).all()
        session.close()

        res = []
        for data in datas:
            res.append(data[0])
        return jsonify(res = res)

if __name__ == '__main__':
	app.run(debug=True)


import os
from datetime import datetime

import cherrypy
from mako.lookup import TemplateLookup
from mako import exceptions

from . import logger
from .CameraControl import CameraControl

class CameraWeb(object):

    def __init__(self, ptzcontroller):
        self.ptzcontroller = ptzcontroller
        self.template_dir = os.path.join(str(ptzcontroller.PROG_DIR), 'html/')
        self._hplookup = TemplateLookup(directories=[self.template_dir], default_filters=['unicode', 'h'])
        self.http_root = '/'
        self.server_name = 'PTZController'
        self.cache_param = '?V1.0.0'
        self.cameraControl = CameraControl(ptzcontroller)

    @cherrypy.expose
    def index(self):
        return self.serve_template(templatename="index.html", title="Home", cameras=self.ptzcontroller.cameras)

    @cherrypy.expose
    def OBSDock(self):
        return self.serve_template(templatename="OBSDock.html")

    @cherrypy.expose
    def get_status(self):
        x = self.cameraControl.get_status()
        return self.serve_template(templatename="index.html", title="Home", cameras=self.ptzcontroller.cameras)

    def serve_template(self, templatename, **kwargs):
        cache_param = self.cache_param + '.' +  datetime.now().strftime("%Y-%m-%d-%H:%M:%S")
        try:
            template = self._hplookup.get_template(templatename)
            return template.render(http_root=self.http_root,
                                   server_name=self.server_name,
                                   cache_param=cache_param,
                                   **kwargs)
        except:
            return exceptions.html_error_template().render()


import requests

# Define the URL
url = "http://127.0.0.1:8000/scrape_and_store"

# Define the payload (data to send)
data = {"url": "https://en.wikipedia.org/wiki/Emmerson_Nogueira"}

# Make the POST request
response = requests.post(url, json=data)

# Print the response
print(response.json())

# curl -X POST -H "Content-Type: application/json" -d '{"url":"https://en.wikipedia.org/wiki/Emmerson_Nogueira"}' http://127.0.0.1:8000/scrape_and_store


import sys
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Polygon

n = int(sys.stdin.readline())
x = []
y = []
color = []

cdict = { '0' : 'y',
          '1' : 'g',
          '2' : 'b',
          '3' : 'r',
          '4' : 'c'
          }

for line in range(0,n):
    line = sys.stdin.readline()
    data = line.split()
    x.append(float(data[0]))
    y.append(float(data[1]))
    color.append(cdict[data[2]])

plt.scatter(x,y,c=color)
plt.show()


import codecs
import os
import re
from setuptools import setup
from setuptools import find_packages

def find_version(*file_paths):
    here = os.path.abspath(os.path.dirname(__file__))
    with codecs.open(os.path.join(here, *file_paths), 'r') as f:
        version_file = f.read()
    version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
                              version_file, re.M)
    if version_match:
        return version_match.group(1)
    raise RuntimeError("Unable to find version string.")

setup(
    name="jans-pycloudlib",
    version=find_version("jans", "pycloudlib", "version.py"),
    url="",
    license="Apache",
    author="Janssen",
    author_email="isman@gluu.org",
    description="",
    long_description=__doc__,
    packages=find_packages(),
    zip_safe=False,
    install_requires=[
        "requests>=2.22.0",
        "python-consul>=1.0.1",
        "hvac>=0.7.0",
        "kubernetes>=11.0",
        "ldap3>=2.5",
        "backoff>=2.1.2",
        "docker>=3.7.2",
        "requests-toolbelt>=0.9.1",
        "cryptography>=2.8",
        "google-cloud-secret-manager>=2.2.0",
        "pymysql>=1.0.2",
        "sqlalchemy>=1.3,<1.4",
        "psycopg2>=2.8.6",
        "google-cloud-spanner>=3.3.0",
        "Click>=6.7",
        "ldif>=4.1.1",
        # handle CVE-2022-36087
        "oauthlib>=3.2.1",
        "boto3",
    ],
    classifiers=[
        "Intended Audience :: Developers",
        "License :: OSI Approved :: MIT License",
        "Topic :: Software Development :: Libraries :: Python Modules",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.9",
        "Programming Language :: Python :: 3.10",
    ],
    include_package_data=True,
    entry_points={
        "console_scripts": ["jans-pycloudlib=jans.pycloudlib.cli:cli"],
    },
    package_data={
        "jans.pycloudlib": ["py.typed"],
    },
    python_requires=">=3.9",
)


# -*- coding: utf-8 -*-
import requests
import pandas as pd
url = "https://iftp.chinamoney.com.cn/ags/ms/cm-u-bond-md/BondMarketInfoListEN"
rows = []
for i in range(1,5):
    params = {'bondType': '100001', 'issueYear': '2023', 'pageNo': str(i),'pageSize': '15'}
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'
    }
    response = requests.post(url, params=params,headers=headers)
    data = response.json()
    # 解析返回的表格数据
    table_data = data["data"]["resultList"]
    #print(len(table_data))
    columns = ["isin","bondCode","entyFullName","bondType","issueStartDate","debtRtng"]

    for record in table_data:
        row = [record[col] for col in columns]
        rows.append(row)
new_colum = ["ISIN", "Bond Code", "Issuer", "Bond Type", "Issue Date", "Latest Rating"]

df = pd.DataFrame(rows, columns=new_colum)
df.to_csv("bond_data.csv", index=False)


from argparse import ArgumentParser
import matplotlib
from matplotlib import pyplot as plt
matplotlib.rcParams['pdf.fonttype'] = 42 # Editable text when exporting pdfs
import numpy as np
import pandas as pd
import scipy as sp
from sklearn.decomposition import PCA
from sklearn.linear_model import LogisticRegressionCV
from sklearn.metrics import confusion_matrix

#%% Normalize count matrix using arcsinh transformation. Assumes each row is a sample/cell and columns correspond to genes
def normalize_arcsinh(count_matrix,cofactor = 1000):
    total_counts = np.squeeze(np.sum(count_matrix,axis = 1))
    normalized_counts = np.arcsinh(count_matrix / total_counts[:,np.newaxis] * cofactor)
    return(normalized_counts)

#%% Train a classifer using training_object generated by create_training_data
def train_LR( training_data, training_labels, num_genes = 1024, pct_var = 0.90, save_results = True ):
    """
    Description
    -----------
    Trains logistic regression classification model.
        1) Select subset of features based on standard deviation
        2) Run PCA using only selected features
        3) Use PCA components as inputs into logistic regression classifier
    Calls sklearn logistic regression, which has cross-validation built-in

    Parameters
    ----------
    training_data : numpy array
        contains count data
    training_labels : numpy array
        cell type labels corresponding to each row of training data
    num_genes : int
        number of genes to select
    pct_var : int
        percentage of variance to explain by PCA. Determines number of components
        used as inputs to logistic regression
    save_results : boolean
        specify whether to save cross validation results
    See individual arguments

    Returns
    -------
    a dictionary containing handles to functions for each step in the classification
    procedure (i.e. normalization, feature selection, pca transformation, and prediction)
    """

    normalized_expression = normalize_arcsinh(training_data)
    # Select subset of genes to use based on standard deviation
    std_dev = np.std( normalized_expression, axis = 0)
    sorted_idx = np.argsort(std_dev)
    # Train on PCA componenets rather than gene expression
    pca = PCA(n_components = pct_var)
    training_pca_coordinates = pca.fit_transform( normalized_expression[:,sorted_idx[-num_genes:]] )
    # Fit logisitc regression model and cross validate
    LR = LogisticRegressionCV( penalty = "l2", Cs = np.logspace(-7,1,9), dual = False, class_weight = "balanced", multi_class = "ovr", solver = "sag")
    LR.fit( training_pca_coordinates, training_labels )

    # Save cross validation results
    if save_results:
        directory = "results/classification/training_results/"
        parameters = str(num_genes) + "_" + str(int(pct_var*100))
        for ct in np.unique( training_labels ):
            pd.DataFrame( LR.scores_[ct], columns=[f'{x:.20f}' for x in LR.Cs_]).to_csv(
                         directory + ct + "_" + parameters + ".csv")
        confusion_mat = pd.DataFrame(confusion_matrix( training_labels, LR.predict( training_pca_coordinates )),
                                     index = np.unique(training_labels), columns = np.unique(training_labels))
        confusion_mat.to_csv(directory + "confusion_matrix" + "_" + parameters + ".csv")

    def feature_selection(X):
        return(X[:,sorted_idx[-num_genes:]])
    # Pack functions for classification into dictionary
    return {'normalization': normalize_arcsinh,
            'feature_selection': feature_selection,
            'pca_function': pca.transform,
            'prediction_function': LR.predict,
            'prediction_probability': LR.predict_proba}
    #    return(LR)
#%%
def _argparse():
    """
    Description
    -----------
    Parse command line arguments
    Parameters
    ----------
    See individual arguments
    Returns
    -------
    """
    parser = ArgumentParser('Train classifier using given parameters')
    parser.add_argument("training_data", help = "sparse matrix (.npz containing training data)",
                        type=str,default="")
    parser.add_argument("training_labels", help = ".csv file containing training labels (cell types)",
                        type=str,default="")
    #parser.add_argument("gene_names", help = "file containing gene names",
    #                    type=str,default="")
    parser.add_argument("-k","--k_fold",
                        help = "Number of partitions to use for k-fold cross validation"
                        ,type=int,default=5)
    parser.add_argument("-n","--num_genes",
                        help = "Only use the n highest genes by standard deviation"
                        ,type=int ,default=1024)
    parser.add_argument("-pca","--pca_variance",
                        help = "% of PCA variance to keep when defining training inputs"
                        ,type=float  ,default=0.90)
    return parser

#%% Compute expression thresholds for each marker using otsu's method
#   Assumes two groups (high expressing and low expressing)
def main(args):
    """
    Description
    -----------
    Predict cell-types of each cell in the single cell expression data using
    pre-defined marker genes

    Parameters
    ----------
    see _argparse

    Returns
    -------
    No return value.
    train_LR function will save cross-validation results if specified.
    For each cell type in the training labels, train_LR will write accuracy
    scores over a range of regularization parameters. Will also save a confusion matrix
    """
    # Parse inputs
    parser = _argparse()
    argp = parser.parse_args(args[1:])
    # Read data
    sparse_training = sp.sparse.load_npz( argp.training_data )
    training_counts = np.array( sparse_training.todense() )
    training_labels = np.squeeze(np.array(pd.read_csv("results/classification/training_labels.csv",index_col = 0 )))

    train_LR( training_counts, training_labels, num_genes = argp.num_genes, pct_var = argp.pca_variance/100 )

if __name__ == '__main__':
    from sys import argv
    exit(main(argv))


import os.path
import requests
from configparser import ConfigParser
from pathlib import Path
from time import sleep
# Third party
import click
from hangoutsclient import HangoutsClient
from lxml import etree

APP_NAME = 'wynweather'

def create_dir(ctx, param, directory):
    if not os.path.isdir(directory):
        os.makedirs(directory, exist_ok=True)
    return directory

@click.command()
@click.argument('bom_url')
@click.argument('search_string')
@click.argument('notify_user')
@click.option(
    '--config-path',
    type=click.Path(),
    default=os.path.join(os.environ.get('XDG_CONFIG_HOME', os.path.expanduser('~/.config')), APP_NAME),
    callback=create_dir,
    help='Path to directory containing config file. Defaults to XDG config dir.',
)
@click.option(
    '--cache-path',
    type=click.Path(),
    default=os.path.join(os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache')), APP_NAME),
    callback=create_dir,
    help='Path to directory to store logs and such. Defaults to XDG cache dir.',
)
def main(config_path, cache_path, bom_url, search_string, notify_user):
    """
    This script scrapes the weather warning feed given by BOM_URL, and if there are any current warning(s) that
    contain the keyword SEARCH_STRING, a message is sent to NOTIFY_USER using Google Hangouts.

    bom_url - URL for the BOM weather warning feed. See http://www.bom.gov.au/rss/.

    search_string - keyword used to trigger message being sent. e.g. "Hawthorn"

    notify_user - JID or full name of the Hangouts user to notify. e.g. "xxx@public.talk.google.com" or "Itiot Gimp"
    """
    config_file = os.path.join(config_path, 'config.ini')
    config = ConfigParser()
    config.read(config_file)
    client_id = config.get('Hangouts', 'client_id')
    client_secret = config.get('Hangouts', 'client_secret')
    token_file = os.path.join(cache_path, 'hangouts_cached_token')
    if not os.path.isfile(token_file):
        Path(token_file).touch()

    message = ''

    response = requests.get(bom_url)
    if response.status_code == 200:
        tree = etree.fromstring(response.content)
        warning_titles = tree.xpath("//item/title/text()")
        warning_links = tree.xpath("//item/link/text()")
    else:
        response.raise_for_status()

    for title, link in zip(warning_titles, warning_links):
        response = requests.get(link)
        if response.status_code == 200:
            if search_string in response.text:
                message = f"Watch out wyn! ({title}: {link})"
        else:
            response.raise_for_status()

    if message:
        hangouts = HangoutsClient(client_id, client_secret, token_file)
        if hangouts.connect():
            hangouts.process(block=False)
        else:
            raise RuntimeError('Unable to connect to Hangouts.')
        sleep(5)  # need time for Hangouts roster to update
        hangouts.send_to([notify_user, ], message)
        hangouts.disconnect(wait=True)

if __name__ == '__main__':
    main()


"""Tests for the GQCISD model.
"""

import unittest
import pytest

import numpy as np
from pyscf import cc, gto, lib, scf
from pyscf.cc import qcisd as pyscf_qcisd
from pyscf.cc import ccsd_rdm

from ebcc import GEBCC, NullLogger, Space, util

@pytest.mark.reference
class GQCISD_PySCF_Tests(unittest.TestCase):
    """Test GQCISD against PySCF's RQCISD values.
    """

    @classmethod
    def setUpClass(cls):
        mol = gto.Mole()
        mol.atom = "Li 0 0 0; H 0 0 1.4"
        mol.basis = "cc-pvdz"
        mol.verbose = 0
        mol.build()

        mf = scf.RHF(mol)
        mf.conv_tol = 1e-12
        mf.kernel()

        qcisd_ref = pyscf_qcisd.QCISD(mf)
        qcisd_ref.conv_tol = 1e-10
        qcisd_ref.conv_tol_normt = 1e-14
        qcisd_ref.max_cycle = 200
        qcisd_ref.kernel()

        qcisd = GEBCC(
                mf.to_ghf(),
                ansatz="QCISD",
                log=NullLogger(),
        )
        qcisd.options.e_tol = 1e-10
        eris = qcisd.get_eris()
        qcisd.kernel(eris=eris)

        cls.mf, cls.qcisd_ref, cls.qcisd, cls.eris = mf, qcisd_ref, qcisd, eris

    @classmethod
    def tearDownClass(cls):
        del cls.mf, cls.qcisd_ref, cls.qcisd

    def test_converged(self):
        self.assertTrue(self.qcisd.converged)
        self.assertTrue(self.qcisd_ref.converged)

    def test_energy(self):
        a = self.qcisd_ref.e_tot
        b = self.qcisd.e_tot
        self.assertAlmostEqual(a, b, 7)

@pytest.mark.regression
class GQCISD_Tests(unittest.TestCase):
    """Test GQCISD against regression.
    """

    @classmethod
    def setUpClass(cls):
        mol = gto.Mole()
        mol.atom = "Li 0 0 0; H 0 0 1.4"
        mol.basis = "cc-pvdz"
        mol.verbose = 0
        mol.build()

        mf = scf.RHF(mol)
        mf.conv_tol = 1e-12
        mf.kernel()

        qcisd = GEBCC(
                mf.to_ghf(),
                ansatz="QCISD",
                log=NullLogger(),
        )
        qcisd.options.e_tol = 1e-10
        eris = qcisd.get_eris()
        qcisd.kernel(eris=eris)

        cls.mf, cls.qcisd, cls.eris = mf, qcisd, eris

    @classmethod
    def tearDownClass(cls):
        del cls.mf, cls.qcisd, cls.eris

    def test_converged(self):
        self.assertTrue(self.qcisd.converged)

    def test_t1(self):
        ci = self.qcisd.mo_coeff[:, self.qcisd.mo_occ > 0]
        ca = self.qcisd.mo_coeff[:, self.qcisd.mo_occ == 0]
        t1 = util.einsum("ia,pi,qa->pq", self.qcisd.t1, ci, ca)
        self.assertAlmostEqual(lib.fp(t1), 0.002669284765052053)

    def test_t2(self):
        ci = self.qcisd.mo_coeff[:, self.qcisd.mo_occ > 0]
        ca = self.qcisd.mo_coeff[:, self.qcisd.mo_occ == 0]
        t2 = util.einsum("ijab,pi,qj,ra,sb->pqrs", self.qcisd.t2, ci, ci, ca, ca)
        self.assertAlmostEqual(lib.fp(t2), -0.0008836462499253184)

if __name__ == "__main__":
    print("Tests for GQCISD")
    unittest.main()


import os.path as osp
import string

import pandas as pd
from sklearn.metrics import precision_recall_fscore_support, accuracy_score, \
    roc_auc_score, confusion_matrix

import const
from arguments import args
from setup import project_setup
from utils.utils_misc import map_prediction_to_binary_MedAlpaca, get_model_prefix

def map_prediction_to_binary(x, return_string=True, return_unknown=False):
    """Map the predictions to binary (Yes/No/Unknown).

    GPT models tend to generate more verbose contents than we instructed them to do.
    This function maps their verbose answers to categorical outputs.

    Args:
        x (str): The prediction string.
        return_string (bool): Whether to return the result as a string.
        return_unknown (bool): Whether to return 'Unknown' when result is ambiguous.

    Returns:
        str or int: Mapped prediction in either string or integer format.
    """

    ANSWER_MAP = {
        "Yes": {"Yes"'Entailment', 'Sí', 'Yes', 'जी हाँ', 'निश्चित रूप से हाँ।', 'हाँ', '可能性很大', '可能是正确的',
                '对',
                '是的', '正确'
                },
        "No": {'No', 'Not entailment', "I'm sorry, but i am unable to understand",
               "I'm sorry, but the response you provided is not related to the question",
               "I'm sorry, but your response does not seem to be related to the question asked", 'नहीं', 'नो',
               'माफ़ कीजिए, लेकिन यह प्रतिक्रिया दिए गए प्रश्न से मेल नहीं खाती है',
               'यह उत्तर दिया गया प्रश्न संबंधित नहीं है। कृपया सही प्रश्न पूछें',
               'यह उत्तर दिया गया प्रश्न से मेल नहीं खाता है', 'यह उत्तर नहीं है',
               'यह उत्तर प्रश्न के साथ संबंधित नहीं है। कृपया पुनः प्रश्न देखें', '不是', '不正确', '以上都不', '错误'},

        "Unknown": {'Unknown', 'Depends', 'N/a', 'कारण पता नहीं है', '不确定', '无法判断'}
    }

    if not isinstance(x, str):
        return -1 if not return_string else "Unknown" if return_unknown else x

    x_lower = x.lower()

    if any(prefix in x_lower for prefix in map(str.lower, ANSWER_MAP["No"])) and "yes" not in x_lower:
        return "No" if return_string else 0

    if any(prefix in x_lower for prefix in map(str.lower, ANSWER_MAP["Yes"])) and "no" not in x_lower:
        return "Yes" if return_string else 1

    if any(prefix in x_lower for prefix in map(str.lower, ANSWER_MAP["Unknown"])):
        return "Unknown" if return_string else -1

    return -1 if not return_string else ("Unknown" if return_unknown else x)

def map_prediction_to_binary_med_alpaca(x, return_string=True, return_unknown=False):
    """Map the predictions of MedAlpaca to binary (Yes/No/Unknown).

    MedAlpaca's answers are significantly different from GPT-3.5's,
    requiring a different mapping format.

    Args:
        x (str): The prediction string.
        return_string (bool): Whether to return the result as a string.
        return_unknown (bool): Whether to return 'Unknown' when result is ambiguous.

    Returns:
        str or int: Mapped prediction in either string or integer format.
    """

    ANSWER_MAP = {
        'Yes': {'Yes', 'Entailment'},
        'No': {'No', 'Not entailment'},
        'Unknown': {'Unknown', 'Depends', 'N/a'}
    }

    if isinstance(x, str):
        x = x.split('### Input')[-1].split('Question')[-1].split('Response')[-1]

    if not isinstance(x, str):
        return -1 if not return_string else 'Unknown' if return_unknown else x

    if all(any(segment in x for segment in ANSWER_MAP[key]) for key in ['Yes', 'No']):
        return 'Unknown' if return_string else -1

    for key, value in ANSWER_MAP.items():
        if any(segment in x for segment in value):
            return key if return_string else 1 if key == 'Yes' else 0 if key == 'No' else -1

    return -1 if not return_string else ('Unknown' if return_unknown else x)

def plot_confusion_matrix(df: pd.DataFrame, language: str, pos_label: int,
                          args):
    import matplotlib.pyplot as plt
    import seaborn as sns
    # Compute confusion matrix

    # If labels is [1, 0], we calculate the confusion matrix for class 1 as the positive label
    cm = confusion_matrix(df[const.LABEL], df[const.PRED_BINARY],
                          labels=[1, 0] if pos_label == 1 else [0, 1])

    # Create a dataframe from the confusion matrix.
    cm_df = pd.DataFrame(cm,
                         index=['Actual Positive', 'Actual Negative'],
                         columns=['Predicted Positive', 'Predicted Negative'])

    plt.figure(figsize=(10, 7))
    sns.set(font_scale=1.6)

    sns.heatmap(cm_df, annot=True, fmt='d', cmap='Blues')

    if args.dataset_name in ["healthqa", "mlecqa"]:
        name = f"{args.dataset_name}_{args.split}_{language}_pos-label={pos_label}"
        title = f"{args.dataset_name}, Lang:{language}, Split:{args.split}, Positive Label:{pos_label}"

    else:
        name = f"{args.dataset_name}_{language}_pos-label={pos_label}"
        title = f"{args.dataset_name}, Lang:{language}, Positive Label:{pos_label}"

    plt.title(f'{title}')
    plt.ylabel('Actual')
    plt.xlabel('Predicted')
    plt.show()

def assign_binary_label_verifiability(x):
    if not isinstance(x, str):
        return x
    if "not entailment" in x.lower():
        return "Not entailment"
    elif "entailment" in x:
        return "Entailment"
    else:
        return x

def convert_prediction_to_binary(df: pd.DataFrame, model_name: str):
    df[const.PRED] = df[const.PRED].str.capitalize()
    df[const.PRED] = df[const.PRED].str.strip(string.punctuation)
    print(df[const.PRED].value_counts(dropna=False))

    if model_name in ["gpt35", "gpt4"]:
        f = map_prediction_to_binary

    elif model_name.startswith("medalpaca"):
        f = map_prediction_to_binary_MedAlpaca

    else:
        raise ValueError(f"Unknown model name: {model_name}")

    df[const.PRED_BINARY] = df[const.PRED].apply(
        lambda x: f(x, args.target_language,
                    return_string=False))

    return df

def summarize_verifiability_results(language: str, temperature: float):
    model_prefix = "" if args.model == "gpt35" else f"{args.model}_"

    model_name = args.model

    if args.dataset_name == "healthqa":

        path = osp.join(args.output_dir, "verifiability",
                        f"{model_prefix}{args.dataset_name}_verifiability_temp{temperature}_{args.split}_{language}.xlsx")

    elif args.dataset_name in ["liveqa", "medicationqa"]:

        path = osp.join(args.output_dir, "verifiability",
                        f"{model_prefix}{args.dataset_name}_verifiability_temp{temperature}.xlsx")

    else:
        raise NotImplementedError

    if not osp.exists(path):
        return None

    df = pd.read_excel(path, sheet_name=language)

    if model_name in ["gpt35", "gpt4"]:
        f = map_prediction_to_binary

    elif model_name.startswith("medalpaca"):
        f = map_prediction_to_binary_MedAlpaca

    else:
        raise ValueError(f"Unknown model name: {model_name}")

    df[const.PRED_BINARY] = df[const.PRED].apply(
        lambda x: f(x, return_string=False))

    df[const.PRED_BINARY] = df[const.PRED_BINARY].fillna(-1).astype(
        int)

    print(f"Length (original) {len(df)}")
    df = df[df[const.QUESTION].notna()]
    df = df[df[const.PRED].notna()]
    print(f"Length (after drop null questions/predictions) {len(df)}")
    df = df[df[const.PRED_BINARY] != -1]
    print(f"Length (after drop unknown answers) {len(df)}")

    results = {}

    for pos_label in [0, 1]:

        # Get precision, recall, f1, accuracy, and auc scores
        precision, recall, f1, _ = precision_recall_fscore_support(
            df[const.LABEL], df[
                const.PRED_BINARY], average='binary', pos_label=pos_label)

        macro_precision, macro_recall, macro_f1, _ = precision_recall_fscore_support(
            df[const.LABEL], df[
                const.PRED_BINARY], average='macro', pos_label=pos_label)
        accuracy = accuracy_score(df[const.LABEL], df[const.PRED_BINARY])
        auc = roc_auc_score(df[const.LABEL], df[const.PRED_BINARY])

        results[pos_label] = {
            'precision': precision,
            'recall': recall,
            'f1': f1,
            'macro_precision': macro_precision,
            'macro_recall': macro_recall,
            'macro_f1': macro_f1,
            'accuracy': accuracy,
            'auc': auc,
        }

        if PLOT_CONFUSION_MATRIX and pos_label == 1:
            plot_confusion_matrix(df, language=language, pos_label=pos_label,
                                  args=args)

    return results

if __name__ == '__main__':
    project_setup()
    PLOT_CONFUSION_MATRIX = True

    for temperature in [0.0, 0.25, 0.5, 0.75, 1.0]:

        for language in ["English", "Spanish", "Chinese", "Hindi"]:
            print(f"Language: {language}, Temperature {temperature}")
            args.target_language = language

            model_prefix = "" if args.model == "gpt35" else f"{args.model}_"
            results = summarize_verifiability_results(args.target_language, temperature)

            if results is None:
                print(f"Not found: {args.dataset_name}\t{args.target_language}\t{temperature}")
                continue

            results = pd.DataFrame(results)

            print(
                f"[{args.dataset_name}]\tLanguage: {args.target_language}, Temperature {temperature}")

            print(results)

            path = osp.join(args.output_dir, "verifiability",
                            f"SUMMARY_{get_model_prefix(args)}{args.dataset_name}_verifiability_temp{temperature}.xlsx")
            if osp.exists(path):
                with pd.ExcelWriter(path, mode='a', engine='openpyxl',
                                    if_sheet_exists='replace') as writer:
                    results.to_excel(writer, sheet_name=language)

            else:
                with pd.ExcelWriter(path, mode='w',
                                    engine='openpyxl') as writer:
                    results.to_excel(writer, sheet_name=language)

            print("Done!")


from torch import mode
from .utils import (
    convertForBatchHardTripletLoss,
    convertForContrastiveLoss,
    convertForTripletLoss,
    convertForSoftmaxLoss,
)
from .batchHardDataset import BatchHardDataset, PKSampler
from .softmaxDataset import SoftmaxDataset
from sentence_transformers import losses, SentenceLabelDataset, SentencesDataset
from torch.utils.data import DataLoader
from .customLosses import SoftmaxLayerLoss

class LossHandler:
    def __init__(self, lossName: str, FAQ, model, batchSize: int):
        supportedLosses = (
            "batchHardTriplet",
            "contrastiveLoss",
            "tripletLoss",
            "softmaxLayerLoss",
        )
        assert (
            lossName in supportedLosses
        ), "Error {} loss not supported , supported losses are {}".format(
            lossName, supportedLosses
        )

        if lossName == supportedLosses[0]:
            assert (
                batchSize > 8 and batchSize % 8 == 0
            ), "needed for the pk sampler (p ,k > 2), must give large batches for batch hard"

            self.trainExamples = convertForBatchHardTripletLoss(FAQ=FAQ)
            self.dataset = BatchHardDataset(examples=self.trainExamples, model=model)
            self.trainLoss = losses.BatchHardTripletLoss(model=model)
            sampler = PKSampler(dataSource=self.dataset, p=8, k=batchSize // 8)
            self.trainDataLoader = DataLoader(
                self.dataset, batch_size=batchSize, sampler=sampler
            )

        elif lossName == supportedLosses[1]:
            assert (
                batchSize > 4 and batchSize % 4 == 0
            ), "needed for good contrastive loss"

            self.trainExamples = convertForContrastiveLoss(FAQ=FAQ)
            self.dataset = SentencesDataset(self.trainExamples, model=model)
            self.trainLoss = losses.ContrastiveLoss(model=model)
            self.trainDataLoader = DataLoader(
                self.dataset, shuffle=True, batch_size=batchSize
            )

        elif lossName == supportedLosses[2]:
            self.trainExamples = convertForTripletLoss(FAQ=FAQ)
            self.dataset = SentenceLabelDataset(self.trainExamples, model=model)
            self.trainLoss = losses.TripletLoss(model=model, triplet_margin=1)
            self.trainDataLoader = DataLoader(
                self.dataset, shuffle=True, batch_size=batchSize
            )

        elif lossName == supportedLosses[3]:
            self.trainExamples, numLabels = convertForSoftmaxLoss(FAQ=FAQ)
            self.dataset = SoftmaxDataset(self.trainExamples, model=model)
            self.trainLoss = SoftmaxLayerLoss(model=model, numLabels=numLabels)
            self.trainDataLoader = DataLoader(
                self.dataset, shuffle=True, batch_size=batchSize
            )


#!/usr/bin/env python
# coding: utf-8

# In[102]:

# all this code was borrowed from
# https://towardsdatascience.com/how-i-used-machine-learning-to-automatically-hand-draw-any-picture-7d024d0de997
from typing import OrderedDict
import cv2;
from matplotlib import pyplot as plt
import numpy as np
import pyautogui as pg
import kdtree
import operator
from Paint_CNC import Paint_CNC
from cnc import CNC
from sklearn.cluster import KMeans
from collections import defaultdict
import os, sys, time
from grid import Grid

class AutoDraw(object):
    def __init__(self, name, blur = 0, xDim=None, yDim=None):
        # Tunable parameters
        self.detail = 1
#         self.scale = 7/12
        self.scale = 1
        self.sketch_before = False
        self.with_color = True
        self.num_colors = 10
        self.outline_again = False
        self.possible_colors = OrderedDict()
        self.possible_colors['brown'] = [102, 82, 86]
        self.possible_colors['blue'] = [78, 151, 228]
        self.possible_colors['yellow'] = [249, 216, 36]
        self.possible_colors['orange'] = [238, 75, 29]
        self.possible_colors['green'] = [56, 131, 57]
        self.possible_colors['red'] = [171, 24, 26]
        self.possible_colors['purple'] = [54, 35, 88]
        self.possible_colors['black'] = [31, 25, 33]
        # Load Image. Switch axes to match computer screen
        self.img = cv2.imread(name)
        self.blur = blur
        self.img = np.swapaxes(self.img, 0, 1)
        self.img_shape = self.img.shape

#         self.dim = pg.size()

        # 30 cm x 18 cm
        if xDim == None and yDim == None:
            self.xRatio = 18/30
            self.yDim = 150
            self.xDim = self.yDim * self.xRatio
        else:
            self.xDim = xDim
            self.yDim = yDim
        self.dim = (self.yDim, self.xDim)
        # Scale to draw inside part of screen
        self.startX = ((1 - self.scale) / 2)*self.dim[0]
        self.startY = ((1 - self.scale) / 2)*self.dim[1]
#         self.dim = (self.dim[0] * self.scale, self.dim[1] * self.scale)

#         fit the picture into this section of the screen
        # if self.img_shape[1] > self.img_shape[0]:
        #     # if it's taller that it is wide, truncate the wide section
        #     self.dim = (int(self.dim[1] * (self.img_shape[0] / self.img_shape[1])), self.dim[1])
        # else:
        #     # if it's wider than it is tall, truncate the tall section
        #     self.dim = (self.dim[0], int(self.dim[0] *(self.img_shape[1] / self.img_shape[0])))

        # Get dimension to translate picture. Dimension 1 and 0 are switched due to comp dimensions
        ratio = self.img.shape[0] / self.img.shape[1]
        pseudo_x = int(self.img.shape[1] * self.detail)
        self.pseudoDim = (pseudo_x, int(pseudo_x * ratio))

# Tunable parameters
        # self.detail = 1
        # self.scale = 7/12
        # self.sketch_before = False
        # self.with_color = True
        # self.num_colors = 10
        # self.outline_again = False

        # # Load Image. Switch axes to match computer screen
        # self.img = cv2.imread(name)
        # self.blur = blur
        # self.img = np.swapaxes(self.img, 0, 1)
        # self.img_shape = self.img.shape

        # self.dim = pg.size()

        # # Scale to draw inside part of screen
        # self.startX = ((1 - self.scale) / 2)*self.dim[0]
        # self.startY = ((1 - self.scale) / 2)*self.dim[1]
        # self.dim = (self.dim[0] * self.scale, self.dim[1] * self.scale)

        # # fit the picture into this section of the screen
        # if self.img_shape[1] > self.img_shape[0]:
        #     # if it's taller that it is wide, truncate the wide section
        #     self.dim = (int(self.dim[1] * (self.img_shape[0] / self.img_shape[1])), self.dim[1])
        # else:
        #     # if it's wider than it is tall, truncate the tall section
        #     self.dim = (self.dim[0], int(self.dim[0] *(self.img_shape[1] / self.img_shape[0])))

        # # Get dimension to translate picture. Dimension 1 and 0 are switched due to comp dimensions
        # ratio = self.img.shape[0] / self.img.shape[1]
        # pseudo_x = int(self.img.shape[1] * self.detail)
        # self.pseudoDim = (pseudo_x, int(pseudo_x * ratio))

          # Initialize directions for momentum when creating path
        self.maps = {0: (1, 1), 1: (1, 0), 2: (1, -1), 3: (0, -1), 4: (0, 1), 5: (-1, -1), 6: (-1, 0), 7: (-1, 1)}
        self.momentum = 1
        self.curr_delta = self.maps[self.momentum]

        # Create Outline
        self.drawing = self.process_img(self.img)
        # plt.imshow(self.drawing)
        # plt.show()

    def rescale(self, img, dim):
        resized = cv2.resize(img, dim, interpolation=cv2.INTER_AREA)
        return resized

    def translate(self, coord):
        ratio = (coord[0] / self.pseudoDim[1], coord[1] / self.pseudoDim[0]) # this is correct
        deltas = (int(ratio[0] * self.dim[0]), int(ratio[1] * self.dim[1]))
        return self.startX + deltas[0], self.startY + deltas[1]

    def process_img(self, img):
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        if self.blur == 2:
            gray = cv2.GaussianBlur(gray, (9, 9), 0)
            canny = cv2.Canny(gray, 25, 45)
        elif self.blur == 1:
            gray = cv2.GaussianBlur(gray, (3, 3), 0)
            canny = cv2.Canny(gray, 25, 45)
        else:  # no blur
            canny = cv2.Canny(gray, 50, 75)
        canny = self.rescale(canny, self.pseudoDim)
        r, res = cv2.threshold(canny, 50, 255, cv2.THRESH_BINARY_INV)

        return res

    def drawOutline(self):
        indices = np.argwhere(self.drawing < 127).tolist()  # get the black colors
        index_tuples = map(tuple, indices)

        self.hashSet = set(index_tuples)
        self.KDTree = kdtree.create(indices)
#         self.KDTree = KDTree(indices)
        self.commands = []
        self.curr_pos = (0, 0)
        point = self.translate(self.curr_pos)
        self.commands.append(point)

#         print('Please change pen to thin and color to black.')
#         input("Press enter once ready")
#         print('')

        # DRAW THE BLACK OUTLINE
        self.createPath()
#         self.cleanCommands()
        return self.commands
#         print(self.commands)
#         input("Ready! Press Enter to draw")
#         print('5 seconds until drawing beings')
#         time.sleep(5)

#         self.execute(self.commands)

    def execute(self, commands):
        press = 0  # flag indicating whether we are putting pressure on paper

        for (i, comm) in enumerate(commands):
            if type(comm) == str:
                if comm == 'UP':
                    press = 0
                if comm == 'DOWN':
                    press = 1
            else:
                print(len(commands) - i)
                if press == 0:
                    pg.moveTo(comm[0], comm[1], 0)
                else:
                    pg.dragTo(comm[0], comm[1], 0, button='left')
        return

    def createPath(self):
        # check for closest point. Go there. Add click down. Change curr. Remove from set and tree. Then, begin
        new_pos = tuple(self.KDTree.search_nn(self.curr_pos)[0].data)
#         print(self.curr_pos[0)
#         new_pos = tuple(self.KDTree.query(self.curr_pos)[0].data)
        self.commands.append(new_pos)
        self.commands.append("DOWN")
        self.curr_pos = new_pos
        self.KDTree = self.KDTree.remove(list(new_pos))
        self.hashSet.remove(new_pos)

        while len(self.hashSet) > 0:
            prev_direction = self.momentum
            candidate = self.checkMomentum(self.curr_pos)
            if self.isValid(candidate):
                new = tuple(map(operator.add, self.curr_pos, candidate))
                new_pos = self.translate(new)
                if prev_direction == self.momentum and type(self.commands[-1]) != str:  # the direction didn't change
                    self.commands.pop()
                self.commands.append(new_pos)
            else:
                self.commands.append("UP")
                new = tuple(self.KDTree.search_nn(self.curr_pos)[0].data)
#                 new = tuple(self.KDTree.query(self.curr_pos)[0].data)
                new_pos = self.translate(new)
                self.commands.append(new_pos)
                self.commands.append("DOWN")
            self.curr_pos = new
            self.KDTree = self.KDTree.remove(list(new))
            self.hashSet.remove(new)
            # print('Just went to point ', self.curr_pos)
            # print('Making path...number points left: ', len(self.hashSet))
        return self.commands

#     def cleanCommands(self):
#         self.commands = self.commands[2:]
#         for command in self.commands:
#             if(type(command) is not str):
#                 command[0] -= 25
#                 command[1] -=25

    def isValid(self, delta):
        return len(delta) == 2

    def checkMomentum(self, point):
        # Returns best next relative move w.r.t. momentum and if in hashset
        self.curr_delta = self.maps[self.momentum]
        moments = self.maps.values()
        deltas = [d for d in moments if (tuple(map(operator.add, point, d)) in self.hashSet)]
        deltas.sort(key=self.checkDirection, reverse=True)
        if len(deltas) > 0:
            best = deltas[0]
            self.momentum = [item[0] for item in self.maps.items() if item[1] == best][0]
            return best
        return [-1]

    def checkDirection(self, element):
        return self.dot(self.curr_delta, element)

    def dot(self, pt1, pt2):
        pt1 = self.unit(pt1)
        pt2 = self.unit(pt2)
        return pt1[0] * pt2[0] + pt1[1] * pt2[1]

    def unit(self, point):
        norm = (point[0] ** 2 + point[1] ** 2)
        norm = np.sqrt(norm)
        return point[0] / norm, point[1] / norm

    def get_line_color(self, r, g, b):

        min_distance = 9999999999
        min_color = ""
        for color, rgb in self.possible_colors.items():
            possible_min_distance = abs(r - rgb[0]) + abs(g - rgb[1]) + abs(b - rgb[2])
            if possible_min_distance < min_distance:
                min_distance = possible_min_distance
                min_color = color
        return min_color

    def draw(self):
        if self.with_color:
            # Original:
            # color = self.rescale(self.img, self.pseudoDim)
            # collapsed = np.sum(color, axis=2)/3
            # fill = np.argwhere(collapsed < 230)  # color 2-d indices
            # fill = np.swapaxes(fill, 0, 1)  # swap to index into color
            # RGB = color[fill[0], fill[1], :]
            # k_means = KMeans(n_clusters=self.num_colors).fit(RGB)
            # colors = k_means.cluster_centers_
            # labels = k_means.labels_
            # fill = np.swapaxes(fill, 0, 1).tolist()  # swap back to make dictionary
            # label_2_index = defaultdict(list)

            # for i, j in zip(labels, fill):
            #     label_2_index[i].append(j)
            # print(label_2_index)

            color = self.rescale(self.img, self.pseudoDim)
            collapsed = np.sum(color, axis=2)/3
            fill = np.argwhere(collapsed < 230)  # color 2-d indices
            fill = np.swapaxes(fill, 0, 1)  # swap to index into color
            RGB = color[fill[0], fill[1], :]
            colors = self.possible_colors.values()

            color_commands = OrderedDict()
            color_commands['brown'] = []
            color_commands['blue'] = []
            color_commands['yellow'] = []
            color_commands['orange'] = []
            color_commands['green'] = []
            color_commands['red'] = []
            color_commands['purple'] = []
            color_commands['black'] = []
            labels = []

            for (i, color) in enumerate(RGB):
                line_color = self.get_line_color(color[2], color[1], color[0])
                labels.append(list(color_commands.keys()).index(line_color))

            fill = np.swapaxes(fill, 0, 1).tolist()  # swap back to make dictionary

            label_2_index = defaultdict(list)
            for i, j in zip(labels, fill):
                label_2_index[i].append(j)

            for (i, color) in enumerate(colors):
                if label_2_index[i] == []:
                    continue

                # Grayscale conversion formula found at
                # https://www.dynamsoft.com/blog/insights/image-processing/image-processing-101-color-space-conversion/
                # grayscale = 0.299 * color[2] + 0.587 * color[1] + 0.114 * color[0]
#                 print(grayscale)
                # print('Please change the pen to thick and color to BGR (not RGB) values: ', color)
                # input("Press enter once ready")
                # print('')
                points = label_2_index[i]
                index_tuples = map(tuple, points)
                self.hashSet = set(index_tuples)
                self.KDTree = kdtree.create(points)
                self.commands = []
                self.curr_pos = (0, 0)
                point = self.translate(self.curr_pos)
                self.commands.append("UP")
                self.commands.append(point)

                line_color = self.get_line_color(color[0], color[1], color[2])
                self.commands.append(line_color)
                # if(color[2] >= color[1] and color[2] >= color[0]):
                #     line_color = "red"
                #     print("red")
                #     # self.commands.append("BLUE")
                # if(color[1] >= color[2] and color[1] >= color[0]):
                #     line_color = "green"
                #     print("green")
                #     # self.commands.append("GREEN")
                # if(color[0] >= color[2] and color[0] >= color[1]):
                #     line_color = "blue"
                #     print("blue")
                #     # self.commands.append("RED")
                # if (max(color) <= 50):
                #     line_color = "black"
                color_commands[line_color] += self.createPath()
                self.commands.append("UP")

                # input('Ready! Press enter to draw: ')
                # print('5 seconds until drawing begins...')
                # time.sleep(5)
                # try:
                # self.execute(self.commands)
                # except KeyboardInterrupt:
                #     sys.exit()

            return color_commands
            # return self.commands
            if self.outline_again:
                self.drawOutline()

# In[62]:

    def commands_to_cnc(self, cnc, commands, prevNonUpOrDownCommand, color, pen_color=False):
        newCommands = []
        for index in range(len(commands)):
            if(prevNonUpOrDownCommand == commands[index]):
                continue
            if commands[index] == 'UP':
                if index + 1 == len(commands):
                    newCommands.append(commands[index])
                elif commands[index + 1] != 'UP' and commands[index + 1] != 'DOWN':
                    newCommands.append(commands[index])
    #               cnc.up()
            elif commands[index] == 'DOWN':
                if index + 1 == len(commands):
                    newCommands.append(commands[index])
                elif commands[index + 1] != 'UP' and commands[index + 1] != 'DOWN':
                    newCommands.append(commands[index])
            else:
                newCommands.append(commands[index])
                prevNonUpOrDownCommand = commands[index]

        preCondensedCommands = []

        for index in range(len(newCommands)):
            if newCommands[index] == color:
                preCondensedCommands.append(color)
                # cnc.set_paint_color(color)
            elif newCommands[index] == None:
                continue
            elif newCommands[index] == 'UP' or newCommands[index - 1] == color:
                if index + 1 == len(newCommands):
                    preCondensedCommands.append('UP')
                    # cnc.up()
                elif newCommands[index + 1] != 'UP' and newCommands[index + 1] != 'DOWN':
                    preCondensedCommands.append('UP')
                    # cnc.up()
            elif newCommands[index] == 'DOWN':
                if newCommands[index - 2] == color:
                    preCondensedCommands.append('UP')
                    # cnc.g1(z=5)
                elif index + 1 == len(newCommands):
                    preCondensedCommands.append('DOWN')
                    # cnc.down()
                elif newCommands[index + 1] != 'UP' and newCommands[index + 1] != 'DOWN':
                    preCondensedCommands.append('DOWN')
                    # cnc.down()
            else:
                # print(newCommands[index])
                # print(newCommands[index])
                if newCommands[index - 4] == color:
                    preCondensedCommands.append('DOWN')
                    # cnc.down()
                preCondensedCommands.append((float(newCommands[index][1]), -float(newCommands[index][0])))
                # cnc.g1(x=float(newCommands[index][1]),y=-float(newCommands[index][0]))

        realPreCondensedCommands = []

        isUp = False
        prevCommand = None
        for command in preCondensedCommands:
            if command == color:
                realPreCondensedCommands.append(color)
            if command == 'UP':
                if not isUp:
                    realPreCondensedCommands.append('UP')
                isUp = True
            elif command == 'DOWN':
                if isUp:
                    realPreCondensedCommands.append(prevCommand)
                    realPreCondensedCommands.append('DOWN')
                    isUp = False
            else:
                if not isUp:
                    realPreCondensedCommands.append(command)
                prevCommand = command

        if color == "NA":
            for command in realPreCondensedCommands:
                if command == color:
                    continue
                elif command == 'UP':
                    cnc.up()
                elif command == 'DOWN':
                    cnc.down()
                else:
                    cnc.g0(x=command[0], y=command[1])
            return realPreCondensedCommands
        elif pen_color:
            for command in realPreCondensedCommands:
                if command == color:
                    cnc.comment(color)
                elif command == 'UP':
                    cnc.up()
                elif command == 'DOWN':
                    cnc.down()
                else:
                    cnc.g0(x=command[0], y=command[1])
            return realPreCondensedCommands
        else:
            condensedCommands = []
            grid = Grid(self.xDim, -self.yDim)
            isUp = False
            print(len(realPreCondensedCommands))
            count = 0
            for index in range(len(realPreCondensedCommands)):
                count += 1
                print(count)
                if realPreCondensedCommands[index] == color:
                    condensedCommands.append(color)
                elif realPreCondensedCommands[index] == 'UP':
                    condensedCommands.append('UP')
                    isUp = True
                elif realPreCondensedCommands[index] == 'DOWN':
                    condensedCommands.append('DOWN')
                    isUp = False
                elif index + 1 >= len(realPreCondensedCommands):
                    continue
                elif realPreCondensedCommands[index + 1]  == color or realPreCondensedCommands[index + 1]  == 'UP' or realPreCondensedCommands[index + 1]  == 'DOWN':
                    if not isUp:
                        if(grid.shouldAdd(realPreCondensedCommands[index][0], realPreCondensedCommands[index][1])):
                            condensedCommands.append(realPreCondensedCommands[index])
                    if isUp:
                        condensedCommands.append(realPreCondensedCommands[index])
                    # continue
                else:
                    if isUp:
                        condensedCommands.append(realPreCondensedCommands[index])
                    if not isUp:
                        condensedCommands.extend(grid.getParsedCommands(realPreCondensedCommands[index][0], realPreCondensedCommands[index][1], realPreCondensedCommands[index + 1][0], realPreCondensedCommands[index + 1][1]))

            # print(condensedCommands)

            prevCommand = (-1000, 1000)
            isUp = False
            count = 0
            for command in condensedCommands:
                count += 1
                if command == color:
                    prevCommand = command
                    cnc.set_paint_color(color)
                elif command == 'UP':
                    isUp = True
                    prevCommand = command
                    cnc.up()
                elif command == 'DOWN':
                    isUp = False
                    if prevCommand != 'DOWN' and prevCommand != 'UP' and prevCommand != color:
                        cnc.g0(x=prevCommand[0], y=prevCommand[1])
                    prevCommand = command
                    cnc.down()
                else:
                    if command != prevCommand and not isUp:
                        cnc.g0(x=command[0], y=command[1])
                    prevCommand = command

            return condensedCommands

def main(image_file_path, output_type, blur=0, xDim=None, yDim=None):
    name = image_file_path.split("/")[1].split(".")[0]
    output_str_lookup = {0:"outline", 1:"pen_color", 2:"watercolor"}
    output_path = os.path.join(os.getcwd(), output_str_lookup[output_type], "gcode", f"{name}.gcode")

    if not os.path.exists(output_path):
        dirs = output_path.split("\\")[:-1]
        os.makedirs("\\".join(dirs), exist_ok=True)

    drawing = AutoDraw(image_file_path, blur=blur, xDim=xDim, yDim=yDim)
    if output_type != 0:
        color_commands = drawing.draw()
    else:
        commands = drawing.drawOutline()
    # commands += drawing.drawOutline()

    # cnc.g1(z=0)
    prevNonUpOrDownCommand = (0, 0)
    newCommands = []

    # for output of a black pen outline, we use the base CNC library
    if output_type == 0:
        cnc = CNC()
        cnc.open(output_path)

        cnc.g90()
        cnc.g0(z=5)
        cnc.f(3000)
        cnc.g0(z=5)
        newCommands = drawing.commands_to_cnc(cnc, commands, prevNonUpOrDownCommand, "NA")
    # for output of a colored pen drawing, we use the base CNC library and include comment metadata
    elif output_type == 1:
        cnc = CNC()
        cnc.open(output_path)

        cnc.g90()
        cnc.g0(z=5)
        cnc.f(3000)
        cnc.g0(z=5)
        for color in color_commands.keys():
            cnc.comment(color)
            commands = color_commands[color]
            newCommands.extend(drawing.commands_to_cnc(cnc, commands, prevNonUpOrDownCommand, color, pen_color=True))
    # for output of a watercolor painting, we use the paint library which extends the CNC library and include comment metadata
    # to encode color changes, brush adjustments, etc.
    else:
        cnc = Paint_CNC()
        cnc.open(output_path)

        cnc.g90()
        cnc.g0(z=5)
        cnc.f(3000)
        cnc.g0(z=5)
        for color in color_commands.keys():
            cnc.comment(color)
            commands = color_commands[color]
            newCommands.extend(drawing.commands_to_cnc(cnc, commands, prevNonUpOrDownCommand, color))

    cnc.close()

# In[ ]:

if __name__ == "__main__":
    import argparse

    process_parser = argparse.ArgumentParser()

    process_parser.add_argument('image_file_path', type=str, help="Path to image file you would like turned into G-Code")
    # process_parser.add_argument("with_color", type=int, help="True for with color, false for without")
    process_parser.add_argument("blur", nargs="?", type=int, help="Amount of blur for image (between 0 and 2)")
    process_parser.add_argument("xDim", nargs="?", type=float, help="x dimension in mm")
    process_parser.add_argument("yDim", nargs="?", type=float, help="y  dimension in mm")
    process_parser.add_argument("output_type", default=0, type=int, help="The algorithm can generate G-code to render images 3 ways. \
    Input the corresponding int: pen outline=0, colored pen=1, watercolor=2")

    args = process_parser.parse_args()
    image_file_path = args.image_file_path
    output_type = args.output_type
    xDim = args.xDim
    yDim = args.yDim
    blur = args.blur

    if not os.path.isfile(image_file_path):
        raise FileNotFoundError("The reference image file specified does not exist on this path.")
        sys.exit()
    if output_type not in [0, 1, 2]:
        raise ValueError("Invalid output type given. Please select either pen outline (0), colored pen(1), or watercolor (2)")

    main(image_file_path, output_type, blur, xDim, yDim)