"""Common functions and classes."""
import os
import datetime
import time
import json
import yaml
import sqlite3

CODES = {'CancelledError': 0,  # can appear and this is in normal
         # all below exceptions should not appear, try to decrease load then:
         'ServerDisconnectedError': -1,
         'ClientOSError': -2,
         }

def get_timestamp():
    return time.time()

def timestamp2str(fmt):
    return datetime.datetime.now().strftime(fmt)

def log_timestamp_str():
    return timestamp2str('[%Y-%m-%d %H:%M:%S.%f]')

def generate_test_run_id():
    return timestamp2str('%Y-%m-%d_%H-%M-%S_%f')

def get_last_logs(lines):
    return '\n'.join([line.replace('"', '') for line in lines if 'HTTP/1.1 200' not in line][-10:])

def load_testrun_data(file_name):
    """Test run load is described in a json file <test_run_title>.json.
    A function reads the json file and returns a Python dictionary.
    """
    load = {}
    if os.path.isfile(file_name):
        with open(file_name, 'r+') as _f:
            load = json.loads(_f.read())
    return load

def load_conf(file_name='conf.yaml'):
    """A function reads slaves configuration from a YAML-file stored on master in app folder."""
    hosts = []
    if os.path.isfile(file_name):
        with open(file_name, 'r+') as _f:
            hosts = yaml.load(_f.read())['hosts']
    return hosts

def save_conf(hosts, file_name='conf.yaml'):
    """A function (re)writes slaves configuration in a YAML-file stored on master in app folder."""
    with open(file_name, 'w+') as _f:
       _f.write(yaml.safe_dump({'hosts': hosts}, default_flow_style=False))
    return True

def load_users(file_name='users.txt', folder=None):
    """A function reads users ids from a txt-file stored on master in app/engine folder."""
    lines = []
    folder = folder or os.path.dirname(os.path.abspath(__file__))
    file_name = os.path.join(folder, file_name)
    if os.path.isfile(file_name):
        with open(file_name, 'r') as _f:
            lines = _f.readlines()
    users = list(set([line.strip() for line in lines if line.strip()]))
    if len(lines) != len(users):
        with open(file_name, 'w') as _f:
            _f.write('\n'.join(users))
    return users

def get_db_conn(file_name, folder=None):
    folder = folder or os.path.dirname(os.path.abspath(__file__))
    db = os.path.join(folder, file_name)
    db_conn = sqlite3.connect(db)
    return db_conn

def db_init(db_conn):
    sqls = ['''CREATE TABLE IF NOT EXISTS recs (\
                             id INTEGER PRIMARY KEY, \
                             atomic BOOLEAN default 1, \
                             timestamp TEXT, \
                             action TEXT, \
                             user TEXT, \
                             latency TEXT default NULL, \
                             code INTEGER default NULL, \
                             reason TEXT default NULL);''',
            '''CREATE TABLE IF NOT EXISTS info (\
                             id INTEGER PRIMARY KEY, \
                             test_run_id TEXT NOT NULL, \
                             test_run_status TEXT default 'IN PROGRESS',
                             slave_name TEXT NOT NULL, \
                             timestamp_started TEXT NOT NULL, \
                             timestamp_completed TEXT, \
                             total_load_info TEXT,
                             slave_load_info TEXT);''',
            'DELETE FROM recs;',
            'DELETE FROM info;']
    for sql in sqls:
        db_conn.execute(sql)
    db_conn.commit()
    return True

def db_query(db_conn, sql):
    cur = db_conn.cursor()
    try:
        cur.execute(sql)
        db_conn.commit()
        return cur.lastrowid, cur.fetchall()
    except Exception as err:
        print('ERROR DB: %s' % err)
        return 0, []

def log_sql(func):  # to be applied only to act() methods of classes inherited from Action()
    async def wrapper(*args):
        timestamp = get_timestamp()
        sql = "INSERT INTO recs (atomic, timestamp, action, user) VALUES (0, '%s', '%s', '%s')" \
              % (timestamp, args[0].action, args[0].user)
        row_id = db_query(args[0].conn, sql)[0]  # args[0] is an instance of Action
        code, reason = await func(*args)
        sql = "UPDATE recs SET latency='%s', code=%d, reason='%s' WHERE id=%s" \
              % (get_timestamp() - timestamp, code, reason, row_id)
        db_query(args[0].conn, sql)
        return code, reason
    return wrapper

class LogSQL(object):
    """A class used for logging in atomic_get/atomic_post methods of Action-based classes."""
    def __init__(self, conn, action, user):
        self.conn = conn
        self.action = action
        self.user = user
        self.row_id = None  # id of a row in the database (each request is kept in one row)
        self.code = None    # response code of the atomic HTTP request
        self.reason = None  # returned reason of the atomic HTTP request
        self.timestamp = get_timestamp()

    def __enter__(self):
        sql = "INSERT INTO recs (timestamp, action, user) VALUES ('%s', '%s', '%s')" \
              % (self.timestamp, self.action, self.user)
        self.row_id = db_query(self.conn, sql)[0]
        return self

    def __exit__(self, type, value, traceback):
        sql = "UPDATE recs SET latency='%s', code=%d, reason='%s' WHERE id=%s" \
              % (get_timestamp() - self.timestamp, self.code, self.reason, self.row_id)
        db_query(self.conn, sql)
        return False


from AST import *
from Visitor import *
from Utils import Utils
from StaticError import *
from dataclasses import dataclass

class Raiser:
    def __init__(self, e: Exception):
        raise e

@dataclass
class Symbol:
    pass

class StaticChecker(BaseVisitor, Utils):

    def __init__(self, ast):
        self.ast = ast

    def check(self):
        self.visit(self.ast, None)

    def visitProgram(self, ctx: Program, o):
        o = []
        exp = ctx.exp
        decl_list = ctx.decl
        for decl in decl_list:
            self.visit(decl, o)
        self.visit(exp, o)

    def visitVarDecl(self, ctx: VarDecl, o):
        o.append(ctx)

    def visitBinOp(self, ctx: BinOp, o):
        left = ctx.e1
        right = ctx.e2
        literals = [BoolType, FloatType, IntType]
        # recursive left & right till it become literals
        while not (type(left) in literals):
            left = self.visit(left, o)
        while not (type(right) in literals):
            right = self.visit(right, o)

        if (ctx.op in ["+", "-", "*"]):
            # Check TypeMismatchInExpression
            if (not (type(left) in [FloatType, IntType])) or (not (type(right) in [FloatType, IntType])):
                raise TypeMismatchInExpression(ctx)
            # Output
            if (type(left) is FloatType or type(right) is FloatType):
                return FloatType()
            return IntType()

        elif (ctx.op in ["/"]):
            # Check TypeMismatchInExpression
            if (not (type(left) in [FloatType, IntType])) or (not (type(right) in [FloatType, IntType])):
                raise TypeMismatchInExpression(ctx)
            # Output
            return FloatType()

        elif (ctx.op in ['&&', '||']):
            # Check TypeMismatchInExpression
            if (not (type(left) in [BoolType])) or (not (type(right) in [BoolType])):
                raise TypeMismatchInExpression(ctx)
            # Output
            return BoolType()

        elif (ctx.op in ["<", '>', '==', '!=']):
            # Check TypeMismatchInExpression
            if (not (type(left) is type(right))):
                raise TypeMismatchInExpression(ctx)
            # Output
            return BoolType()

    def visitUnOp(self, ctx: UnOp, o):
        val = ctx.e
        literals = [BoolType, FloatType, IntType]
        # recursive left & right till it become literals
        while not (type(val) in literals):
            val = self.visit(val, o)

        if (ctx.op in ['!']):
            # Check TypeMismatchInExpression
            if (not (type(val) in [BoolType])):
                raise TypeMismatchInExpression(ctx)
            # Output
            return BoolType()
        elif (ctx.op in ["-"]):
            # Check TypeMismatchInExpression
            if (not (type(val) in [FloatType, IntType])):
                raise TypeMismatchInExpression(ctx)
            # Output
            if (type(val) is FloatType):
                return FloatType()
            return IntType()

    def visitIntLit(self, ctx: IntLit, o):
        return IntType()

    def visitFloatLit(self, ctx, o):
        return FloatType()

    def visitBoolLit(self, ctx, o):
        return BoolType()

    def visitId(self, ctx, o):
        if not ([el.typ for el in o if ctx.name == el.name]):
            raise UndeclaredIdentifier(ctx.name)
        for el in o:
            if ctx.name == el.name:
                return el.typ


from django.db.models import query
from django.shortcuts import render
from rest_framework import viewsets
from rest_framework import generics
from rest_framework.serializers import Serializer
from api_lojinha import serializers
from rest_framework.generics import get_object_or_404
from produto import models
#from pedido import models
from .serializers import OpcoesSerializer,CategoriaSerializer,ProdutoSerializer,AdicionalSerializer

class ListarOpcoesViewSet(generics.ListCreateAPIView):
    serializer_class =OpcoesSerializer
    queryset = models.Opcoes.objects.all()

class OpcoesAPIView(generics.RetrieveUpdateDestroyAPIView):
    serializer_class = OpcoesSerializer
    queryset = models.Opcoes.objects.all()

class ListarProdutoAPIView(generics.ListCreateAPIView):
    serializer_class = ProdutoSerializer
    queryset = models.Produto.objects.all()

class ProdutoAPIView(generics.RetrieveUpdateDestroyAPIView):
    serializer_class = ProdutoSerializer
    queryset = models.Produto.objects.all()

class ListarCategoriaAPIView(generics.ListCreateAPIView):
    serializer_class = CategoriaSerializer
    queryset = models.Categoria.objects.all()

class CategoriaAPIView(generics.RetrieveUpdateDestroyAPIView):
    serializer_class = CategoriaSerializer
    queryset = models.Categoria.objects.all()

class ListarAdicionaisAPIView(generics.ListCreateAPIView):
    serializer_class = AdicionalSerializer
    queryset = models.Adicional.objects.all()

class AdicionalAPIView(generics.RetrieveUpdateDestroyAPIView):
    serializer_class = AdicionalSerializer
    queryset = models.Adicional.objects.all()

#class PedidoAPIView(generics.ListCreateAPIView):
    #serializer_class = PedidoSerializer
    #queryset = models.Pedido.objects.all()

#class ItemPedidoAPIView(generics.RetrieveUpdateDestroyAPIView):
    #serializer_class = ItemPedidoSerializer
    #queryset = models.ItemPedido.objects.all()
    #def get_queryset(self):
        #if self.kwargs.get('pedido_pk'):
            #return self.queryset.filter(pedido_id=self.kwargs.get('pedido_pk'))
        #return self.queryset.all()
# Create your views here.


import pygeoip, os
from django import template
from .. import constants

register = template.Library()

@register.inclusion_tag('uecookie9/message.html', takes_context=True)
def uecookie9(context):
    show = False
    request = context['request']
    closed = request.COOKIES.get('uecookie_closed')

    if not closed:
        ip = request.META.get('HTTP_X_REAL_IP') or request.META.get('REMOTE_ADDR')
        filename = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'resources/GeoIP.dat')
        g = pygeoip.GeoIP(filename)
        result = g.country_name_by_addr(ip)

        if result in constants.UE_MEMBERS:
            show = True

    return {'show' : show}


import os
import mcubes
import numpy as np
import open3d as o3d
from tqdm import tqdm

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from pytorch3d.loss import chamfer_distance
from torch.utils.tensorboard import SummaryWriter

from model import Assembler
from data import ChairDataset, collate_fn
from pointnet2_cls_ssg import get_model_encoder
from utils import farthest_point_sample, compute_sampling_metrics, pc_normalize_sphere, assemble_from_multiple_batch, assemble_chairs

def voxel_grid_to_sparse(voxel_grid):
    voxels = voxel_grid.get_voxels()
    coords = []
    for voxel in voxels:
        coord = np.array(voxel.grid_index).astype(np.int32)
        coords.append(coord)
    coords = np.array(coords)
    return coords

def sparse_to_volume(coords):
    model = np.zeros(list(coords.max(0)+1))
    for i in range(coords.shape[0]):
        coord = coords[i]
        x, y, z = coord[0], coord[1], coord[2]
        model[x, y, z] = 1
    return model

def show_chairs(chairs):
    for i in range(chairs.shape[0]):
        pc = chairs[i]

        pcd = o3d.geometry.PointCloud()
        pcd.points = o3d.utility.Vector3dVector(pc)
        o3d.visualization.draw_geometries([pcd])

hidden_dim = 256

pointnet2 = get_model_encoder(hidden_dim).cuda()
assembler = Assembler(hidden_dim).cuda()

pointnet2.load_state_dict(torch.load('./checkpoints_gan/pointnet2_13.pth'))
assembler.load_state_dict(torch.load('./checkpoints_gan/assembler_13.pth'))

############################################################################################################################################################################
"""
Set A
"""
bs = 5
num_workers = 8
val_dir = '../dataset/SetC'
val_dataset = ChairDataset(val_dir)
val_dataloader = DataLoader(val_dataset, collate_fn=collate_fn, batch_size=bs, num_workers=num_workers, drop_last=False, shuffle=True)

count = 0
final_eval = {}
pointnet2.eval()
assembler.eval()
with torch.no_grad():
    for batch in tqdm(val_dataloader):
        batch_size = batch['mask'].shape[0]
        print(batch_size)

        fake_batch = assemble_from_multiple_batch(batch)
        mask = fake_batch['mask'].float().cuda().unsqueeze(-1)
        parts = fake_batch['parts'].permute(0, 2, 1).float().cuda()

        part_f, _ = pointnet2(parts)
        obj_f = torch.zeros((batch_size, 5, hidden_dim)).cuda()

        select_idx = (fake_batch['mask'] == 1).nonzero(as_tuple=False)
        select_idx = select_idx[:, 0] * 5 + select_idx[:, 1]
        obj_f = obj_f.reshape(-1, hidden_dim)
        obj_f[select_idx] = part_f
        obj_f = obj_f.reshape(batch_size, 5, hidden_dim)

        obj_f = obj_f.reshape(batch_size, -1)
        pred = assembler(obj_f)

        fake_pc = assemble_chairs(parts, mask, pred).detach().cpu().numpy()
        show_chairs(fake_pc)
        print(fake_pc.shape)
        exit()

############################################################################################################################################################################


"""
This script will build the application for running without Python installed.

Usage:
    One must navigate to the parent directory of this directory in a terminal
    and use the command:
        python script/setup.py py2app
    but it might instead be necessary to use:
        python3 script/setup.py py2app
"""

from setuptools import setup

# Use pathlib to get the path of the project folder
import os
import pathlib
MAIN_DIRECTORY = pathlib.Path(__file__).resolve().parent.parent
PROJECT_FOLDER = os.path.join(MAIN_DIRECTORY, 'multitooner')
DATA_FOLDER = os.path.join(MAIN_DIRECTORY, 'assets')

# Read the configuration file
MAIN_PATH = os.path.join(PROJECT_FOLDER, 'main.py')
APPLICATION_PATH = os.path.join(PROJECT_FOLDER, 'app.py')
PREFERENCES_PATH = os.path.join(PROJECT_FOLDER, 'preferences.py')
AUTHENTICATE_PATH = os.path.join(PROJECT_FOLDER, 'authenticate.py')
CONFIG_PATH = os.path.join(PROJECT_FOLDER, 'config.py')
LOGIN_PATH = os.path.join(PROJECT_FOLDER, 'login.py')
ICON_PATH = os.path.join(DATA_FOLDER, 'icon.icns')
MENUBAR_ICON_PATH = os.path.join(DATA_FOLDER, 'icon-desaturated.icns')

APP = [MAIN_PATH]
DATA_FILES = [
    APPLICATION_PATH,
    PREFERENCES_PATH,
    AUTHENTICATE_PATH,
    CONFIG_PATH,
    LOGIN_PATH,
    ICON_PATH,
    MENUBAR_ICON_PATH,
]
OPTIONS = {
    'argv_emulation': True,
    'plist': {
        'LSUIElement': True,
    },
    'packages': [
        'rumps',
        'requests',
        'tooner',
        'LaunchServices',
        'certifi',
    ],
    'iconfile': ICON_PATH,
}

setup(
    app=APP,
    name='MultiTooner',
    data_files=DATA_FILES,
    options={'py2app': OPTIONS},
    setup_requires=['py2app'],
)


from django.contrib.auth import get_user_model
from django.test import Client, TestCase
from django.urls import reverse
from django.core.cache import cache

from posts.models import Group, Post

User = get_user_model()

class TestCache(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.user = User.objects.create_user(username='test_author')
        cls.group = Group.objects.create(
            title='TestGroup1',
            slug='TestGroup1',
            description='TestGroup1'
        )
        cls.test_post = Post.objects.create(
            text='Тест №0',
            group=cls.group,
            author=cls.user
        )

    def setUp(self):
        self.user_1_client = Client()
        self.user_1_client.force_login(TestCache.user)

    def test_cache(self):
        """
        Проверим, что до очистки кэша контент сохраняется даже
        после принудительного удаления объекта из базы данных
        """
        response_1 = self.user_1_client.get(
            reverse(
                'posts:index'
            )
        )
        TestCache.test_post.delete()
        response_2 = self.user_1_client.get(
            reverse(
                'posts:index'
            )
        )
        self.assertEqual(response_1.content, response_2.content)
        cache.clear()
        response_3 = self.user_1_client.get(
            reverse(
                'posts:index'
            )
        )
        self.assertNotEqual(response_1.content, response_3.content)


##
##
##

from dataclasses import dataclass

import torch
from jaxtyping import Bool, Shaped
from torch import Tensor
from typing_extensions import Self

@dataclass(init=False, frozen=True, slots=True)
class Batched2DTensors:
    """Dataclass to store a batch of 2D tensors with different lengths in a single 3D
    tensor.

    Attributes
    ----------
    tensor : Shaped[Tensor, "B L D"]
        The 3D tensor, where B is the batch size, L is the maximum length, and D is
        the number of features.
    mask : Bool[Tensor, "B L"]
        The mask of the 3D tensor, where True indicates a padded value.
    sizes : list[int]
        The lengths of the 2D tensors.
    """

    # -------------------------------------------------------------------------
    # Attributes and properties
    # -------------------------------------------------------------------------

    tensor: Shaped[Tensor, "B L D"]
    mask: Bool[Tensor, "B L"]
    sizes: list[int]

    @property
    def device(self) -> torch.device:
        """The device of the tensor."""
        return self.tensor.device

    @property
    def dtype(self) -> torch.dtype:
        """The dtype of the tensor."""
        return self.tensor.dtype

    @property
    def shape(self) -> torch.Size:
        """The shape of the tensor."""
        return self.tensor.shape

    # -------------------------------------------------------------------------
    # Initialization
    # -------------------------------------------------------------------------

    def __init__(
        self,
        tensor: Shaped[Tensor, "B L D"],
        mask: Bool[Tensor, "B L"] | None = None,
        sizes: list[int] | None = None,
    ) -> None:
        if mask is None and sizes is None:
            raise ValueError("Expected either mask or sizes to be provided.")

        if mask is None:
            mask = tensor.new_ones(tensor.shape[:2], dtype=torch.bool)
            for idx, size in enumerate(sizes):  # type: ignore
                mask[idx, :size] = False

        if sizes is None:
            sizes = (~mask).sum(dim=-1).tolist()

        object.__setattr__(self, "tensor", tensor)
        object.__setattr__(self, "mask", mask)
        object.__setattr__(self, "sizes", sizes)

    @classmethod
    def from_list(
        cls,
        tensors: list[Shaped[Tensor, "L D"]],
        pad_value: float = 0.0,
    ) -> Self:
        """Creates a batch of 2D tensors from a list of 2D tensors.

        Parameters
        ----------
        tensors : list[Shaped[Tensor, "L D"]]
            A list of 2D tensors.
        pad_value : float, optional
            The value to pad the tensors with, by default 0.0.

        Returns
        -------
        Batched2DTensors
            The batch of 2D tensors.
        """

        sizes = [tensor.shape[0] for tensor in tensors]
        B = len(tensors)
        L = max(sizes)
        D = tensors[0].shape[1]

        tensor = torch.full(
            (B, L, D),
            fill_value=pad_value,
            device=tensors[0].device,
            dtype=tensors[0].dtype,
        )

        for idx, size in enumerate(sizes):
            tensor[idx, :size].copy_(tensors[idx])

        return cls(tensor, None, sizes)

    # -------------------------------------------------------------------------
    # Other methods
    # -------------------------------------------------------------------------

    def to_list(self) -> list[Shaped[Tensor, "L D"]]:
        """Converts the Batched2DTensors to a list of 2D tensors without padding.

        Returns
        -------
        list[Shaped[Tensor, "L D"]]
            A list of 2D tensors.
        """

        return [self.tensor[idx, :size] for idx, size in enumerate(self.sizes)]

    # -------------------------------------------------------------------------
    # Magic methods
    # -------------------------------------------------------------------------

    def __len__(self) -> int:
        """Returns the number of 2D tensors.

        Returns
        -------
        int
            The number of 2D tensors.
        """
        return len(self.sizes)


import sys, os, random
from tqdm import tqdm
import torch
import torch.nn.functional as F
from torch.utils.data import Dataset
from config import DefaultConfig
from torch.utils.data import DataLoader, WeightedRandomSampler
from utils.utils import *

config = DefaultConfig()

# load the data
# token 标识resume 或者job
def load_sents(token):
    sents = {}
    sentnum = {}  # 用于标记已有句子数目
    tensor_size = [config.sent_num[token], config.word_num[token]]  # 每个resume or job 张量大小，维度
    filepath = os.path.join(config.train_path, '{}.sent.id'.format(token))
    time_print('\nloading from {}'.format(filepath))
    sys.stdout.flush()
    with open(filepath, 'r', encoding='utf-8') as file:
        for line in file:
            idx, sent = line.strip().split('\t')
            if idx not in sents:
                sents[idx] = torch.zeros(tensor_size).long()
                sentnum[idx] = 0
            if sentnum[idx] == config.sent_num[token]: continue  # 达到最大句子数，跳转继续读文件中的下一行
            sent = torch.LongTensor([int(x) for x in sent.split(' ')])
            sents[idx][sentnum[idx]] = F.pad(sent, (0, config.word_num[token] - len(sent)))
            sentnum[idx] += 1
    return sents

class PJFDataset(Dataset):
    def __init__(self, r_sents, j_sents, token):
        # token用于标注从train，valid，test文件读
        super(PJFDataset, self).__init__()
        self.r_sents = r_sents
        self.j_sents = j_sents
        self.pairs, self.labels = self.load_pairs(token)

    def load_pairs(self, token):
        pairs = []
        labels = []
        assert token in ['train', 'valid', 'test']
        filepath = os.path.join(config.splitdata_path, 'data.{}'.format(token))
        time_print('\nloading from {}'.format(filepath))
        sys.stdout.flush()
        with open(filepath, 'r', encoding='utf-8') as file:
            for line in file:
                resume_id, job_id, label = line.strip().split('\t')
                # 删除没有数据（sents）的pairs
                if resume_id not in self.r_sents or job_id not in self.j_sents: continue
                pairs.append([resume_id, job_id])
                labels.append(int(label))
        return pairs, torch.FloatTensor(labels)

    def getlabels(self):
        return self.labels

    def __getitem__(self, index):
        pair = self.pairs[index]
        resume_sent = self.r_sents[pair[0]]
        job_sent = self.j_sents[pair[1]]
        label = self.labels[index]
        return pair, resume_sent, job_sent, label

    def __len__(self):
        return len(self.pairs)

if __name__ == '__main__':
    resume_sents = load_sents('resume')
    job_sents = load_sents("job")
    train_dataset = PJFDataset(resume_sents, job_sents, 'train')

    train_loader = DataLoader(
        dataset=train_dataset,
        batch_size=config.batch_size['train'],
        shuffle=True,
        num_workers=config.num_workers['train']
    )
    for i, (geek_sent, job_sent, labels) in enumerate(train_loader):
        if i < 2:
            print(geek_sent.shape)
            #         print(emb(geek_sent).shape)
            print(job_sent.shape)
            print(len(labels))


from threading import Thread
from bs4 import BeautifulSoup as BS
import lxml
import time
import requests

def get_html(link):
    time.sleep(1)
    response = requests.get(link).text
    print(response)

links = ("https://github.com/dashboard", "https://pythontutor.ru/", "https://dzen.ru/?clid=2255080&from=dist_bookmark&win=480&yredirect=true", "https://relizua.com/", "https://travel.yandex.ru/avia/" )

threads = [Thread(target=get_html, args=(links[i],)) for i in range(5)]

start_1 = time.time()
for t in threads:
    t.start()
for t in threads:
    t.join()
end_1 = time.time()

start_2 = time.time()
for link in links:
    get_html(link)
end_2 = time.time()
print(f"Время выполнения параллельного запуска {end_1 - start_1} секунд")
print(f"Время выполнения последовательного запуска {end_2 - start_2} секунд")


# -*- coding: utf-8 -*-
import itertools

from carbi.utils.dyprops import DynamicProperties

def build_targets(orderbook_keys, with_okex_futures=False):
  targets = []
  for orderbook_key in orderbook_keys:
    target = OrderbookTarget()
    target.exchange = orderbook_key.split(":")[0]
    target.type = 'spot'
    target.orderbook_key = orderbook_key
    targets.append(target)
  if with_okex_futures:
    equity_pairs = ['eos/usd', 'etc/usd', 'eth/usd', 'btc/usd', 'bch/usd', 'xrp/usd', 'ltc/usd', 'btg/usd']
    contract_types = ['quarter']
    # 일단은 반드시 필요한 분기 선물만 가져오도록한다.
    # 모두 가져오려고 하면 한번에 너무 많은 요청을 동시에 날리기 떄문에 문제가 된다.
    # contract_types = ['this_week', 'next_week', 'quarter']
    for equity_pair, contract_type in itertools.product(equity_pairs, contract_types):
      target = OrderbookTarget()
      target.exchange = 'okex'
      target.type = 'future'
      target.equity_pair = equity_pair
      target.contract_type = contract_type
      targets.append(target)
    for equity_pair in equity_pairs:
      target = OrderbookTarget()
      target.exchange = 'okex'
      target.type = 'spot'
      target.orderbook_key = 'okex:{}/usdt'.format(equity_pair.split("/")[0])
      targets.append(target)
  return targets

class OrderbookTarget(DynamicProperties):
  def __str__(self):
    if self.type == 'spot':
      return "TARGET[{}]".format(self.orderbook_key)
    if self.type == 'future':
      return "TARGET[{}:f_{}_{}]".format(self.exchange, self.equity_pair, self.contract_type)
    raise NotImplementedError("INVALID ORDERBOOK_TARGET ({})".format(self.type))

  def __repr__(self):
    return self.__str__()


import errno
import json
import os.path

import tornado.ioloop
import tornado.web

import GamesManager
import Sessions
import utils
from ActionHandler import ActionHandler
from AuthHandlers import LoginHandler, LogoutHandler, BaseAuthHandler
from GameBoard import BoardState
from GameConnection import GameConnection, GameSocketRouter
from GameHandler import GameHandler
from LobbyConnection import LobbySocketRouter, LobbyConnection
from LobbyHandlers import (
    LobbyHandler,
    LobbyCreateHandler,
    LobbyGameHandler,
    LobbyGameInfoHandler,
    LobbyGameJoinHandler,
    LobbyGameBeginHandler,
    LobbyGameUpdateHandler,
    LobbyGameDeleteHandler,
)
from MapEditorHandler import MapEditorHandler
from ModuleListHandler import ModuleListHandler
from SaveHandler import SaveHandler
from UserInfoHandlers import UserInfoHandler

class IndexHandler(tornado.web.RequestHandler):
    def initialize(self, html_path):
        self.HTML_PATH = html_path

    def get(self):
        self.redirect("/lobby")

class BoardsHandler(BaseAuthHandler):
    #TODO: Consider spliting this class to handle the different scenarios
    actions = utils.Enum(["NEW", "ID", "GET_FIELDS"])
    nextBoardId = 0

    def initialize(self, config, action, gamesManager):
        super(BoardsHandler, self).initialize(config=config)
        self.config = config
        self.action = action
        self.gamesManager = gamesManager

        self.setAuthenticateUser = config.USER_AUTH

    @tornado.web.authenticated
    def get(self, **params):
        if self.action == self.actions.ID:
            #Return info about board with id boardId
            game = self.gamesManager.getGame(int(params["boardId"]))
            if not game:
                self.set_status(404)
                self.write("Board not found")
                return

            board = game.board

            # Return the board info as json
            boardInfo = BoardState.exportBoardToClient(board)

            #See if it is the user's turn
            userSession = Sessions.SessionManager.getSession(self.current_user)
            boardInfo["isPlayerTurn"] = board.isPlayersTurn(userSession.getValue("userid"))

            self.write(json.dumps(boardInfo))

        elif self.action == self.actions.GET_FIELDS:
            requestedFields = [f.decode("utf-8") for f in self.request.arguments.get("fieldNames[]")]
            board = self.gamesManager.getBoard(int(params["boardId"]))
            response = BoardState.getFields(board, requestedFields)
            self.write(json.dumps(response))

    @tornado.web.authenticated
    def post(self, **params):
        self.set_status(405)
        self.write("Method Not Allowed")
        return

def createSaveFileIfMissing():
    # http://stackoverflow.com/a/10979569
    flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY

    try:
        file_handle = os.open('saveGames.json', flags)
    except OSError as e:
        if e.errno == errno.EEXIST:  # Failed as the file already exists.
            pass
        else:  # Something unexpected went wrong so reraise the exception.
            raise
    else:  # No exception, so the file must have been created successfully.
        with os.fdopen(file_handle, 'w') as file_obj:
            # Using `os.fdopen` converts the handle to an object that acts like a
            # regular Python file object, and the `with` context manager means the
            # file will be automatically closed when we're done with it.
            file_obj.write("{}")

class Server:
    def __init__(self, config):
        html_path = os.path.join(config.STATIC_CONTENT_PATH, "html")
        port = 8888
        gameSocketRouter = GameSocketRouter(GameConnection, '/gameStream')
        lobbySocketRouter = LobbySocketRouter(LobbyConnection, '/lobbyStream')

        self.gamesManager = GamesManager.GamesManager()

        createSaveFileIfMissing()

        def saveHandlerOptions(action):
            baseOptions = dict(config=config, gamesManager=self.gamesManager, html_path=html_path)
            baseOptions["action"] = action
            return baseOptions
        self.app = tornado.web.Application([
            (r"/", IndexHandler, dict(html_path=html_path)),

           # module handler
           (r"/modules/list", ModuleListHandler, dict(config=config)),

            #Map editor (Consider moving into a sub-list in MapEditorHandler)
            (r"/mapEditor", MapEditorHandler, dict(config=config, action=MapEditorHandler.actions.MODULE_SELECTOR, html_path=html_path)),
            (r"/mapEditor/(?P<moduleName>[A-z]+)", MapEditorHandler, dict(config=config, action=MapEditorHandler.actions.PAGE, html_path=html_path)),
            (r"/mapEditor/modules/info/(?P<moduleName>[A-z]+)", MapEditorHandler, dict(config=config, action=MapEditorHandler.actions.MODULE_INFO, html_path=html_path)),
            (r"/modules/create/?", MapEditorHandler, dict(config=config, action=MapEditorHandler.actions.CREATE)),
            (r"/modules/(?P<moduleName>[A-z]+)", MapEditorHandler, dict(config=config, action=MapEditorHandler.actions.MODULE)),

            (r"/listImages", MapEditorHandler, dict(config=config, action=MapEditorHandler.actions.LIST_IMAGES, html_path=html_path)),

            #Board control
            #Consider renaming to /games/
            (r"/boardInfo/(?P<boardId>[0-9]+)/?", BoardsHandler, dict(config=config, action=BoardsHandler.actions.ID, gamesManager=self.gamesManager)), #Consider using named regex here
            (r"/getFields/(?P<boardId>[0-9]+)/?", BoardsHandler, dict(config=config, action=BoardsHandler.actions.GET_FIELDS, gamesManager=self.gamesManager)),
            (r"/boards/(?P<boardId>[0-9]+)/action/?", ActionHandler, dict(config=config, gamesManager=self.gamesManager, gameSocket=gameSocketRouter)),

            #Serve the static game page
            (r"/game/?", GameHandler, dict(config=config)),
            (r"/game/(?P<boardId>[0-9]+)/?", GameHandler, dict(config=config, gamesManager=self.gamesManager)),

            #User auth
            (r"/login/?", LoginHandler),
            (r"/logout/?", LogoutHandler),

            #Lobby web routes
            (r"/lobby/?", LobbyHandler, dict(config=config, gamesManager=self.gamesManager)),
            (r"/lobby/create/?", LobbyCreateHandler, dict(config=config, gamesManager=self.gamesManager)),
            (r"/lobby/(?P<gameId>[0-9]+)/?", LobbyGameHandler, dict(config=config, gamesManager=self.gamesManager)),

            ##API ROUTES
            #Lobby API routes
            (r"/lobby/(?P<gameId>[0-9]+)/info/?", LobbyGameInfoHandler, dict(config=config, gamesManager=self.gamesManager)),
            (r"/lobby/(?P<gameId>[0-9]+)/join/?", LobbyGameJoinHandler, dict(config=config, gamesManager=self.gamesManager, lobbySocket=lobbySocketRouter)),
            (r"/lobby/(?P<gameId>[0-9]+)/begin/?", LobbyGameBeginHandler, dict(config=config, gamesManager=self.gamesManager, lobbySocket=lobbySocketRouter)),
            (r"/lobby/(?P<gameId>[0-9]+)/update/?", LobbyGameUpdateHandler, dict(config=config, gamesManager=self.gamesManager, lobbySocket=lobbySocketRouter)),
            (r"/lobby/(?P<gameId>[0-9]+)/delete/?", LobbyGameDeleteHandler, dict(config=config, gamesManager=self.gamesManager)),

            (r"/userinfo/?", UserInfoHandler, dict(config=config)),
            ##END API ROUTES

            #Load/Save games
            (r"/save/?", SaveHandler, saveHandlerOptions(SaveHandler.actions.SAVE)),
            (r"/load/(?P<saveGameId>[A-z0-9\-]+)/?", SaveHandler, saveHandlerOptions(SaveHandler.actions.LOAD)),
            (r"/saves/?", SaveHandler, saveHandlerOptions(SaveHandler.actions.LIST_SAVES)),
            (r"/saves/(?P<saveGameId>[A-z0-9\-]+)/delete/?", SaveHandler, saveHandlerOptions(SaveHandler.actions.DELETE)),

            #Static files
            (r"/shared/(.*)", utils.NoCacheStaticFileHandler, {"path": config.SHARED_CONTENT_PATH}),
            (r"/static/(.*)", utils.NoCacheStaticFileHandler, {"path": config.STATIC_CONTENT_PATH}), #This is not a great way of doing this TODO: Change this to be more intuative
        ] + gameSocketRouter.urls + lobbySocketRouter.urls,
        cookie_secret=config.COOKIE_SECRET,
        login_url="/login",
        debug=True
        )

        self.app.listen(port)
        print("Listening on port {}".format(port))
        self.ioloop = tornado.ioloop.IOLoop.instance()

    def start(self):
        print("Starting Server")
        self.ioloop.start()


from collections import defaultdict
from index_map import IndexMap
import utils
from trie import Trie
from queue import Queue
import numpy as np
import argparse
import time

class LM:
  """Represents the main entry point of the language model"""

  def __init__(self, data_train, vocabs_file=None):

    self.data_train = data_train

    self.corpus = IndexMap()
    if vocabs_file:
      self.vocabs = IndexMap(vocabs_file)

    self.running_wrds_num = 0
    self.sent_num = 0
    self.sent_len_sum = 0
    self.sent_len_to_freq = defaultdict(int)
    self.prepare_lm()
    self.avg_sent_len = (1.0 * self.sent_len_sum)/self.sent_num

    self.unk_cnt = 0
    self.oov = 0.0 # out-of-vocabulary rate

    # NOTE: The purpose of creating this dict of roots is only
    # for the following experiment: Comparing the computation of bigrams
    # and unigrams probs using trigram counts vs the computation of bigrams
    # and unigrams probs using their own tries. You should not create multiple
    # trie trees usually since you have prefix information

    self.ngrams_root = {} # contains the root of (key)-gram trie

    self.b = {} # contain b discounting params of (gram_rank -> b)

  def prepare_lm(self):
    """Prepare the language model for analysis and computations"""

    print('Reading corpus for analysis...')

    with open(self.data_train, 'r') as read_corpus:
      for line in read_corpus:
        words = line.strip().split(' ')
        sent_len = len(words)
        self.sent_len_sum += sent_len
        self.sent_len_to_freq[sent_len] += 1
        self.sent_num += 1
        for wrd in words:
          self.corpus.add_wrd(wrd)
          self.running_wrds_num += 1

  ################################ Ex1 (a)-(b)-(c) ################################

  def show_corpus_analysis(self):
    """Show corpus data analysis"""

    print('Size of vocabulary: %d' % self.vocabs.get_num_of_words())
    print('Number of running words: %d' % self.running_wrds_num)
    print('Number of sentences: %d' % self.sent_num)
    print('Average sentence length: %.2f' % self.avg_sent_len)

    utils.plot_kv_iterable(self.sent_len_to_freq,
                           xlabel='Sentence length',
                           ylabel='Frequency',
                           xticks=5)

  ################################ Ex2 (a) ################################

  def get_word_freq_dict(self):
    """Return a dict where key is a word and value is it's frequency

      It transforms the wrd_freq dict from index-based key to word-based key
      so that it can be used for retrieving the top k frequent words

    :return: A dict, word to freq mapping
    """

    res = {}
    for idx, freq in self.corpus.get_wrd_freq_items():
      wrd = self.corpus.get_wrd_by_idx(idx)
      res[wrd] = freq
    return res

  @staticmethod
  def get_top_10_freq_words(res):
    """Return the top 10 frequent words

    :return: A list, top 10 frequent words
    """

    return utils.get_top_k_freq_items(res, k=10)

  ############################# Ex2 (b)-(c)-(d)-(e) ####################################

  def get_corpus_tokens(self, vocabulary):
    """Read corpus and store the tokens in a list

    If with_unk is True, then unk_cnt is used to store the number
    of OOV words to compute it's rate later

    :param vocabulary: IndexMap, either corpus or vocabs
    :return: A list of word tokens with start and end symbols
    """

    tokens = []
    start_id = vocabulary.get_start_id()
    end_id = vocabulary.get_end_id()
    unk_id = vocabulary.get_unk_id()
    with open(self.data_train, 'r') as read_corpus:
      for line in read_corpus:
        sent = line.strip().split(' ')
        tokens.append(start_id)
        for wrd in sent:
          idx = vocabulary.get_idx_by_wrd(wrd)
          if idx == unk_id:
            self.unk_cnt += 1
          tokens.append(idx)
        tokens.append(end_id)
    return tokens

  def generate_ngrams(self, n, vocabulary):
    """Generate ngrams from the given vocabulary (corpus or vocabs)
       and store them in a Trie data structure

    :param n: An integer, the rank of the grams that are generated
    :param vocabulary: An IndexMap, either corpus or vocabs
    :return: A Trie representing the ngrams
    """

    start_id = vocabulary.get_start_id()
    end_id = vocabulary.get_end_id()
    unk_id = vocabulary.get_unk_id()

    print('Generating %d-grams from %s' % (n, 'corpus' if vocabulary == self.corpus else 'vocabulary'))

    self.ngrams_root[n] = Trie()
    with open(self.data_train, 'r') as read_corpus:
      for line in read_corpus:
        sent = line.strip().split(' ')
        ngram = [start_id]
        for wrd in sent:
          if len(ngram) == n:
            self.ngrams_root[n].add_ngram(ngram)
            ngram = ngram[1:]
          idx = vocabulary.get_idx_by_wrd(wrd)
          if idx == unk_id:
            self.unk_cnt += 1
          ngram.append(idx)
        if len(ngram) == n:
          self.ngrams_root[n].add_ngram(ngram)
          ngram = ngram[1:]
        ngram.append(end_id)
        self.ngrams_root[n].add_ngram(ngram)

    print('%d-grams are now stored in a Trie' % n)

    if vocabulary != self.corpus:
      self.oov = (self.unk_cnt / self.running_wrds_num) * 100.0
      print('OOV rate: %.02f %%' % self.oov)

  def extract_ngrams_and_freq(self, n, vocabulary):
    """Extract ngrams and their frequencies, display top 10 frequent ngrams,
      plot the count of counts distribution

    :param n: An integer, the rank of the gram
    :param vocabulary: An indexMap, either corpus or vocabs
    """

    if n not in self.ngrams_root:
      self.generate_ngrams(n, vocabulary)

    print('Extracting %d-grams with their frequencies using %s' % \
          (n, 'corpus' if vocabulary == self.corpus else 'vocabulary'))

    res = self.ngrams_root[n].bfs(n)

    print('Extraction is done.')

    if n not in self.b:
      self.compute_b(n, vocabulary=self.corpus)
      print('Discounting parameters:')
      for k, v in self.b.items():
        print('b_{} = {}'.format(k, v))

    return res

  def get_top_10_ngram_freq(self, n, vocabulary):
    """Return the top 10 frequent ngrams

    :param res: A dict, keys are ngrams and values are counts
    :return: A list of top 10 frequent ngrams
    """

    top_10_ngrams = utils.get_top_k_freq_items(self.extract_ngrams_and_freq(n, vocabulary), k=10)

    # map to words
    res = []
    for kv in top_10_ngrams:
      ngram_wrds = []
      for idx in kv[0]:
        ngram_wrds.append(vocabulary.get_wrd_by_idx(idx))
      res.append((ngram_wrds, kv[1]))
    return res

  ############################# Ex3 ####################################

  def get_summed_counts(self, n, vocabulary):
    """Calculate bi-/uni- grams from trigrams and compare to directly extracted bi-/uni- grams

    :param n: An integer, the rank of the source n-gram
    :param vocabulary: An indexMap, either corpus or vocabs
    """

    assert n >= 3, 'n should be at least 3'

    res_trigram = self.extract_ngrams_and_freq(n, vocabulary)
    res_bi = self.extract_ngrams_and_freq(n-1, vocabulary)
    res_uni = self.extract_ngrams_and_freq(n-2, vocabulary)

    summed_bigrams = defaultdict(int)
    summed_unigrams = defaultdict(int)

    for k, v in res_trigram.items():
      summed_bigrams[k[1:]] += v
      summed_unigrams[k[-1]] += v

    print("bigram difference: ")
    for k, v in res_bi.items():
      if v != summed_bigrams[k]:
        print("bigram: {} extracted value: {} summed value: {}".format(k, v ,summed_bigrams[k]))

    print("unigram difference: ")
    for k, v in res_uni.items():
      if v != summed_unigrams[k]:
        print("unigram: {} extracted value: {} summed value: {}".format(k, v, summed_unigrams[k]))

  ########################### Ex4 ##################################

  def compute_b(self, n, vocabulary):
    """Computes the discounting parameters for up to n-gram
       e.g if n = 2, then it will compute b_uni, and b_bi

    :param n: An integer, the rank of gram
    :param vocabulary: An IndexMap, either corpus or vocabs
    """

    if n in self.b: return

    print('Computing discounting parameters...')

    # ngrams are not added to the trie yet
    if n not in self.ngrams_root:
      self.generate_ngrams(n, vocabulary)

    q = Queue()
    q.put(self.ngrams_root[n]) # add root
    for i in range(n):
      singeltons = 0
      doubletons = 0
      next_q = Queue()
      while not q.empty():
        u = q.get()
        for idx, child in u.get_children().items():
          next_q.put(child)
          if child.get_freq() == 1:
            singeltons += 1
          elif child.get_freq() == 2:
            doubletons += 1
      self.b[i+1] = (singeltons/(singeltons + 2.0 * doubletons))
      q = next_q

  def compute_prob(self, w, h, n):
    """Computes the bigram probability p(w|h) using absolute discounting with
       interpolation where h is word history

    :param w: An integer, the index of word w
    :param h: A list containing the indexes of word history
    :param n: An integer, the rank of the grams
    :return: A float, p(w|h)
    """

    # backoff to unigram (base case)
    if len(h) == 0:
      prob = self.b[1]
      prob *= self.ngrams_root[n].get_num_of_children() # W - N_0(.)
      prob /= float(self.vocabs.get_num_of_words() * self.ngrams_root[n].get_freq()) # W * N

      w_node = self.ngrams_root[n].get_ngram_last_node([w])
      if w_node is not None:
        w_freq = w_node.get_freq()
        prob += max(float(w_freq - self.b[1]) / self.ngrams_root[n].get_freq(), 0.0)

      return prob

    h_node = self.ngrams_root[n].get_ngram_last_node(h)

    # history is not found so backoff
    if h_node is None:
      return self.compute_prob(w, h[1:], n)

    prob = self.b[len(h)+1]
    prob *= float(h_node.get_num_of_children()) / h_node.get_freq() # (W - N_0(v,.))/N(v)
    prob *= self.compute_prob(w, h[1:], n) # recursively backoff

    # add the first term of the interpolation
    w_node = h_node.get_ngram_last_node([w])
    if w_node is not None:
      w_h_freq = w_node.get_freq()
      prob += max(float(w_h_freq - self.b[len(h)+1]) / h_node.get_freq(), 0.0)

    return prob

  def verify_normalization(self, n):
    """Verify the normalization of bigram and unigram probabilities

    :return: True if the probabilities are normalized and False otherwise
    """

    print('Verifying probability normalization...')

    bigram_probs = 0.0
    unigram_probs = 0.0
    for w in range(0, self.vocabs.get_num_of_words()):
      bigram_probs += self.compute_prob(w, [10], n) # any word for history
      unigram_probs += self.compute_prob(w, [], n)

    print('bigram_probs: {}, unigram_probs: {}'.format(bigram_probs, unigram_probs))
    return abs(1.0 - bigram_probs) <= 1e-02 and abs(1.0 - unigram_probs) <= 1e-02

  ########################### Ex5 ##################################

  def perplexity(self, corpus_file, n):
    """Compute the perplexity of the language model on the given corpus
       using n-grams

    :param corpus_file: A string, the path of the corpus
    :param n: An integer, the rank of the grams to be used for computing PP
    :return: A float, perplexity of the LM
    """

    print('Computing model perplexity...')

    LL = 0.0
    norm = 0
    with open(corpus_file, 'r') as test_corpus:
      for line in test_corpus:
        sent = line.strip().split()
        h = [self.corpus.get_start_id()]
        for wrd in sent:
          w = self.corpus.get_idx_by_wrd(wrd)
          if len(h) == n-1:
            prob = self.compute_prob(w, h, n)
            LL += np.log(prob)
            h.append(w)
            h = h[1:]
          else:
            h.append(w)
        LL += np.log(self.compute_prob(self.corpus.get_end_id(), h, n))
        norm += len(sent)+1
    return np.exp(-LL/norm)

#######################################################################

if __name__ == '__main__':
  parser = argparse.ArgumentParser()

  parser.add_argument('--train_file', type=str, default='./data/train.corpus',
                      help='training data path')
  parser.add_argument('--vocab_file', type=str, default='./data/vocabulary',
                      help='vocabulary file path')
  parser.add_argument('--test_file', type=str, default='./data/test.corpus',
                      help='test file.')
  parser.add_argument('--gram_rank', type=int, default=2,
                      help='e.g 2 for bigram')

  args = parser.parse_args()

  start = time.time()

  lm = LM(args.train_file, args.vocab_file)
  lm.extract_ngrams_and_freq(n=args.gram_rank, vocabulary=lm.corpus)
  # if lm.verify_normalization(args.gram_rank):
  #   print('Probabilities are normalized!')
  # else:
  #   print('Probabilities are not normalized!')
  print('Test PP: {}'.format(lm.perplexity(corpus_file=args.test_file, n=args.gram_rank)))
  print('Execution time: %.3f min' % ((time.time()-start)/60))


# -*- coding: utf-8 -*-
"""
Created on Wed Mar 10 17:38:32 2021

@author: Not Your Computer
"""

import os
import numpy as np
import pandas as pd
import random
import itertools
from scipy.ndimage.filters import convolve
from skimage.transform import resize, rescale
import matplotlib.pyplot as plt
from PIL import Image

def Invert(img):
    img = img + 1
    img[np.where(img == 2)] = 0
    return img

def RemoveStrayPixels(img, k = 3):
    h,w = np.shape(img)
    rows = []
    cols = []
    for i in range(k,h-k):
        for j in range(k,w-k):
            if img[i,j] == 1:
                if np.sum(img[i-k:i+k,j-k:j+k]) == 1:
                    rows.append(i)
                    cols.append(j)
    img[rows,cols] = 0
    return img

# Calculate bounds occupied by pixels
def PixelBounds(img, buffer = 10, k = 3):
    h,w = np.shape(img)
    img = RemoveStrayPixels(img, k = k)
    xmin = np.where(img.sum(axis = 0) > 0)[0][0] - buffer
    ymin = np.where(img.sum(axis = 1) > 0)[0][0] - buffer
    xmax = np.where(img.sum(axis = 0) > 0)[0][-1] + buffer
    ymax = np.where(img.sum(axis = 1) > 0)[0][-1] + buffer
    return max(0,xmin), max(0,ymin), min(xmax,w), min(ymax,h)

def IOU(boxA, boxB):
	# determine the (x, y)-coordinates of the intersection rectangle
	xA = max(boxA[0], boxB[0])
	yA = max(boxA[1], boxB[1])
	xB = min(boxA[2], boxB[2])
	yB = min(boxA[3], boxB[3])
	# compute the area of intersection rectangle
	interArea = max(0, xB - xA + 1) * max(0, yB - yA + 1)
	# compute the area of both the prediction and ground-truth
	# rectangles
	boxAArea = (boxA[2] - boxA[0] + 1) * (boxA[3] - boxA[1] + 1)
	boxBArea = (boxB[2] - boxB[0] + 1) * (boxB[3] - boxB[1] + 1)
	# compute the intersection over union by taking the intersection
	# area and dividing it by the sum of prediction + ground-truth
	# areas - the interesection area
	iou = interArea / float(boxAArea + boxBArea - interArea)
	# return the intersection over union value
	return iou

# Save all logged annotations as image chips in separate folder
def SaveChips(entry, outdir = r'D:\DataStuff\bms\chips'):
    entry = entry.split(',')
    path = entry[0]
    fname = os.path.basename(os.path.dirname(path))
    n_entries = int(len(entry[3:]) / 9)
    img = plt.imread(path)
    for i in range(n_entries):
        x,y,w,h,label,pxmin,pymin,pxmax,pymax = entry[3+9*i:3+9*(i+1)]
        x = int(float(x)); y = int(float(y))
        w = int(float(w)/2); h = int(float(h)/2)

        chip = img[y-h:y+h, x-w:x+w]
        plt.imshow(chip, cmap = 'gray')
        plt.show()
        outfile = '{}\\{}_{}_{}.png'.format(outdir, fname, label, i)
        plt.imsave(outfile, chip, cmap = 'gray')

# Create a new image w/ image chips
def AlphabetSoup(labels,
                 srcdir = r'D:\DataStuff\bms\chips',
                 target_shape = (640,640),
                 output_type = 'yolo',
                 size_thresh = 100,
                 preprocess = True,
                 min_size = 10,
                 buffer = 100,
                 S = (10,10),
                 B = 1,
                 C = 0):

    flist = os.listdir(srcdir)
    sf = random.random() * 1.5 + 1
    img = np.zeros(target_shape)
    boxes = []

    if output_type == 'yolo':
        annotations = np.zeros((S[0], S[1], (5*B+C)))
        grid_h = target_shape[0] / S[0]
        grid_w = target_shape[1] / S[1]

    if output_type == 'unet':
        mask = np.zeros(target_shape)

    for fname in random.sample(flist, random.randint(1,15)):
        fpath = os.path.join(srcdir, fname)
        label = fpath.split('.')[0].split('_')[-2]
        chip = Invert(plt.imread(fpath)[:,:,0])

        x = int(random.random() * (target_shape[1] - 2*buffer)) + buffer
        y = int(random.random() * (target_shape[0] - 2*buffer)) + buffer

        # Convolve the image to flesh out broken lines
        if preprocess:
            chip = convolve(chip, [[1,1,1],[1,1,1],[1,1,1]])
            chip = chip / np.max(chip)
            chip = convolve(chip, [[1,1,1],[1,1,1],[1,1,1]])
            chip = chip / np.max(chip)

        chip = rescale(chip, sf)
        h,w = np.shape(chip)
        chip_box = [x,y,x+w,y+h]

        no_overlap = True
        for box in boxes:
            if IOU(chip_box, box) > 0.10:
                no_overlap = False
                break

        if no_overlap:
            img[y:y+h, x:x+w] = chip
            boxes.append(chip_box)

            if output_type == 'yolo':
                row = int(np.floor(y / grid_h))
                col = int(np.floor(x / grid_w))

                x = x - col*grid_w
                y = y - row*grid_h

                x = x / grid_w
                y = y / grid_h

                h = min(h/(size_thresh-min_size),1)
                w = min(w/(size_thresh-min_size),1)
                annotations[row,col,:] = [x,y,w,h,1]

            if output_type == 'unet':
#                mask[y:y+h,x:x+w] = labels.index(label)
                mask[y:y+h,x:x+w] = 1

    if output_type == 'yolo':
        return img, annotations

    if output_type == 'unet':
        return img, mask

#
## Create a new image w/ image chips
#def AlphabetSoup(srcdir = r'D:\DataStuff\bms\chips', target_shape = (640,640), size_thresh = 100, buffer = 100, S = (10,10), B = 1, C = 0, preprocess = True, min_size = 10):
#    flist = os.listdir(srcdir)
#    fnames = random.sample(flist, random.randint(1,15))
#    sf = random.random() * 1 + 1
#    img = np.zeros(target_shape)
#    boxes = []
#    annotations = np.zeros((S[0], S[1], (5*B+C)))
#    grid_h = target_shape[0] / S[0]
#    grid_w = target_shape[1] / S[1]
#
#    for fname in fnames:
#        fpath = os.path.join(srcdir, fname)
#        label = fpath.split('.')[0].split('_')[-2]
#        chip = Invert(plt.imread(fpath)[:,:,0])
#
#        x = int(random.random() * (target_shape[1] - 2*buffer)) + buffer
#        y = int(random.random() * (target_shape[0] - 2*buffer)) + buffer
#
#        # Convolve the image to flesh out broken lines
#        if preprocess:
#            chip = convolve(chip, [[1,1,1],[1,1,1],[1,1,1]])
#            chip = chip / np.max(chip)
#            chip = convolve(chip, [[1,1,1],[1,1,1],[1,1,1]])
#            chip = chip / np.max(chip)
#
#        chip = rescale(chip, sf)
#        h,w = np.shape(chip)
#
#        chip_box = [x,y,x+w,y+h]
#
#        no_overlap = True
#        for box in boxes:
#            if IOU(chip_box, box) > 0.10:
#                no_overlap = False
#                break
#
#        if no_overlap:
#            img[y:y+h, x:x+w] = chip
#            boxes.append(chip_box)
#            row = int(np.floor(y / grid_h))
#            col = int(np.floor(x / grid_w))
#
#            x = x - col*grid_w
#            y = y - row*grid_h
#
#            x = x / grid_w
#            y = y / grid_h
#
#            h = min(h/(size_thresh-min_size),1)
#            w = min(w/(size_thresh-min_size),1)
#
#            annotations[row,col,:] = [x,y,w,h,1]
#
#    return img, annotations


from flask import (Flask, g, render_template, flash, redirect, url_for, request, make_response, jsonify)
from flask.ext.login import (LoginManager, login_user, logout_user, login_required, current_user)
from flask.ext.bcrypt import check_password_hash
from blueprints.usuarios import usuarios
from blueprints.competencias import competencias
from blueprints.preguntas import preguntas
from blueprints.respuestas import respuestas
import models
from werkzeug import secure_filename
import os
import json
import ast
DEBUG = True
PORT = 5000
HOST = '127.0.0.1'
UPLOAD_FOLDER = os.path.dirname(os.path.abspath(__file__)) + '/static/media/uploads'
app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.secret_key = 'niggersan'
app.register_blueprint(usuarios, url_prefix='/api/usuarios')
app.register_blueprint(competencias, url_prefix='/api/competencias')
app.register_blueprint(preguntas, url_prefix='/api/preguntas')
app.register_blueprint(respuestas, url_prefix='/api/respuestas')
login_manager = LoginManager()
login_manager.init_app(app)
login_manager.login_view = '/'

@login_manager.user_loader
def cargar_usuario(usuarioid):
    try:
        return models.Usuario.get(models.Usuario.id == usuarioid)
    except models.DoesNotExist:
        return None

@app.before_request
def antes_request():
	"""Conectarse a la base de datos """
	g.db = models.DATABASE
	g.db.connect()
	g.user = current_user

@app.after_request
def despues_request(response):
	"""Cerrar la conexion despues de cada request"""
	g.db.close()
	return response

@app.route('/')
def main():
	usuario=g.user._get_current_object()
	#return make_response(open('templates/index.html').read())
	return render_template('index.html')

@app.route('/usuario')
@login_required
def get_usuario():
	return json.dumps(g.user._get_current_object().to_json())

####
####    seleccionar a los que te permitiran evaluarlos
####

@app.route('/seleccionar_colegas')
@login_required
def seleccionar_colegas():

	return render_template('seleccionar_colegas.html')

@app.route('/colegas')
@login_required
def colegas():
	usuarios = models.Usuario.select().where(
		(models.Usuario.puesto==g.user.puesto) &
		(models.Usuario.id != g.user.id) )
	colegas = list()
	for usuario in usuarios:
		colegas.append(usuario.to_json())
	return json.dumps(colegas)

@app.route('/get_colegas_evaluar')
@login_required
def get_colegas():
	evaluadores = models.Usuario.get(models.Usuario.id**g.user._get_current_object().id)
	colegas = list()
	for evaluador in evaluadores.evaluan():
		colegas.append(evaluador.to_json())
	return json.dumps(colegas)

@app.route('/get_subordinados')
@login_required
def get_subordinados():
	usuario = g.user._get_current_object()
	subordinados = list()
	for subordinado in usuario.subordinados():
		subordinados.append(subordinado.to_json())
	return json.dumps(subordinados)

@app.route('/get_superiores')
@login_required
def get_superiores():
	usuario = g.user._get_current_object()
	superiores = list()
	for superior in usuario.superiores():
		superiores.append(superior.to_json())
	return json.dumps(superiores)

@app.route('/puede_evaluarme', methods=['POST'])
@login_required
def puede_evaluarme():
	colega = request.get_json()
	get_modelo = models.Usuario.get(models.Usuario.nombre**colega['nombre'])
	models.PermisoEvaluar.create(
		evaluado=g.user._get_current_object(),
		evaluador=get_modelo
		)
	string = "{} ahora podra ser evaluada por {}".format(g.user.nombre, colega['nombre'])
	return json.dumps({'response': string})

@app.route('/numero_evaluadores')
@login_required
def numero_evaluadores():
	numero = models.PermisoEvaluar.select().where(models.PermisoEvaluar.evaluado==g.user._get_current_object()).count()
	ids = models.PermisoEvaluar.select().where(models.PermisoEvaluar.evaluado==g.user._get_current_object())

	lista_ids = list()
	for Id in ids:
		lista_ids.append(Id.evaluador.rpe)
	return json.dumps({
	                  "cantidad": numero,
	                  "seleccionados": lista_ids,
	                  })

@app.route('/evaluando', methods=['POST'])
@login_required
def evaluando():
	evaluacion = request.get_json()
	usuario = g.user._get_current_object()
	colega = models.Usuario.get(models.Usuario.nombre**evaluacion['colega']['nombre'])
	competencia = models.Competencias.get(models.Competencias.nombre**evaluacion['competencia']['nombre'])
	tipo = models.TipoCompetencia.get(models.TipoCompetencia.nombre**evaluacion['tipo'])
	models.Evaluando.nuevo(
		empleado=usuario,
		colega=colega,
		competencia=competencia,
		tipo=tipo
		)

	return json.dumps(
		{
		"response" : "{} evaluo a {} decidiendo que la competencia: {} es {}".format(
			usuario.nombre, colega.nombre, competencia.nombre, tipo.nombre)
		}
	)

@app.route('/evaluando_neutras', methods=['POST'])
@login_required
def evaluando_neutras():

	evaluacion = request.get_json()

	for competencia in evaluacion['competencias']:
		usuario = g.user._get_current_object()
		colega = models.Usuario.get(models.Usuario.nombre**evaluacion['colega']['nombre'])
		competencia = models.Competencias.get(models.Competencias.nombre**competencia['nombre'])
		tipo = models.TipoCompetencia.get(models.TipoCompetencia.nombre**'Neutras')

		models.Evaluando.nuevo(
			empleado=usuario,
			colega=colega,
			competencia=competencia,
			tipo=tipo

			)
	return "ok"

# @app.route('/nuevo', methods['POST'])
# def registrarUsuario():
# 	try:
# 		Usuario.nuevo(
# 	    	rpe=request.form['rpe'],
# 	    	nombre=request.form['nombre'],
# 	    	puesto=request.form['puesto'],
# 	    	departamento=request.form['departamento'],
# 	    	correo=request.form['correo'],
# 	    	zona=request.form['zona'])
# 		return "OK!", 200
# 	except:
# 		return "Error!", 404

@app.route('/iniciar_sesion', methods=['POST'])
def iniciar_sesion():
	print(request.form['correo'])
	usuario = models.Usuario.get(models.Usuario.correo**request.form['correo'])
	if request.form['password'] == usuario.rpe:
		session['login_user(usuario)']=True
		status= True
		return 'Login successfull'
	else:
		status = False
		return 'Login failed.'

@app.route('/get_competencias')
@login_required
def get_competencias():
	competencias = models.Competencias.select()
	lista_competencias = list()
	for competencia in competencias:
		lista_competencias.append(competencia.to_json())
	return json.dumps(lista_competencias)

@app.route('/get_competencias/existentes', methods=['POST'])
@login_required
def existentes():
	respuesta = list()
	resultados = list()
	peticion = request.get_json()
	empleado = g.user._get_current_object()
	tipo = models.TipoCompetencia.get(models.TipoCompetencia.nombre ** peticion['tipo'])
	colega = models.Usuario.get(models.Usuario.nombre**peticion['colega']['nombre'])
	evaluaciones = models.Evaluando.select().where(
	    (models.Evaluando.empleado == g.user._get_current_object()) &
		(models.Evaluando.colega == colega) &
		(models.Evaluando.tipo == tipo)
		)
	for evaluacion in evaluaciones:
		resultados.append(models.Competencias.get(models.Competencias.id**evaluacion.competencia))
	for resultado in resultados:
		respuesta.append(resultado.to_json())
	return json.dumps(respuesta)

@app.route('/get_competencias/inexistentes', methods=['POST'])
@login_required
def inexistentes():
	respuesta = list()
	resultados = list()
	peticion = request.get_json()
	empleado = g.user._get_current_object()
	tipo = models.TipoCompetencia.get(models.TipoCompetencia.nombre ** peticion['tipo'])
	colega = models.Usuario.get(models.Usuario.nombre**peticion['colega']['nombre'])
	evaluaciones = models.Evaluando.select().where(
	    (models.Evaluando.empleado == g.user._get_current_object()) &
		(models.Evaluando.colega == colega) &
		(models.Evaluando.tipo == tipo)
		)
	for evaluacion in evaluaciones:
		resultados.append(models.Competencias.get(models.Competencias.id**evaluacion.competencia))
	for resultado in resultados:
		respuesta.append(resultado.to_json())
	return json.dumps(respuesta)

@app.route('/get_competencias/neutras', methods=['POST'])
@login_required
def neutras():
	respuesta = list()
	resultados = list()
	peticion = request.get_json()
	existente = models.TipoCompetencia.get(models.TipoCompetencia.nombre ** 'Existentes')
	inexistente = models.TipoCompetencia.get(models.TipoCompetencia.nombre ** 'Inexistentes')

	colega = models.Usuario.get(models.Usuario.nombre**peticion['colega']['nombre'])
	evaluaciones_inexistentes = models.Evaluando.select().where(
	    (models.Evaluando.empleado == g.user._get_current_object()) &
		(models.Evaluando.colega == colega) &
		(models.Evaluando.tipo == inexistente)
		)
	evaluaciones_existentes = models.Evaluando.select().where(
	    (models.Evaluando.empleado == g.user._get_current_object()) &
		(models.Evaluando.colega == colega) &
		(models.Evaluando.tipo == existente)
		)
	for evaluacion in evaluaciones_existentes:
		resultados.append(models.Competencias.get(models.Competencias.id**evaluacion.competencia))

	for evaluacion in evaluaciones_inexistentes:
		resultados.append(models.Competencias.get(models.Competencias.id**evaluacion.competencia))

	for resultado in resultados:
		respuesta.append(resultado.to_json())

	return json.dumps(respuesta)

@app.route('/reportes')
@login_required
def reportes():
	competencias = models.Competencias.select()
	tipocompetencia = models.TipoCompetencia.select()
	datos = {
		"nombre" : g.user._get_current_object().nombre,
		"competencias": [
		{
			"nombre": competencia.nombre,
			"numero" : competencia.get_numero_por_usuario(g.user._get_current_object()),
			"tipo": [{
				"nombre": tipo.nombre,
				"numero": tipo.numero_tipo_competencia(competencia,g.user._get_current_object())
			} for tipo in tipocompetencia
			]
		} for competencia in competencias
		]
	}
	return json.dumps(datos)

@app.route('/get_preguntas', methods=['POST'])
@login_required
def get_preguntas():
	competencia =  models.Competencias.select().where(models.Competencias.nombre == request.get_json()['nombre']).get()
	return json.dumps(competencia.preguntas())

@app.route('/respuestas_preguntas', methods=['POST'])
@login_required
def respuestas_preguntas():
	print(request.form)
	return "ok"

if __name__ == '__main__':
	#models.drop()

	models.initialize()
	app.run(
    	debug=DEBUG,
    	port=PORT,
    	host=HOST
    )


# Training Decission Tree for Classification
import streamlit as st
from sklearn.tree import DecisionTreeClassifier
from sklearn import metrics
from sklearn.metrics import classification_report
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import StandardScaler
# from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder

# Basic preprocessing required for all the models.
def preprocessing(df):
	# Assign X and y
	X = df.iloc[:, [0, 3, 5]].values
	y = df.iloc[:, -1].values

	# X and y has Categorical data hence needs Encoding
	le = LabelEncoder()
	y = le.fit_transform(y.flatten())

	# 1. Splitting X,y into Train & Test
	X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=0)
	return X_train, X_test, y_train, y_test, le

@st.cache_resource(experimental_allow_widgets=True)
def decisionTree(X_train, X_test, y_train, y_test):
	# Train the model
	tree = DecisionTreeClassifier(max_leaf_nodes=3, random_state=0)
	tree.fit(X_train, y_train)
	y_pred = tree.predict(X_test)
	score = metrics.accuracy_score(y_test, y_pred) * 100
	report = classification_report(y_test, y_pred)

	return score, report, tree

# Training Neural Network for Classification.
@st.cache_resource(experimental_allow_widgets=True)
def neuralNet(X_train, X_test, y_train, y_test):
	# Scalling the data before feeding it to the Neural Network.
	scaler = StandardScaler()
	scaler.fit(X_train)
	X_train = scaler.transform(X_train)
	X_test = scaler.transform(X_test)
	# Instantiate the Classifier and fit the model.
	clf = MLPClassifier(solver='lbfgs', alpha=1e-5, hidden_layer_sizes=(5, 2), random_state=1)
	clf.fit(X_train, y_train)
	y_pred = clf.predict(X_test)
	score1 = metrics.accuracy_score(y_test, y_pred) * 100
	report = classification_report(y_test, y_pred)

	return score1, report, clf

# # Training KNN Classifier
# @st.cache_resource(experimental_allow_widgets=True)
# def Knn_Classifier(X_train, X_test, y_train, y_test):
# 	clf = KNeighborsClassifier(n_neighbors=5)
# 	clf.fit(X_train, y_train)
# 	y_pred = clf.predict(X_test)
# 	score = metrics.accuracy_score(y_test, y_pred) * 100
# 	report = classification_report(y_test, y_pred)

# 	return score, report, clf


from torchvision.models import vgg16
from .base_model import _BaseModel
from torch import nn

class VGG16(_BaseModel):
    def __init__(self, nClass, pretrain=True):
        super(VGG16, self).__init__(nClass, pretrain)
        original_mod = vgg16(pretrain)

        # change input_num of vgg16
        in_kernel2 = original_mod.features[2].in_channels
        original_mod.features[0] = nn.Conv2d(in_channels=1, out_channels=in_kernel2, kernel_size = 3)

        self.encoder = nn.Sequential(original_mod.features, original_mod.avgpool)

        self.classify = nn.Sequential(
            nn.Linear(25088, 4096),
            nn.ReLU(),
            nn.Dropout(0.5),
            nn.Linear(4096, nClass),
            nn.Sigmoid()
        )


import copy
import math
import numpy as np
import torch
import torch.nn.functional as F
import gym
import highway_env

from random import randint

from lib.policy.batch import Batch
from lib.policy.continuous_mlp import ContinuousMLP
from lib.optimizers.actor_critic.actor_critic import TRPO

from lib.environments.parking import ParkingEmpty

from lib.vi.value_iteration_parking_orientation_new import ValueIteration33SyncNN

class Buffer:
    def __init__(self, max_size=int(16000)):
        self.max_size = max_size
        self.ptr = 0
        self.size = 0

        self.x = np.zeros((max_size, 3))
        self.y = np.zeros((max_size, 2))
        self.w = np.zeros((max_size, 5))

    def add(self, x, y):
        self.x[self.ptr] = x
        self.y[self.ptr] = y
        # self.w[self.ptr] = w

        self.ptr = (self.ptr + 1) % self.max_size
        self.size = min(self.size + 1, self.max_size)

    def sample(self, batch_size):
        ind = np.random.randint(0, self.size, size=batch_size)

        return (
            self.x[ind],
            self.y[ind]
        )

    def get_weights(self):
        class_sum = np.sum(self.w, axis=0)
        class_sum = class_sum / sum(class_sum)
        class_sum += 1e-3 * np.ones_like(class_sum)
        class_sum = np.array([1 / e for e in list(class_sum)])
        return class_sum / sum(class_sum)

class TRPOTrainer:

    def __init__(self,
                 op_mode,
                 state_dim,
                 action_dim,
                 hidden_sizes,
                 max_kl,
                 damping,
                 batch_size,
                 inner_episodes,
                 max_iter,
                 optimistic_model,
                 use_fim=False,
                 use_gpu=False
                 ):

        self.grid = ParkingEmpty()
        self.policy = ContinuousMLP(state_dim,
                                    action_dim,
                                    hidden_sizes=hidden_sizes,
                                    activation=F.relu,
                                    is_disc_action=False)
        self.optimizer = TRPO(policy=self.policy,
                              use_gpu=use_gpu,
                              max_kl=max_kl,
                              damping=damping,
                              use_fim=use_fim,
                              discount=0.99,
                              imp_weight=False)
        self.buffer = Buffer()
        gx = self.grid.x_size
        gy = self.grid.y_size
        self.vi = ValueIteration33SyncNN(grid_size_x=gx,
                                         grid_size_y=gy,
                                         gamma=self.optimizer.discount)

        self.op_mode = op_mode
        self.optimistic_model = optimistic_model
        self.batch_size = batch_size
        self.inner_episodes = inner_episodes
        self.max_iter = max_iter
        self.state_dim = state_dim
        self.dqn_steps = 0
        self.eps = 1.0
        self.time_scale = 2
        self.episode_steps = 0

    def roll_out_in_env(self, start, horizon):
        roll_out = Batch()
        s = start
        s_list = []
        a_list = []
        r_list = []

        d = False
        oob = False
        break_var = False
        s_u_goal = self.grid.lls2hls(s['observation'])
        target_vec = self.vi.get_target_vec(s['observation'])
        goal = self.grid.lls2hls(s['observation'] + np.array(list(target_vec[:2]) + [0, 0] + list(target_vec[2:4])))

        for step_i in range(horizon):
            self.episode_steps += 1

            target_vec = self.vi.get_target_vec(s['observation'])
            s_save = copy.deepcopy(np.array(list(target_vec[:2]) + list(s['observation'][2:4]) + list(target_vec[2:4])))

            s_list.append(copy.deepcopy(s['observation']))
            s_tensor = torch.tensor(s_save, dtype=torch.float).unsqueeze(0)
            a = self.policy.select_action(s_tensor)

            s_new, r, d, info = self.grid.env.step(a)
            s_new_bar = self.grid.lls2hls(s_new['observation'])
            success_var = info["is_success"]
            info = not info["is_success"]

            d = (s_new_bar[0] == goal[0]) and (s_new_bar[1] == goal[1]) and (s_new_bar[2] == goal[2])
            r = 0.0
            if d:
                r = 1.0
                info = False

            if success_var:
                info = False
                break_var = True

            break_var = break_var or (not info) or (step_i + 1 == horizon) or (self.episode_steps == self.max_iter)

            ib = self.grid.check_in_bounds(s_new['observation'])
            if not ib:
                oob = True

            a_list.append(a)
            r_list.append(r)
            roll_out.append(
                Batch([a.astype(np.float32)],
                      [s_save.astype(np.float32)],
                      [r],
                      [s_new['observation'].astype(np.float32)],
                      [0 if (break_var or oob) else 1],
                      [info],
                      [1.0]))

            s = s_new
            if break_var or oob:
                break

        s_list.append(copy.deepcopy(s['observation']))

        return roll_out, s_list, a_list, r_list, d, success_var, s_new, s_new_bar, oob

    def simulate_env(self, mode):
        batch = Batch()
        num_roll_outs = 0
        num_steps = 0
        total_success = 0
        total_wp_success = 0
        j = 0.
        jwp = 0.

        if mode == 'train':

            while num_steps < self.batch_size:

                """ INITIALIZE THE ENVIRONMENT """
                s_init = self.grid.reset()
                s_start = self.grid.lls2hls(s_init['observation'])
                s_goal = self.grid.lls2hls(s_init['desired_goal'])
                self.episode_steps = 0

                """ V MAP """
                if self.optimistic_model:
                    self.vi.update_p_table_optimistic(occupancy_map=self.grid.occupancy_map, walls=False)
                else:
                    self.vi.update_p_table(occupancy_map=self.grid.occupancy_map, walls=True)

                v, pi = self.vi.run_vi(grid=self.grid, goal=(s_goal[0], s_goal[1], s_goal[2]))

                """ START THE EPISODE """
                horizon_left = self.max_iter
                success = False

                hl_s_list = []
                hl_s_new_list = []
                hl_a_list = []
                hl_r_list = []
                hl_d_list = []

                while (horizon_left > 0) and not success:

                    # GET THE TARGET VECTOR
                    self.dqn_steps += 1
                    self.eps = 0.01 + 0.99 * math.exp(-1. * self.dqn_steps / 10000)

                    s_bar = self.grid.lls2hls(s_init['observation'])
                    hl_s_list.append(s_bar)

                    if torch.rand(1)[0] > self.eps:
                        a_bar = pi[s_bar[0], s_bar[1], s_bar[2]]
                    else:
                        a_bar = (randint(0, 7), randint(0, 7))

                    self.vi.set_target(s_bar, a_bar)

                    roll_out, _, _, _, wp_success, success, l_state, s_bar_p, oob = self.roll_out_in_env(
                        horizon=self.time_scale,
                        start=s_init)

                    hl_s_new_list.append(s_bar_p)
                    hl_a_list.append(a_bar)

                    s_init = l_state

                    num_roll_outs += 1
                    num_steps += roll_out.length()
                    horizon_left -= roll_out.length()

                    total_wp_success += wp_success
                    jwp += 1

                    if success:
                        hl_r_list.append(0)
                        hl_d_list.append(True)
                    else:
                        hl_r_list.append(-1)
                        hl_d_list.append(False)

                    batch.append(roll_out)

                    if oob:
                        break

                total_success += success
                j += 1

                if not self.optimistic_model:
                    x_temp, y_temp = self.vi.generate_dataset_flat(hl_s_list,
                                                                   hl_a_list,
                                                                   hl_s_new_list
                                                                   )

                    for bi in range(x_temp.shape[0]):
                        self.buffer.add(x_temp[bi], y_temp[bi])

                    self.vi.train_net(buffer=self.buffer, bs=128, opt_iterations=40)

            return batch, total_success / j, total_wp_success / jwp, num_steps / j, num_steps / num_roll_outs

        else:

            """ INITIALIZE THE ENVIRONMENT """
            s_init = self.grid.reset()
            s_start = self.grid.lls2hls(s_init['observation'])
            s_goal = self.grid.lls2hls(s_init['desired_goal'])
            self.episode_steps = 0

            """ V MAP """
            if self.optimistic_model:
                self.vi.update_p_table_optimistic(occupancy_map=self.grid.occupancy_map, walls=False)
            else:
                self.vi.update_p_table(occupancy_map=self.grid.occupancy_map, walls=True)

            v, pi = self.vi.run_vi(grid=self.grid, goal=(s_goal[0], s_goal[1], s_goal[2]))

            """ START THE EPISODE """
            horizon_left = self.max_iter
            success = False

            while (horizon_left > 0) and not success:

                # GET THE TARGET VECTOR
                s_bar = self.grid.lls2hls(s_init['observation'])
                a_bar = pi[s_bar[0], s_bar[1], s_bar[2]]

                self.vi.set_target(s_bar, a_bar)

                roll_out, states, actions, rewards, wp_success, success, l_state, s_bar_p, oob = self.roll_out_in_env(
                    horizon=self.time_scale,
                    start=s_init)

                s_init = l_state

                num_roll_outs += 1
                num_steps += roll_out.length()
                horizon_left -= roll_out.length()

                total_wp_success += wp_success
                jwp += 1

                if oob:
                    break

            return success

    def train(self):
        for iter_loop in range(self.inner_episodes):
            batch, g_success, wp_success, steps_trajectory, steps_wp = self.simulate_env(mode='train')
            self.optimizer.process_batch(self.policy, batch, [])
            print("G_S, WP_S, G_ST, WP_ST")
            print(g_success)
            print(wp_success)
            print(steps_trajectory)
            print(steps_wp)

    def test(self):
        g_success = self.simulate_env(mode='test')
        return g_success


import gym
from gym_anytrading.envs import TradingEnv, ForexEnv, StocksEnv, Actions, Positions
from gym_anytrading.datasets import FOREX_EURUSD_1H_ASK, STOCKS_GOOGL
import matplotlib.pyplot as plt
import pandas as pd
from stocks_env import StocksEnv

# env = gym.make('forex-v0', frame_bound=(50, 100), window_size=10)
# env = gym.make('stocks-v0', frame_bound=(50, 100), window_size=10)
env = StocksEnv(df=pd.read_csv('STOCKS_GOOGL.csv'),frame_bound=(50, 100), window_size=10)
print("==========env information:========")
print("> shape:", env.shape)
print("> df.shape:", env.df.shape)
print("> prices.shape:", env.prices.shape)
print("> signal_features.shape:", env.signal_features.shape)
print("> max_possible_profit:", env.max_possible_profit())
print(env.action_space.sample())

observation = env.reset()
while True:
    action = env.action_space.sample()   # Short=0 and Long=1

    # action =-1  # Short=0 and Long=1

    # print('action:', action)
    observation, reward, done, info = env.step(action)
    print('steps:',action, reward)
    # env.render()
    if done:
        print("info:", info)
        break

plt.cla()
env.render_all()
plt.show()


from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
import csv
import os
from ast import literal_eval

def new_status(eps, total):
    try:
        if (eps / total) * 100 > 90:
            return 'star'
        elif (eps / total) * 100 > 50:
            return 'main cast'
        elif eps > 1:
            return 'guest'
        else:
            return 'cameo/extra'
    except TypeError:
        return 'cameo/extra'

class Actor:
    def __init__(self, link, show, role, eps, total):
        self.link = link
        self.roles = {show: role}
        self.episodes = {show: eps}
        self.status = {show: new_status(eps, total)}

    def add_role(self, show, role, eps, total):
        self.roles.update({show: role})
        self.episodes.update({show: eps})
        self.status.update({show: new_status(eps, total)})

def new_completed():
    if os.path.exists('csv_file.csv'):
        os.remove('csv_file.csv')
    with open('completed_titles.txt', 'w') as completed:
        completed.writelines("Completed titles by LAextra:")
    with open('blacklist.txt', 'w') as completed:
        completed.writelines("Titles shot outside the US:")

def write_setup(reset=False):
    completed_titles = []
    blacklist = []
    if reset:
        new_completed()
    else:
        try:
            with open('completed_titles.txt', 'r') as completed:
                completed_titles = completed.readlines()
                if len(completed_titles) > 1:
                    completed_titles = [i.strip() for i in completed_titles[1:]]
                else:
                    completed_titles.clear()
            with open('blacklist.txt', 'r') as blacktitles:
                blacklist = blacktitles.readlines()
                if len(blacklist) > 1:
                    blacklist = [i.strip() for i in blacklist[1:]]
                else:
                    blacklist.clear()
        except FileNotFoundError:
            new_completed()
    return completed_titles, blacklist

def add_to_blacklist(title):
    with open('blacklist.txt', 'a') as blacktitles:
        blacktitles.write('\n' + title)

def new_csv(actors, title, stop):
    fieldnames = ['name', 'weblink', 'shows', 'roles', 'episodes', 'status']
    overwrite_actors = []
    with open('new_file.csv', 'w', newline='') as new_file:
        csv_writer = csv.DictWriter(new_file, fieldnames=fieldnames)
        csv_writer.writeheader()
        try:
            with open('csv_file.csv', 'r') as old_file:
                csv_reader = csv.DictReader(old_file)
                for line in csv_reader:
                    if line['name'] in actors:
                        name = line['name']
                        overwrite_actors.append(name)
                        link = line['weblink']
                        shows = int(line['shows']) + 1
                        roles = literal_eval(line['roles'])
                        episodes = literal_eval(line['episodes'])
                        status = literal_eval(line['status'])
                        roles.update(actors[name].roles)
                        episodes.update(actors[name].episodes)
                        status.update(actors[name].status)
                        csv_writer.writerow({
                            'name': name,
                            'weblink': link,
                            'shows': shows,
                            'roles': roles,
                            'episodes': episodes,
                            'status': status
                        })
                    else:
                        csv_writer.writerow(line)
        except FileNotFoundError:
            pass
        for actor in actors:
            if actor not in overwrite_actors:
                csv_writer.writerow({
                    'name': actor,
                    'weblink': actors[actor].link,
                    'shows': len(actors[actor].roles),
                    'roles': actors[actor].roles,
                    'episodes': actors[actor].episodes,
                    'status': actors[actor].status
                })
    with open('completed_titles.txt', 'a') as completed:
        completed.write('\n' + title)
    try:
        os.replace('new_file.csv', 'csv_file.csv')
    except FileNotFoundError:
        os.rename('new_file.csv', 'csv_file.csv')
    if stop:
        breakpoint()

def main(completed_titles, blacklist, stop=False):
    PATH = "C:\Program Files (x86)\chromedriver.exe"
    driver = webdriver.Chrome(PATH)
    start_url = "https://www.imdb.com/search/title/?genres=comedy&explore=title_type,genres&title_type=tvSeries&ref_=adv_explore_rhs"
    driver.get(start_url)
    titles = []
    ids_ = []
    titles_dict = {}
    for i in range(3):
        load_titles = []
        while len(load_titles) < 50:
            load_titles = driver.find_elements_by_xpath("//a[contains(@href, '?ref_=adv_li_tt')]")
        titles.extend(j.text for j in load_titles)
        ids_.extend(j.get_attribute("href") for j in load_titles)
        if i < 2:
            driver.find_element_by_class_name('next-page').click()
            WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "styleguide-v2")))
    if any(titles.count(i) > 1 for i in titles):
        for title in set([i for i in titles if titles.count(i) > 1]):
            for e in range(2, titles.count(title) + 1):
                ind = [i for i, n in enumerate(titles) if n == title][e - 1]
                titles[ind] = f'{title} ({e})'
    titles_dict.update({title: id_ for title, id_ in zip(titles, ids_)})
    for title in completed_titles + blacklist:
        if title in titles_dict:
            titles_dict.pop(title)
    for title in titles_dict:
        actor_dict = {}
        driver.get(titles_dict[title])
        try:
            WebDriverWait(driver, 10).until(
                EC.presence_of_element_located((By.XPATH, "//a[contains(@href, 'country_of_origin')]")))
        except TimeoutException:
            blacklist.append(title)
            add_to_blacklist(title)
            continue
        origin = driver.find_element_by_xpath("//a[contains(@href, 'country_of_origin')]").text
        if origin != 'USA':
            blacklist.append(title)
            add_to_blacklist(title)
            continue
        WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'add_episode')))
        sub_text = driver.find_elements_by_class_name('bp_sub_heading')
        if len(sub_text) == 1:
            total_episodes = int(sub_text[0].text.split(' ')[0])
        else:
            total_episodes = int(sub_text[1].text.split(' ')[0])
        print(f'{title}, eps: {total_episodes}')
        driver.find_element_by_xpath("//a[contains(@href, 'fullcredits')]").click()
        WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'cast_list')))
        cast_list = driver.find_element_by_class_name('cast_list')
        WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'add_episode')))
        rows = cast_list.find_elements(By.TAG_NAME, "tr")[1::2]
        for row in rows:
            name = row.find_elements(By.TAG_NAME, "td")[1].text
            if name in actor_dict and title in actor_dict[name].roles:
                continue
            link = row.find_element(By.XPATH, ".//a[contains(@href, '/name/')]").get_attribute('href')
            try:
                character = row.find_element(By.XPATH, ".//a[contains(@href, '/characters/')]").text
            except NoSuchElementException:
                character = 'N/A'
            episodes = row.find_element(By.XPATH, ".//a[contains(@href, '#')]").text
            try:
                episodes = int(episodes.split(' ')[0])
            except ValueError:
                episodes = episodes.split(' ')[0]
            if name in actor_dict:
                actor_dict[name].add_role(title, character, episodes, total_episodes)
            else:
                actor_dict.update({name: Actor(link, title, character, episodes, total_episodes)})
            print(name + ', ' + actor_dict[name].roles[title] + ', ' + actor_dict[name].status[title])
        new_csv(actor_dict, title, stop)
    print('Finished!')
    driver.quit()

if __name__ == '__main__':
    completed_titles, blacklist = write_setup() # pass True to write_setup to reset and delete files
    main(completed_titles, blacklist) # pass True as the third value to main to add breakpoints after each title


#importing libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import itertools
import math

#importing dataset
dataset = pd.read_csv('Social_Network_Ads.csv')
X = dataset.iloc[:, 1:4].values
Y = dataset.iloc[:, 4].values

#Encoding categorical data
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
labelencoder_X = LabelEncoder()
X[:, 0] = labelencoder_X.fit_transform(X[:, 0])
#dummy encoding (making seprate columns for seprate categories )
onehotencoder = OneHotEncoder(categorical_features = [0])
X = onehotencoder.fit_transform(X).toarray()

#Splitting the dataset into training set and Test set
from sklearn.model_selection import train_test_split
X_train,X_test,Y_train,Y_test = train_test_split(X, Y, test_size = 0.1, random_state = 0)

#Feature Scaling
from sklearn.preprocessing import StandardScaler
sc_X = StandardScaler()
X_train = sc_X.fit_transform(X_train)
X_test = sc_X.transform(X_test) #here we transformed X_test by fitting sc_X to X_train so that both have same scaling

#Logistic regression using gradient descent

theta = [0,0,0,0] #values of coffiecient of independent variable

#Making X[i][0] = 1 so that we get genral form like h = theta0+ theta1*x + theta2*x ... after multiplying theta and X_train matrices
i = 0
j = 0
while(i<360):
    X_train[i][j] = 1
    if i<40:
        X_test[i][j] = 1
    i += 1

#function to return  derivative of cost function that will be subtracted form each theta respectevely.
def averageCost(temp):
    total = 0
    for (i,j) in zip(X_train,Y_train):
        y_pred = h_theta(i) #calculating value of predicted y by signoid or logistic function
        total = total + (y_pred - j)*i[temp]
    return total/len(X_train)

#calculating derivative of cost function for each theta
cost = [0,0,0,0]

def updateCostMatrix():
    s = 0 #it denotes index of respective theta
    while s<len(theta):
        cost[s] = averageCost(s)
        s += 1

def updateTheta():
    i = 0
    while(i<len(theta)):
        theta[i] = theta[i] - 0.1*cost[i]
        i += 1

def h_theta(i):
    mul = np.matmul(theta,i)
    if(mul<0):
            mul = 1 - 1/(1+math.exp(mul))
    else:
        mul = 1/(1+math.exp(-mul))
    return(mul)

#Here loop stops when cost becomes approximately to 0
#This loop is gradient descent algorithm
while(True):
    count = 0 #it is used to check that all cost values are zeroes
    updateCostMatrix()
    updateTheta()
    #print(cost)
    for i in cost:
        if(i < 0.5 and i>-0.5):
            count += 1

    if(count == 4):  #4 is no of features in X_train
        break

#Calculating error of predicted values and also storing predicted values in temp
error = 0 #initialising it.
temp = []
for i,j in zip(X_test,Y_test):
    y_pred = h_theta(i)
    if(y_pred>=0.5):
        y_pred  = 1
    else:
        y_pred = 0
    temp.append(y_pred)
    if(y_pred != j):
        error = error + 1
error = error*100/40
print("Error is:",error,"%")


import os
import pickle
from pathlib import Path

import numpy as np

import Main

database = {}
ids_names = {}

def evaluate_all():
    """
    Evaluates all query files and prints info
    :return: None
    """
    global database, ids_names
    # Load database from file
    db_file = open("fingerprints" + os.path.sep + "db_S.pkl", "rb")
    ids_names_file = open("fingerprints" + os.path.sep + "ids_names.pkl", "rb")
    database = pickle.load(db_file)
    ids_names = pickle.load(ids_names_file)

    # Path to wavs
    wav_path = "data" + os.path.sep + "query_recordings"

    wavs = Path(wav_path).rglob("*.wav")
    number_of_query_files = sum(1 for x in wavs)

    f_maxs = []
    all_f_measures = []
    all_precisions = []
    all_recalls = []
    map = 0
    counter = 0
    for wav in Path(wav_path).rglob("*.wav"):
        print("Evaluating " + str(counter) + " of " + str(number_of_query_files))
        precision, recall, f_measures, f_max, p_avg = evaluate(str(wav).split(os.path.sep)[-1])

        all_precisions.append(precision)
        all_recalls.append(recall)
        f_maxs.append(f_max)
        all_f_measures.append(f_measures)
        map += p_avg
        counter += 1

    avg_all_f = np.mean(all_f_measures, axis=0)
    f_max_avg = np.mean(f_maxs)
    map /= number_of_query_files

    print("Average precision for first three ranks : " + np.array2string(np.mean(all_precisions, axis=0), separator=", "))
    print("Average recall for first three ranks : " + np.array2string(np.mean(all_recalls, axis=0), separator=", "))
    print("Average F-measue values for the first three ranks: " + np.array2string(avg_all_f, separator=", "))
    print("Average maximum F-measure: " + str(f_max_avg))
    print("Mean Average Precision: " + str(map))

def evaluate(query_path):
    """
    Evaluate a given *.wav file
    :param query_path: The path to the *.wav file
    :return: Precision, recall and f_measures for the first three ranks, the maximum f-measure and the average precision
    """
    # Get matching results
    best_three, is_correct = Main.analyse(database, ids_names, str(query_path))

    # Get ground truth song ID
    song_name = query_path.split(os.path.sep)[-1].split('-')[0] + ".wav"

    # Get ground truth best ten
    best_ten_gt, is_correct = Main.analyse(database, ids_names, song_name)

    relevant = np.zeros(len(best_three))
    for i in range(len(best_three)):
        if best_three[i] in best_ten_gt:
            relevant[i] = 1

    precision = np.zeros(len(best_three))
    recall = np.zeros(len(best_three))

    for i in range(1, len(best_three) + 1):
        sum = np.sum(relevant[:i])
        precision[i - 1] = (1 / i) * sum
        recall[i - 1] = sum

    # Calculate F-measure
    f_measures = []
    for i in range(len(best_three)):
        if precision[i] == 0 and recall[i] == 0:
            f_measures.append(0)
        else:
            f_measures.append((2 * precision[i] * recall[i]) / (precision[i] + recall[i]))

    # Calculate maximum f measure
    f_max = np.max(f_measures)

    # Calculate average precision
    p_avg = np.sum(precision * relevant) / 1

    return precision, recall, f_measures, f_max, p_avg

# Evaluate
# relevance("pop.00050-snippet-10-0.wav")
# evaluate_all()


from collections import defaultdict

# Bull and cows game, count match and unmatch char
# https://leetcode.com/problems/bulls-and-cows/
# Preprocess secret if guess match, increase A, else increase char
# loop through remain guess, if char in A, decrease char
# Optmize when encounter number in secrect, increase count, number in guess
# decrease count, if 2 number same => increase bull, if different, if count(secret)
# < 0, increase cows (as guess decrease it), if count(guess) > 0, increase cows
# (as secret increase it)
class BullCowSolution:
    def getHint(self, secret, guess):

        table = defaultdict(int)
        A = B = 0
        p = []
        for idx in range(len(guess)):
            if secret[idx] == guess[idx]:
                A += 1
            else:
                table[secret[idx]] += 1
                p.append(idx)

        for idx in p:
            if guess[idx] in table and table[guess[idx]] > 0:
                table[guess[idx]] -= 1
                B += 1

        return '{}A{}B'.format(A, B)

    def getHint2(self, secret, guess):
        count = [0 for i in range(10)]
        bull = cow = 0
        for idx in range(len(guess)):
            s = int(secret[idx])
            g = int(secret[idx])
            if s == g:
                bull += 1
            else:
                if count[s] < 0:
                    cow += 1
                if count[g] > 0:
                    cow += 1

                count[s] += 1
                count[g] -= 1
        return '{}A{}B'.format(bull, cow)


from utils import filter_predicted_growth_subset, filter_predicted_stable_subset, grid_search
from typing import List
from cfmodel import CfModel
from lightgbm import LGBMClassifier
import pandas as pd
class LGBM(CfModel):
    """Create a LGBM model object
    """

    def __init__(self, l90d: pd.DataFrame, n180d: pd.DataFrame, growth_data: pd.DataFrame, stable_data: pd.DataFrame,
                 features: List[str], subset_type: str):
        super().__init__(l90d, n180d, growth_data, stable_data, features, subset_type)
        self.subset_type = subset_type
        self.get_x_train_y_train()
        self.model, self.y_pred = self.train()
        self.pred, self.true = self.get_subset_pred_true()
        self.filtered_pred = filter_predicted_growth_subset(self.pred) if self.subset_type == 'growth' else \
            filter_predicted_stable_subset(self.pred)

        print('LGBM Model has finished training')
    def train(self) -> (LGBMClassifier, pd.DataFrame):
        parameters = {
            "max_depth": [10],
            "num_leaves": [20],
            "learning_rate": [0.05],
            "n_estimators": [1000],
        }

        model = LGBMClassifier(
            random_state=0,
        )
        model.set_params(verbose=0)
        model, y_pred = grid_search(parameters, model, self.X_train, self.y_train, self.X_test)
        return model, y_pred