import json
from datetime import datetime as dt
from datetime import date as dto
import copy

class ProcessJsonPortfolio:

    def calculate_average_price(self, _dict):
        """Calculate dollar cost average per security.

        Args:
            _dict (:obj:`dict`): Portfolio loaded from json.

        Return:
            (:obj:`tuple` of :obj:`dict`, :obj:`list`, :obj:`list`): new portfolio object; list of securities; list of holdings
        """
        securities = []
        holdings = []
        result = copy.deepcopy(_dict)
        for key in _dict:
            if key == 'test':
                continue
            total_count = 0
            total_cost = 0
            for i, val in enumerate(_dict[key]['Lots']):  #{'date': ... 'shares': ... 'executed price'}
                total_count += val['Shares']
                total_cost += val['Executed Price'] * val['Shares']
                result[key]['Lots'][i]['Processed'] = "True"
            avg_cost = total_cost / total_count
            result[key]['Total Holdings'] = total_count
            result[key]['Total Cost'] = total_cost
            result[key]['Average Cost'] = avg_cost
            securities.append(key)
            holdings.append(total_count)
        return result, securities, holdings

    def get_earliest_date(self, _dict):
        """Get the earliest unprocessed date.

        Args:
            _dict (:obj:`dict`): portfolio object.
        """
        earliest_date = dto.today()
        for key in _dict:
            if key == 'test':
                continue
            for val in _dict[key]['Lots']:
                if val.get('Processed') == "True":
                    continue
                else:
                    cur_date = dt.strptime(val['Date'], '%Y-%m-%d').date()
                    if cur_date < earliest_date:
                        earliest_date = cur_date
        return earliest_date


from ray import serve
from typing import List, Dict

import json
import numpy as np
from scipy.optimize import linprog

@serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0})
class LinearProgrammingService(object):
    # def __init__(self):
    def LinearProgramming(self, body: Dict):
        print(123)
        try:

            event = body
            MinOrMax = event['MinOrMax']
            target = event['target']
            A = event['A']
            b = event['b']
            bounds = event['bounds']
            print("线性规划求解器:")
            if MinOrMax == 'min':
                pass
            elif MinOrMax == 'max':
                target = np.array(target) * (-1)
            # minimize
            res = linprog(target, A, b, bounds=bounds)

        except Exception as e:
            print(e)
            print(e.__traceback__.tb_frame.f_globals["__file__"])  # 发生异常所在的文件
            print(e.__traceback__.tb_lineno)  # 发生异常所在的行数
        else:
            print("success")

        return {
            "OptimalValue": res.fun,
            "OptimalSolution": res.x
        }


import os
import sys

FILE_DIR = os.path.dirname(os.path.abspath(__file__))
# PROJ_DIR = FILE_DIR[:FILE_DIR.index('src')]
# sys.path.append(PROJ_DIR)

PROJ_DIR = os.path.abspath("..")
print(f"proj_dir is: {PROJ_DIR}, adding to sys.path")

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import pytorch_lightning as pl
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from pytorch_lightning.loggers import TensorBoardLogger
from torchmetrics.functional import bleu_score

from transformers import get_linear_schedule_with_warmup

from q_snippets.data import load_json, save_json

from data_utils import Seq2seqDataModule
from model import get_model_by_name

class Seq2seqGeneration(pl.LightningModule):
    def __init__(self, config, model) :
        super().__init__()
        self.config = config
        self.model = model
        self.save_hyperparameters(ignore='model')  # ignore model to avoid assigning model to Omegaconf when load_from_ckpt

        self.val_pred_ids = []
        self.val_target_ids = []
        self.gold_corpus = []
        self.pred_corpus = []

    def forward(self, batch):
        def _custom_forward(batch):
            """ for BART """
            if batch.target_ids is not None:
                target_ids = batch.target_ids[:, :-1].contiguous()
                lm_labels = batch.target_ids[:, 1:].clone()
                lm_labels[batch.target_ids[:, 1:] == self.model.config.pad_token_id] = -100
            else:
                target_ids, lm_labels = None, None
            # print(batch.input_ids.size(), target_ids.size(), lm_labels.size())

            output = self.model(
                input_ids=batch.input_ids, attention_mask=batch.attention_mask,
                decoder_input_ids=target_ids,
                labels=lm_labels,
                # output_attentions=True  # for copy mechanism
            )
            return output

        def _default_forward(batch):
            """ 训练时模型会自动从labels参数右移得到decoder_input_ids
                for T5
            """
            return self.model(batch.input_ids, attention_mask=batch.attention_mask, labels=batch.target_ids)

        return _default_forward(batch)
        # return _custom_forward(batch)

    def training_step(self, batch, batch_idx):
        output = self(batch)
        self.log('train_loss', output.loss, prog_bar=True, sync_dist=True)
        return output.loss

    def validation_step(self, batch, batch_idx) :
        output = self(batch)
        # self.val_pred_ids.extend(output.logits.argmax(-1).cpu().numpy().tolist())
        self.log('val_loss', output.loss, prog_bar=True, sync_dist=True)                 #log the val_loss

        pred_ids = self.model.generate(
            input_ids=batch.input_ids, max_length=500, use_cache=True,
            num_beams=1, do_sample=False                                        #  greedy search is the fastest
            )
        self.val_pred_ids.extend(pred_ids)

        # save gold ids for bleu computing
        if self.gold_corpus == [] and batch.target_ids is not None:
            self.val_target_ids.extend(batch.target_ids.cpu().numpy().tolist())

    def _save_val_result(self):

        self.gold_corpus = ["None" for _ in self.pred_corpus ] if self.gold_corpus == [] else self.gold_corpus
        R = []
        for p, sample, g in zip(self.pred_corpus, self.trainer.datamodule.valset.samples, self.gold_corpus):
            R.append(dict(
                **sample.__dict__,
                **{
                    'expected': g,
                    'generated':p}
            ))
        # logdir = trainer.logger.log_dir if hasattr(trainer.logger, 'log_dir') else trainer.logger.save_dir
        logdir = self.trainer.logger.log_dir
        filename = os.path.join(logdir, f"val_epoch{self.current_epoch:02}.json")
        save_json(R, filename)

    def validation_epoch_end(self, outputs):

        tokenizer = self.trainer.datamodule.tokenizer
        self.pred_corpus = tokenizer.batch_decode(self.val_pred_ids, skip_special_tokens = True, clean_up_tokenization_spaces = True)

        if self.gold_corpus == [] and self.val_target_ids != [] :
            self.gold_corpus = tokenizer.batch_decode(self.val_target_ids, skip_special_tokens = True, clean_up_tokenization_spaces = True)

        print(len(self.pred_corpus), len(self.gold_corpus))
        bleu = bleu_score(self.pred_corpus, [ [_] for _ in self.gold_corpus])
        self.log('val_bleu', bleu, prog_bar=True, sync_dist=True)

        self._save_val_result()
        self.val_pred_ids, self.val_target_ids =[], []

    def _get_grouped_params(self):
        no_decay = ["bias", "LayerNorm.weight"]

        # Group parameters to those that will and will not have weight decay applied
        optimizer_grouped_parameters = [
            {
                "params": [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay)],
                "weight_decay": 0.01,
            },
            {
                "params": [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay)],
                "weight_decay": 0.0,
            },
        ]
        return optimizer_grouped_parameters

    def configure_optimizers(self):
        optimizer = optim.AdamW(self._get_grouped_params(), lr=self.config.lr)
        # return optimizer
        total_steps = int(len(self.trainer.datamodule.train_dataloader()) // self.config.accumulate_grads ) * self.config.max_epochs # accumulate_grads
        warmup_step =  int(total_steps * self.config.warmup_rate)
        # lr_scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=warmup_step, num_training_steps=steps_per_epoch*self.config.max_epochs)
        lr_scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=warmup_step, num_training_steps=total_steps)

        return [optimizer], [{'scheduler': lr_scheduler, 'interval': 'step', 'frequency': 1, 'strict': True, 'monitor': None}]

    def predict_step(self, batch , batch_idx) :
        batch_pred_ids = self.model.generate(
                input_ids=batch.input_ids, max_length=500, use_cache=True)
        return batch_pred_ids.cpu().numpy().tolist()

    def on_predict_epoch_end(self, results) -> None:
        """ results = [[ batch_result ]]
            batch_result = [[],[],...]
            聚合每个predict_step的结果，解码并保存到文件
        """
        all_pred_ids = sum(results[0], [])
        preds = self.trainer.datamodule.tokenizer.batch_decode(all_pred_ids, skip_special_tokens = True, clean_up_tokenization_spaces = True)
        R = []
        for sample, p in zip(self.trainer.datamodule.testset.samples, preds):
            R.append(dict(
                    **sample.__dict__,
                    **{
                        'generated':p}
                ))
        save_json(R, self.config.preds.result_path)
        return preds

def train_model(config):

    _model = get_model_by_name(config)
    model = Seq2seqGeneration(config, _model)  # 创建Lightning框架

    dm = Seq2seqDataModule(config=config)

    logger = TensorBoardLogger(
        save_dir="./lightning_logs/",
        name=None,                # 指定experiment, ./lightning_logs/exp_name/version_name
        version=config.version,   # 指定version, ./lightning_logs/version_name
    )
    # 设置保存模型的路径及参数
    CUR_DIR = os.getcwd()
    dirname = os.path.join(CUR_DIR, "./lightning_logs/", config.version)
    ckpt_callback = ModelCheckpoint(
        dirpath=dirname,
        filename="{epoch}_{train_loss:.4f}_{val_bleu:.4f}",   # 模型保存名称， epoch信息以及验证集分数
        monitor='val_bleu',
        mode='max',
        save_top_k=3,
        verbose=True,
    )
    es = EarlyStopping('train_loss', patience=10, mode='min')
    trainer = pl.Trainer(
        accumulate_grad_batches=config.accumulate_grads,
        logger=logger,
        num_sanity_val_steps=0,    # 如果使用sanity_check  会导致val时self.gold_corpus数量出现问题
        # limit_train_batches=64,  # 限制训练集数量，方便快速调试
        # limit_val_batches=64,  # 一般直接用全量测试数据吧, 验证函数可能会报错
        max_epochs=config.max_epochs,
        callbacks=[ckpt_callback, es],
        accelerator="gpu",
        devices=1,
        # resume_from_checkpoint="/home/qing/repos/demo/conditional_generation/lightning_logs/2rd/epoch=1_train_loss=2.0390_val_bleu=0.0197.ckpt"
    )
    # dm.setup(stage='fit')
    trainer.fit(model, dm)

def predict_ckpt(config):
    dm = Seq2seqDataModule(config)
    dm.setup(stage='test')
    _model = get_model_by_name(config)
    model = Seq2seqGeneration.load_from_checkpoint(config.preds.ckpt_path, config=config, model=_model)
    trainer = pl.Trainer(accelerator="gpu", devices=1)
    x = trainer.predict(model, dm)  # 预测结果已经在on_predict_epoch_end中保存了

    print(type(x))

def raw_generate(config):
    from tqdm import tqdm
    device = 'cuda' if torch.cuda.is_available() else 'cpu'

    dm = Seq2seqDataModule(config)
    dm.setup(stage='test')
    _model = get_model_by_name(config)
    model = Seq2seqGeneration.load_from_checkpoint(config.preds.ckpt_path, config=config, model=_model).to(device)

    with torch.no_grad():
        R = []
        for i, batch in enumerate(tqdm(dm.test_dataloader())):
            batch = batch.to(device)
            pred_ids = model.model.generate(input_ids=batch.input_ids, max_length=500, use_cache=True, num_beams=1, do_sample=False)
            R.extend(pred_ids.cpu().numpy().tolist())

    x = dm.tokenizer.batch_decode(R)


from functools import partial

from ._derived import Derived
from . import utilities

class Operation(Derived):
    __slots__ = ('opkwargs', 'opfn')

    def __init__(self,
            *terms,
            op = None,
            **kwargs,
            ):
        if type(op) is tuple:
            sops, op = op[:-1], op[-1]
            for sop in sops:
                terms = Operation(*terms, op = sop)
                if not type(terms) is tuple:
                    terms = terms,
        self.opfn = partial(op, **kwargs)
        self.opfn.__name__ = op.__name__
        self.opkwargs = kwargs
        super().__init__(*terms, op = op, **kwargs)

    def evaluate(self):
        return self.opfn(*self._resolve_terms())

    def _titlestr(self):
        return self.opfn.__name__
    def _kwargstr(self):
        kwargs = self.kwargs.copy()
        del kwargs['op']
        if kwargs:
            return utilities.kwargstr(**kwargs)
        else:
            return ''


# Standard imports
import copy
import pandas as pd
import logging
import jsonpickle as jpickle
import sklearn.cluster as sc

# Our imports
import emission.storage.timeseries.abstract_timeseries as esta

import emission.analysis.modelling.tour_model.get_scores as gs
import emission.analysis.modelling.tour_model.get_users as gu
import emission.analysis.modelling.tour_model.label_processing as lp
import emission.analysis.modelling.tour_model.evaluation_pipeline as ep
import emission.analysis.modelling.tour_model.load_predict as load
import emission.analysis.modelling.tour_model.data_preprocessing as preprocess

def find_best_split_and_parameters(user,test_data):
    # find the best score
    filename = "user_"+str(user)+".csv"
    df = pd.read_csv(filename, index_col='split')
    scores = df['scores'].tolist()
    best_split_idx = scores.index(max(scores))
    # use the position of best_score to find best_split
    best_split = test_data[best_split_idx]
    # use best_split_idx to find the best parameters
    low = df.loc[best_split_idx, 'lower boundary']
    dist_pct = df.loc[best_split_idx, 'distance percentage']
    return best_split,best_split_idx,low,dist_pct

# def find_best_parameters(user,best_split_idx):
#     tradeoff_filename = 'tradeoff_' + str(user)
#     tradeoff_1user = load.loadModelStage(tradeoff_filename)
#     best_parameters = tradeoff_1user[best_split_idx]
#     return best_parameters

def save_models(obj_name,obj,user):
    obj_capsule = jpickle.dumps(obj)
    filename = obj_name + '_' + str(user)
    with open(filename, "w") as fd:
        fd.write(obj_capsule)

def main():
    all_users = esta.TimeSeries.get_uuid_list()
    radius = 100
    for a in range(len(all_users)):
        user = all_users[a]
        trips = preprocess.read_data(user)
        filter_trips = preprocess.filter_data(trips, radius)
        # filter out users that don't have enough valid labeled trips
        if not gu.valid_user(filter_trips, trips):
            logging.debug("This user doesn't have enough valid trips for further analysis.")
            continue
        tune_idx, test_idx = preprocess.split_data(filter_trips)
        test_data = preprocess.get_subdata(filter_trips, tune_idx)

        # find the best split and parameters, and use them to build the model
        best_split, best_split_idx, low, dist_pct = find_best_split_and_parameters(user,test_data)

        # run the first round of clustering
        sim, bins, bin_trips, filter_trips = ep.first_round(best_split, radius)
        # It is possible that the user doesn't have common trips. Here we only build models for user that has common trips.
        if len(bins) is not 0:
            gs.compare_trip_orders(bins, bin_trips, filter_trips)
            first_labels = ep.get_first_label(bins)
            first_label_set = list(set(first_labels))

            # second round of clustering
            model_coll = {}
            bin_loc_feat = {}
            fitst_round_labels = {}
            for fl in first_label_set:
                # store second round trips data
                second_round_trips = []
                for index, first_label in enumerate(first_labels):
                    if first_label == fl:
                        second_round_trips.append(bin_trips[index])
                x = preprocess.extract_features(second_round_trips)
                # collect location features of the bin from the first round of clustering
                # feat[0:4] are start/end coordinates
                bin_loc_feat[str(fl)] = [feat[0:4] for feat in x]
                # here we pass in features(x) from selected second round trips to build the model
                method = 'single'
                clusters = lp.get_second_labels(x, method, low, dist_pct)
                n_clusters = len(set(clusters))
                # build the model
                kmeans = sc.KMeans(n_clusters=n_clusters, random_state=0).fit(x)
                # collect all models, the key is the label from the 1st found
                # e.g.{'0': KMeans(n_clusters=2, random_state=0)}
                model_coll[str(fl)] = kmeans
                # get labels from the 2nd round of clustering
                second_labels = kmeans.labels_

                first_label_obj = []

                # save user labels for every cluster
                second_label_set = list(set(second_labels))
                sec_round_labels = {}
                for sl in second_label_set:
                    sec_sel_trips = []
                    sec_label_obj = []
                    for idx, second_label in enumerate(second_labels):
                        if second_label == sl:
                            sec_sel_trips.append(second_round_trips[idx])
                    user_label_df = pd.DataFrame([trip['data']['user_input'] for trip in sec_sel_trips])
                    user_label_df = lp.map_labels(user_label_df)
                    # compute the sum of trips in this cluster
                    sum_trips = len(user_label_df)
                    # compute unique label sets and their probabilities in one cluster
                    # 'p' refers to probability
                    unique_labels = user_label_df.groupby(user_label_df.columns.tolist()).size().reset_index(name='uniqcount')
                    unique_labels['p'] = unique_labels.uniqcount / sum_trips
                    labels_columns = user_label_df.columns.to_list()
                    for i in range(len(unique_labels)):
                        one_set_labels = {}
                        # e.g. labels_only={'mode_confirm': 'pilot_ebike', 'purpose_confirm': 'work', 'replaced_mode': 'walk'}
                        labels_only = {column: unique_labels.iloc[i][column] for column in labels_columns}
                        one_set_labels["labels"] = labels_only
                        one_set_labels['p'] = unique_labels.iloc[i]['p']
                        # e.g. one_set_labels = {'labels': {'mode_confirm': 'walk', 'replaced_mode': 'walk', 'purpose_confirm': 'exercise'}, 'p': 1.0}
                        # in case append() method changes the dict, we use deepcopy here
                        labels_set = copy.deepcopy(one_set_labels)
                        sec_label_obj.append(labels_set)

                    # put user labels from the 2nd round into a dict, the key is the label from the 2nd round of clustering
                    #e.g. {'0': [{'labels': {'mode_confirm': 'bus', 'replaced_mode': 'bus', 'purpose_confirm': 'home'}, 'p': 1.0}]}
                    sec_round_labels[str(sl)] = sec_label_obj
                sec_round_collect = copy.deepcopy(sec_round_labels)
                # collect all user labels from the 2nd round, the key is to the label from the 1st round
                # e.g. fitst_round_labels = {'0': [{'0': [{'labels': {'mode_confirm': 'drove_alone', 'purpose_confirm': 'work', 'replaced_mode': 'drove_alone'}, 'p': 1.0}]}]}
                first_label_obj.append(sec_round_collect)
                fitst_round_labels[str(fl)] = first_label_obj
            # wrap up all labels
            # e.g. all_labels = [{'first_label': [{'second_label': [{'labels': {'mode_confirm': 'shared_ride',
            # 'purpose_confirm': 'home', 'replaced_mode': 'drove_alone'}, 'p': 1.0}]}]}]
            all_labels = [fitst_round_labels]

            # save all user labels
            save_models('user_labels',all_labels,user)

            # save models from the 2nd round of clustering
            save_models('models',[model_coll],user)

            # save location features of all bins
            save_models('locations',[bin_loc_feat],user)

if __name__ == '__main__':
    logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',
        level=logging.DEBUG)
    main()


# 最短距离算法
import networkx as nx

debug = False
start_node = ('start', -1)    # 初始节点
end_node = ('end', -1)        # 终止节点

def fmt_edges(points, max_score=1.):
    """将节点得分列表格式化成距离矩阵
    :param points list[[left, right, score]]
    :return edges [(node_id1, node_id2, score)]
    :return nodes [(left, right)]
    """
    points_dict = dict()
    for i, j, score in points:
        # 默认值为虚拟节点
        points_dict.setdefault(i, [(-1, max_score)]).append((j, max_score - score))

    edges, last_nodes = [], [start_node]
    nodes = [start_node]
    for left, points_score in points_dict.items():
        curr_nodes = []
        for right, score in points_score:
            node = (left, right)
            curr_nodes.append(node)
            edges += init_edges(last_nodes, node, score)

        nodes += curr_nodes
        last_nodes = curr_nodes

    # 终止节点
    nodes.append(end_node)
    if debug:
        print('edges:', [edge[:2] for edge in edges if edge[0] != start_node])

    edges += init_edges(last_nodes, end_node, 0.)
    node_keys = {val: key for key, val in enumerate(nodes)}
    edges = [(node_keys[f_node], node_keys[t_node], score) for f_node, t_node, score in edges]
    return edges, nodes

def init_edges(last_nodes, point, score):
    """"""
    edges = []
    for last in last_nodes:
        if last[1] >= 0 and point[1] >= 0 and last[1] >= point[1]:
            continue
        edges.append((last, point, score))

    return edges

def shortest_distance(edges, nodes, target=None, source=0):
    """最短距离算法
    :return path list[(left, right)]
    """
    if target is None:
        target = len(nodes) - 1

    G = nx.DiGraph()
    G.add_weighted_edges_from(edges)
    path = nx.dijkstra_path(G, source=source, target=target)
    if debug:
        print('shortest path: ', path)
    path = [p for p in path if p not in set((source, target))]
    path = [nodes[p] for p in path]
    return path

if __name__ == '__main__':
    debug = True

    def test(points):
        edges, nodes = fmt_edges(points)
        path = shortest_distance(edges, nodes)
        print(path)
        print("-------")

    points = [[0, 0, 0.5]]
    test(points)
    points = [[0, 0, 0.7], [0, 1, 0.1], [1, 0, 0.2], [1, 1, 0.6]]
    test(points)
    points = [[0, 0, 0.7], [0, 1, 0.1], [1, 0, 0.8], [1, 1, 0.6]]
    test(points)
    points = [[0, 0, 0.7], [1, 0, 0.1], [1, 1, 0.2], [2, 1, 0.8]]
    test(points)


from aiogram.types import (
    InlineKeyboardMarkup,
    InlineKeyboardButton,
    ReplyKeyboardMarkup,
    KeyboardButton,
)

from main import admins_id
from utils.db_api.schemas.table_db import session, Contest

kb = ReplyKeyboardMarkup(resize_keyboard=True)
kb.add(KeyboardButton("Добавить конкурс")).add(KeyboardButton("Отмена"))

kb_auth = ReplyKeyboardMarkup(resize_keyboard=True)
kb_auth.add(KeyboardButton("Авторизация по телефону", request_contact=True))

def genmarkup(data: list) -> InlineKeyboardMarkup:
    markup = InlineKeyboardMarkup()
    markup.row_width = 1
    for i in data:
        markup.add(InlineKeyboardButton(i[0], callback_data=f"con_{i[0]}"))
    return markup

def to_pay(user_id: int, contest: str, skip: bool = False):
    markup = InlineKeyboardMarkup()
    markup.row_width = 1
    contest_query = session.query(Contest).filter(Contest.name == contest).first()
    if not (contest_query.winner):
        if str(user_id) in admins_id:
            markup.add(
                InlineKeyboardButton(
                    "Выбрать победителя и закончить конкурс",
                    callback_data=f"win_{contest}",
                )
            )
        else:
            if not skip:
                markup.add(
                    InlineKeyboardButton(
                        "Внести плату",
                        callback_data=f"pay_{contest}"
                    )
                )
    return markup


import numpy as np
import cv2
import math
import subprocess
import shutil
import os

if not os.path.exists('/home/martin/fotos'):
    os.makedirs('/home/martin/fotos')
image_sudoku_original = cv2.imread('/home/martin/sudoku/sudoku_recognition/testing3.jpeg')

cv2.imshow("Imagen original",image_sudoku_original)
cv2.waitKey(0)

img = cv2.GaussianBlur(image_sudoku_original,(5,5),0)
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

cv2.imshow("Imagen en escala de grises",gray)
cv2.waitKey(0)

thresh1 = cv2.adaptiveThreshold(gray,255,0,1,19,2)

cv2.imshow("Imagen binarizada",thresh1)
cv2.waitKey(0)

contours, hierarchy = cv2.findContours(thresh1, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

#image_sudoku_candidates = image_sudoku_original.copy()

size_rectangle_max = 0;
biggest = None
max_area = 0
for i in contours:
    area = cv2.contourArea(i)
    if area > 100:
        peri = cv2.arcLength(i, True)
        approximation = cv2.approxPolyDP(i, 0.02 * peri, True)
        if area > max_area and len(approximation) == 4:
            biggest = approximation
            max_area = area

for i in range(len(approximation)):
    cv2.line(image_sudoku_original,
             (biggest[(i % 4)][0][0], biggest[(i % 4)][0][1]),
             (biggest[((i + 1) % 4)][0][0], biggest[((i + 1) % 4)][0][1]),
             (255, 0, 0), 2)

cv2.imshow("Contorno principal",image_sudoku_original)
cv2.waitKey(0)

def rectify(h):
    h = h.reshape((4, 2))
    hnew = np.zeros((4, 2), dtype=np.float32)

    add = h.sum(1)
    hnew[0] = h[np.argmin(add)]
    hnew[2] = h[np.argmax(add)]

    diff = np.diff(h, axis=1)
    hnew[1] = h[np.argmin(diff)]
    hnew[3] = h[np.argmax(diff)]

    return hnew

approx = rectify(biggest)
h = np.array([[0, 0], [449, 0], [449, 449], [0, 449]], np.float32)

retval = cv2.getPerspectiveTransform(approx, h)
warp_gray = cv2.warpPerspective(gray, retval, (450, 450))

h, w = warp_gray.shape[:2]

cv2.imshow("Imagen con cambio perspectiva",warp_gray)
cv2.waitKey(0)

var2 = cv2.adaptiveThreshold(warp_gray,255,0,1,19,2)
#close = cv2.morphologyEx(var2,cv2.MORPH_CLOSE,kernel1)

gauss = cv2.GaussianBlur(warp_gray, (5, 5), 0)
thresh = cv2.adaptiveThreshold(gauss,255,0,1,19,2)

kernel = np.ones((5, 5), np.uint8)
erosion = cv2.erode(thresh, kernel, iterations=1)
dilation = cv2.dilate(thresh, kernel, iterations=1)

closing = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel)

#opening = cv2.morphologyEx(closing, cv2.MORPH_OPEN, kernel)

#
# close = cv2.morphologyEx(thresh,cv2.MORPH_CLOSE,kernel1)
# div = np.float32(warp_gray)/(close)
# res = np.uint8(cv2.normalize(div,div,0,255,cv2.NORM_MINMAX))
# res2 = cv2.cvtColor(res,cv2.COLOR_GRAY2BGR)

#img = cv2.GaussianBlur(var2,(5,5),0)

cv2.imshow("Imagen con Closing",closing)
cv2.waitKey(0)

# cv2.imshow("Imagen ultimo",thresh)
# cv2.waitKey(0)

contours, hierarchy = cv2.findContours(closing, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

def nroCuadrado(x, y,w,h):

    width = 449
    height = 449

    x = x+(w/2)
    y = y+(h/2)

    widthxCuadrado = width / 9
    heightxCuadrado = height / 9

    for i in range(0, 9):
        for j in range(0, 9):
            proximoenAncho = (i + 1) * widthxCuadrado
            actualenAncho = i * widthxCuadrado
            proximoenAlto = (j + 1) * heightxCuadrado
            actualenAlto = j * heightxCuadrado
            if (x >=  actualenAncho and x <= proximoenAncho and y >=  actualenAlto and y <= proximoenAlto):
                return i, j

sudoku_matrix = np.zeros((9,9))
squares = []
size_rectangle_max = 0;
biggest = None
max_area = 0
count = 0
area_total = 0
for i in contours:
   area = cv2.contourArea(i)
   if area > 100:
           approximation = cv2.approxPolyDP(i, 0.04 * peri, True)
           if len(approximation) == 4:
                   area = cv2.contourArea(approximation)
                   if area > 1000 and area <=3000:
                       squares.append(approximation)
                       area = cv2.contourArea(approximation)
                       area_total += area
                       count +=1

                       x, y, w, h = cv2.boundingRect(approximation)
                       #print("X: "+str(x)+" Y: "+str(y)+" W: "+str(w)+ " H: "+str(h))
                       cv2.rectangle(gauss, (y, x), (y + w, x + h), (0, 255, 0), 2)

                       new_image = gauss[x+7:x+h-7, y+7:y+w-7]

                       f, g = nroCuadrado(x, y,w,h)

                       var2 = cv2.adaptiveThreshold(new_image, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, \
                                                     cv2.THRESH_BINARY, 11, 2)

                       name = '/home/martin/fotos/var%s%d.jpg' % (f, g)

                       cv2.imwrite(name, var2)

                       non_black = cv2.countNonZero(var2)
                       total = var2.size

                       percent = (float(non_black)/float(total))*100

                       if percent > 90.0:
                           number = -1
                       else:
                           #number = predict.main(var2)
                           command = name
                           number = subprocess.check_output(['python', 'predict.py', command])
                           #var = 1

                       sudoku_matrix[f][g] = number

                       #print(number)

                       #cv2.imshow("Imagen perspectiva", var2)
                       #cv2.waitKey(0)
                       #name = '/home/lcorniglione/Documents/sudoku_recognition/fotos/var%s%d.jpg' %(f,g)

result = (area_total/count)
area_prom = math.sqrt(result)

print ("CANTIDAD RECONOCIDA:")
print (len(squares))

cant_squares = len(squares)
for i in range(0,9):
        for j in range(0,9):
            num = sudoku_matrix[i][j]
            if num==(-1.0):
                sudoku_matrix[i][j] = 0
            if num==(0.0) and cant_squares<81:

                im_number = gauss[i * (area_prom + 8):(i+1) * (area_prom + 8)][:,
                            j * (area_prom + 8):(j+1) * (area_prom + 8)]

                var2 = cv2.adaptiveThreshold(im_number, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, \
                                             cv2.THRESH_BINARY, 11, 2)

                non_black = cv2.countNonZero(var2)
                total = var2.size

                percent = (float(non_black) / float(total)) * 100

                name = '/home/martin/fotos/var%s%d.jpg' % (i, j)
                cv2.imwrite(name, var2)

                if percent > 85.0:
                    number = -1
                else:
                    command = name
                    number = subprocess.check_output(['python', 'predict.py', command])

                sudoku_matrix[i][j] = number

print ("FINALIZADO")
print (sudoku_matrix)

cv2.imshow("Imagen cuadrados", gauss)
cv2.waitKey(0)

shutil.rmtree('/home/martin/fotos')


import pytest

from backend.utils.assertions import assert_equals, assert_true
from backend.utils.helper import Helper
from front.data_for_tests.calender_data_for_tests import DataForTests
from front.pages.page import CalendarPage, CalendarConfiguration

@pytest.mark.usefixtures("setup", "test_config")
class TestCalender:

    @pytest.fixture(scope="class")
    def calendar_page(self, setup, test_config):
        driver = setup
        calendar_page = CalendarPage(driver)
        calendar_page.open_url(Helper.get_config_value_by_name(test_config, ["calender", "url"]))
        return calendar_page

    @pytest.fixture(scope="class")
    def switch_to_infinite_scroll_and_month_view(self, setup, calendar_page):
        driver = setup
        calendar_config = CalendarConfiguration(driver)
        calendar_config.enable_infinite_scroll()
        calendar_page.switch_view("Month")

    @pytest.mark.parametrize("test_data", DataForTests.switch_to_infinite_scroll_and_month_view())
    def test_switch_to_infinite_scroll_and_month_view(self, setup, test_data, calendar_page,
                                                      switch_to_infinite_scroll_and_month_view):
        driver = setup
        calendar_config = CalendarConfiguration(driver)
        calendar_configuration = calendar_config.get_calendar_configuration()
        assert_equals(calendar_configuration, test_data["configuration"])
        calendar_page.verify_requested_view_checked(test_data["view_type"])

    def test_create_events_check_element_count_increased(self, calendar_page,
                                                         switch_to_infinite_scroll_and_month_view):
        event_resource_id = calendar_page.create_event()
        assert_true(calendar_page.verify_event_was_created(event_resource_id),
                    "Error: After creating an event, Event not found!")
        event_resource_id = calendar_page.create_event()
        assert_true(calendar_page.verify_event_was_created(event_resource_id),
                    "Error: After creating an event, Event not found!")

    def test_create_event_and_go_ahead_one_month_and_check_dom_decrease(self, setup, calendar_page,
                                                                        switch_to_infinite_scroll_and_month_view):
        event_resource_id = calendar_page.create_event()
        calendar_page.navigation_forward(1)
        assert_true(not calendar_page.verify_event_was_created(event_resource_id),
                    "Error: After creating an event and going one month forward, Event found!")

    @pytest.mark.skip("This test have bug, so it will failed")
    def test_create_event_change_month_and_check_event_still_exist(self, setup, calendar_page,
                                                                   switch_to_infinite_scroll_and_month_view):
        event_resource_id = calendar_page.create_event()
        calendar_page.navigation_forward(1)
        assert_true(not calendar_page.verify_event_was_created(event_resource_id),
                    "Error: After creating an event and going one month forward, Event found!")
        calendar_page.navigation_backward(1)
        assert_true(calendar_page.verify_event_was_created(event_resource_id),
                    "Error: After creating an event, change month and go back.Event not found!")


import cant_utils as cu
import numpy as np
import matplotlib.pyplot as plt
import glob
import bead_util as bu
import tkinter
import tkinter.filedialog
import os, sys
from scipy.optimize import curve_fit
import bead_util as bu
from scipy.optimize import minimize_scalar as minimize
import pickle as pickle
import time

####################################################
####### Input parameters for data processing #######

TESTING = True

ddict = bu.load_dir_file( "/home/charles/opt_lev_analysis/scripts/dirfiles/dir_file_june2017.txt" )
#print ddict

pow_axis = 4

cant_axis = 1           # stage control axis
straighten_axis = 2     # axis with coherent drive to straighten
fit_pows = True

load_charge_cal = True
maxfiles = 1000

plot_forward_backward = False #True
#subtract_background = True

drivefreq = 18.0
cant_volts_to_um = 8.0    # 80 um / 10 V

#fig_title = ('Force vs. Cantilever Position: %s Hz, %s - %s, ' + bead) % (drivefreq, gas, num)

#dirs = [1,2,3,4,5,6,7]
dirs = [8,9,10,11,12,13,14,15,16]

tf_path = './trans_funcs/Hout_20160808.p'

step_cal_path = './calibrations/step_cal_20160808.p'

thermal_cal_file_path = '/data/20160808/bead1/1_5mbar_zcool_final.h5'

def poly2(x, a, b, c):
    return a * (x - b)**2 + c

def proc_dir(d):
    dv = ddict[d]

    dir_obj = cu.Data_dir(dv[0], [0,0,dv[-1]], dv[1])
    dir_obj.load_dir(cu.simple_loader, maxfiles = maxfiles)

    amps = []
    for fil_obj in dir_obj.fobjs:
        fil_obj.psd()
        stagestuff = fil_obj.get_stage_settings(axis=straighten_axis)
        amp = stagestuff[2] * cant_volts_to_um
        amps.append(amp)
    uamps = np.unique(amps)
    if len(uamps) > 1:
        print('STUPIDITYERROR: Multiple dirve amplitudes in directory')

    newlist = []
    for i in [0,1,2]:
        if i == straighten_axis:
            newlist.append(uamps[0])
        else:
            newlist.append(0.0)
    dir_obj.drive_amplitude = newlist

    return dir_obj

dir_objs = list(map(proc_dir, dirs))

colors_yeay = bu.get_color_map( len(dir_objs) )

psds = {}
pows = {}
bpows = {}

for ind, obj in enumerate(dir_objs):
    psd = []

    col = colors_yeay[ind]

    amp = obj.drive_amplitude[straighten_axis]

    filcount = 0
    for fobj in obj.fobjs:
        filcount += 1
        fobj.psd()
        if not len(psd):
            freqs = fobj.other_psd_freqs
            psd = fobj.other_psds[pow_axis-3]
        else:
            psd += fobj.other_psds[pow_axis-3]

    psd = psd / float(filcount)
    psds[amp] = psd

    ind = np.argmin(np.abs(freqs - drivefreq))

    totpow = np.sum(psd[ind-1:ind+2])
    pows[amp] = totpow

    badind = int(ind*1.5)
    totbadpow = np.sum(psd[badind-1:badind+2])
    bpows[amp] = totbadpow

amps = list(pows.keys())
amps.sort()
powsarr = []
bpowsarr = []
for amp in amps:

    powsarr.append(pows[amp])
    bpowsarr.append(bpows[amp])

if fit_pows:
    p0 = [1, 0, 0]
    popt, pcov = curve_fit(poly2, amps, powsarr, p0 = p0, maxfev = 10000)

    fitpoints = np.linspace(amps[0], amps[-1], 100)
    fit = poly2(fitpoints, *popt)

    plt.plot(amps, powsarr, 'o')
    plt.plot(fitpoints, fit, color='r', linewidth=1.5)

    title = 'Best fit straightening amplitude: %0.2g um' % popt[1]
    plt.title(title)

else:
    plt.plot(amps, powsarr)
    plt.plot(amps, bpowsarr)

plt.show()


from re import T
from django.db import models

from machine.computations.examples import victorious_payment
from django.contrib.postgres.fields import ArrayField

# Create your models here.

class Machine(models.Model):

    def empty_list():
        return list()
    # identificar la maquina que estamos usando
    name = models.CharField(max_length=200, blank=True)

    payments = models.JSONField()
    free_spins = ArrayField(
        models.IntegerField(blank=True),
        size=5,
        default=empty_list
    )

    normal_reel = ArrayField(
        models.CharField(max_length=200, blank=True),
        size=5,
        default=empty_list
    )
    bonus_reel = ArrayField(
        models.CharField(max_length=200, blank=True),
        size=5,
        default=empty_list
    )
    visible = ArrayField(
        models.IntegerField(blank=True),
        size=5,
        default=empty_list
    )
    multiplier = models.IntegerField(default=3)

    # corregir:
    # ver si se puede mejorar
    roi = models.FloatField(default=0)

    def payment(self, roll):
        return victorious_payment(self, roll)

    def save(self, *args, **kwargs):
        self.roi += 0.01
        super(Machine, self).save(*args, **kwargs)

    def __str__(self) -> str:
        return self.name


# Importar Librerias
import pandas as pd
import json

# Opening JSON file
f = open('orderbooks_05jul21.json')
print(f)

# Returns JSON object as a dictionary
orderbooks_data = json.load(f)
ob_data = orderbooks_data['bitfinex']

# Drop Keys with none values
ob_data = {i_key: i_value for i_key,i_value in ob_data.items() if i_value is not None}

# Convert to DataFrame and rearange columns
ob_data = {i_ob: pd.DataFrame(ob_data[i_ob])[['bid_size', 'bid', 'ask', 'ask_size']]
          if ob_data[i_ob] is not None else None for i_ob in list(ob_data.keys())}


import json
import os
import random
import shutil
from predictor import get_predictor
from yolox.tracking_utils.timer import Timer
import cv2
import numpy as np

def get_gt_by_frame(bbox_file: str):
    gtByFrames = {}
    # convert to List[bboxes, List[int]]
    with open(bbox_file) as f:
        annot = json.load(f)
    labels = annot['labels']
    infos = annot['info']
    frames_map = [int(u.split('_')[-1].split('.')[0]) for u in infos['url']]
    for pIdx, player in enumerate(labels):
        frames = player['data']['frames']
        #group by frame
        for frame in frames:
            frame_idx = frames_map[frame['frame']]
            if frame_idx not in gtByFrames:
                gtByFrames[frame_idx] = [[], []]
            gtByFrames[frame_idx][0].append(frame['points'] + [1])
            gtByFrames[frame_idx][1].append(pIdx)
    for k, v in gtByFrames.items():
        gtByFrames[k] = (np.array(v[0]), np.array(v[1]))

    return gtByFrames

def ensure_folder(path: str):
    try:
        os.makedirs(path)
    except: pass

def remove_folder(path:str):
    try:
        shutil.rmtree(path)
    except: pass

if __name__ == '__main__':
    gt_labels = os.listdir('./input/bboxes')
    detector = get_predictor()
    OUTPUT_ROOT = './output'
    remove_folder(OUTPUT_ROOT)
    ensure_folder(OUTPUT_ROOT)
    splits = ['train', 'val', 'test']
    for split in splits:
        ensure_folder(OUTPUT_ROOT + f'/{split}/negative')
        ensure_folder(OUTPUT_ROOT + f'/{split}/positive')

    for gt_label_file in gt_labels:
        gt_bboxes = get_gt_by_frame(f'./input/bboxes/{gt_label_file}')
        video_id = gt_label_file.split('.')[0]
        print(video_id)

        if not os.path.exists(f'./input/images/{video_id}'):
            continue

        # extract positive patches
        for frame_idx, (player_bboxes, player_ids) in gt_bboxes.items():
            img = cv2.imread(f'./input/images/{video_id}/{frame_idx}.jpg')
            img_masked = img.copy()
            for bbox in player_bboxes:
                x1, y1, x2, y2, _ = bbox.astype(int)
                img_masked[y1:y2, x1:x2] = 0
            cv2.imwrite(OUTPUT_ROOT + '/masked.png', img_masked)
            outputs, img_info = detector.inference(img_masked[:, :, :3], Timer())

            output_results = outputs[0]
            if output_results is None: break
            imgH, imgW = img_info['height'], img_info['width']

            # human_bboxes = []
            output_results = output_results.cpu().numpy()
            scores = output_results[:, 4] * output_results[:, 5]
            human_bboxes = output_results[:, :4]  # x1y1x2y2
            remain_indx = scores > 0.6
            scores = scores[remain_indx]
            human_bboxes = human_bboxes[remain_indx]

            img_size = (800, 1440)
            scale = min(img_size[0] / float(imgH), img_size[1] / float(imgW))
            human_bboxes /= scale

            # negative samples
            negative_samples = [(bIdx, b) for bIdx, b in enumerate(human_bboxes) if b.min() > 0]
            negative_samples = random.sample(negative_samples, min(20, len(negative_samples)))
            random.shuffle(negative_samples)
            train_split_idx = int(len(negative_samples) * 0.7)
            val_split_idx = int(len(negative_samples) * 0.8)
            for idx, (bIdx, bbox) in enumerate(negative_samples):
                split = 'train' if idx < train_split_idx else ('val' if idx < val_split_idx else 'test')
                x1, y1, x2, y2 = bbox.astype(int)
                cv2.imwrite(f'{OUTPUT_ROOT}/{split}/negative/{video_id}_{frame_idx}_{bIdx}.png', img[y1:y2, x1:x2])

            positive_samples = list(zip(player_ids, player_bboxes))
            random.shuffle(positive_samples)
            train_split_idx = int(len(positive_samples) * 0.7)
            val_split_idx = int(len(positive_samples) * 0.8)
            for idx, (player_id, player_bboxe) in enumerate(positive_samples):
                split = 'train' if idx < train_split_idx else ('val' if idx < val_split_idx else 'test')
                x1, y1, x2, y2, _ = player_bboxe.astype(int)
                x1 = max(x1, 0)
                try:
                    cv2.imwrite(f'{OUTPUT_ROOT}/{split}/positive/{video_id}_{frame_idx}_{player_id}.png', img[y1:y2, x1:x2])
                except:
                    print(f'{OUTPUT_ROOT}/{split}/positive/{video_id}_{frame_idx}_{player_id}.png', x1, y1, x2, y2)


import pathlib
import json

import numpy as np
import pandas as pd
from scipy.interpolate import SmoothBivariateSpline, UnivariateSpline
import matplotlib as mpl
import matplotlib.pyplot as plt

from .common import Timer, Tools
from .sim_ctr import RgbGrid
from .sim_reduce import Steps, ReduceModel

class SynthGrid:

    POPT_PATH = 'rgb_calibr/Salaris-off_vary-both.json'
    AMLT_KEY = 'ms2_mt1'
    AMLT_MODEL = lambda x, a, b1, b2, c1: a + b1*x[0] + b2*x[0]**2 + c1*x[1]

    OUT_MASS_LIST = [0.9, 1. , 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8]
    OUT_FEH_LIST = [-0.4, -0.3, -0.2, -0.1,  0. ,  0.1,  0.2,  0.3,  0.4]
    OUT_MASS_MINI = [1. , 1.4, 1.8]
    OUT_FEH_MINI = [-0.4,  0. ,  0.4]

    def __init__(self, aMLT_list: [float], mass_list: [float] = RgbGrid.MASS_LIST,
                 FeH_list: [float] = RgbGrid.FEH_LIST, **kwargs):
        self.indir = pathlib.Path('rgb_grid')
        assert self.indir.exists(), 'rgb_grid does not exist'
        self.outdir = pathlib.Path('synth_grid')
        self.outdir.mkdir(exist_ok=True)

        popt_path = pathlib.Path(SynthGrid.POPT_PATH)
        assert popt_path.exists(), 'popt_path does not exist'
        with open(popt_path, 'r') as f:
            popt_dict = json.load(f)
        self.aMLT_popt = popt_dict[SynthGrid.AMLT_KEY]

        self.aMLT_list = aMLT_list
        self.mass_list = mass_list
        self.FeH_list = FeH_list
        self.kwargs = kwargs  # Ybirth, Zbirth, Z_over_X_sun, YBBN

        self.timer = Timer()
        self()

    def __call__(self):
        self.extract_sim_data()
        self.build_interps()

        for mass in SynthGrid.OUT_MASS_LIST:
            for FeH in SynthGrid.OUT_FEH_LIST:
                aMLT_fit = SynthGrid.AMLT_MODEL([mass-1, FeH], *self.aMLT_popt)
                self.synthesize_model(mass, FeH, aMLT_fit)

        self.clear()
        print(' > All models synthesized!', '@', self.timer(), flush=True)

    def extract_sim_data(self):
        shape = (len(self.aMLT_list), Steps.end+1,
                 len(self.mass_list), len(self.FeH_list))
        self.existence = np.ones(shape[:1] + shape[2:], dtype=bool)
        self.data_dict = {qty: np.zeros(shape) for qty in ReduceModel.QTY_LIST}

        for k, aMLT in enumerate(self.aMLT_list):
            for j, mass in enumerate(self.mass_list):
                for i, FeH in enumerate(self.FeH_list):
                    Y, Z = RgbGrid.Y_Z_calc(FeH, **self.kwargs)
                    model = SynthModel(self, aMLT=aMLT, mass=mass, Z=Z, FeH=FeH)

                    if model.exists:
                        for qty in ReduceModel.QTY_LIST:
                            self.data_dict[qty][k, :, j, i] = model.data[qty]
                    else:
                        self.existence[k, j, i] = False

                    model.clear_data(); del model

    def build_interps(self):
        self.interp_dict = {}
        for qty in ReduceModel.QTY_LIST:
            self.interp_dict[qty] = [[None for step in range(Steps.end+1)]
                                           for aMLT in self.aMLT_list]

            for k in range(len(self.aMLT_list)):
                for step in range(Steps.end+1):
                    self.interp_dict[qty][k][step] = SmoothBivariateSpline(
                        self.data_dict['star_mass']     [k, step][self.existence[k]],
                        self.data_dict['surface_[Fe/H]'][k, step][self.existence[k]],
                        self.data_dict[qty]             [k, step][self.existence[k]], kx=2, ky=2)

    def synthesize_model(self, mass: float, FeH: float, aMLT_fit: float):
        Y, Z = RgbGrid.Y_Z_calc(FeH, **self.kwargs)
        model_name = f'{mass:.2f}M_Z={Z:.4f}_FeH={FeH:+.2f}'
        print(' > Synthesizing', model_name, '@', self.timer())

        pred = {}; data = {}
        for qty in ReduceModel.QTY_LIST:
            pred[qty] = np.zeros((len(self.aMLT_list), Steps.end+1))
            data[qty] = np.zeros(Steps.end+1)

        for qty in ReduceModel.QTY_LIST:
            for step in range(Steps.end+1):
                for k, aMLT in enumerate(self.aMLT_list):
                    pred[qty] [k, step] = self.interp_dict[qty][k][step](mass, FeH)[0, 0]
                data[qty][step] = UnivariateSpline(self.aMLT_list, pred[qty][:, step], k=1)(aMLT_fit)

        if mass in SynthGrid.OUT_MASS_MINI and FeH in SynthGrid.OUT_FEH_MINI:
            self._visualize_data(model_name, pred, data, aMLT_fit)

        df = pd.DataFrame(data)
        df.to_csv(self.outdir / f'{model_name}.csv')
        pred.clear(); data.clear()
        del df, pred, data

    def _draw_curve(self, pred, data, ax, x, y, colors):
        for k, aMLT in enumerate(self.aMLT_list):
            ax.plot(pred[x][k], pred[y][k], '--', c=colors[k])
        ax.plot(data[x], data[y], '-', c=colors[-1])

        if x in ['Teff', 'log_g']: ax.invert_xaxis()
        if y in ['Teff', 'log_g']: ax.invert_yaxis()

        for step in range(Steps.end+1):
            ax.plot([pred[x][k, step] for k in range(len(self.aMLT_list))],
                    [pred[y][k, step] for k in range(len(self.aMLT_list))],
                    ls='-', lw=0.5, c='lightgrey', zorder=-1)

        for EEP in ['mid_PMS', 'ZAMS', 'mid_MS', 'TAMS', 'mid_SGB',
                    'pre_FDU', 'post_FDU', 'pre_RGBB', 'post_RGBB']:
            idx = getattr(Steps, EEP)
            color = getattr(Steps, f'{EEP}_c', 'tab:cyan')
            ax.plot([pred[x][k, idx] for k in range(len(self.aMLT_list))],
                    [pred[y][k, idx] for k in range(len(self.aMLT_list))],
                    ls='-', lw=0.5, c=color, zorder=-1)
            ax.plot(data[x][idx], data[y][idx], 'o', c=color, ms=4)

        ax.set_xlabel(x)
        ax.set_ylabel(y)
        Tools.format_axis(ax)

    def _visualize_data(self, model_name, pred, data, aMLT_fit):
        cmap = mpl.colormaps['summer_r']
        norm = mpl.colors.Normalize(vmin=self.aMLT_list[0],
                                    vmax=self.aMLT_list[-1])
        colors = [cmap(norm(a)) for a in self.aMLT_list + [aMLT_fit]]

        # draw evolutionary tracks
        fig, axs = plt.subplots(1, 2)
        self._draw_curve(pred, data, axs[0], 'Teff', 'log_L', colors)
        self._draw_curve(pred, data, axs[1], 'Teff', 'log_g', colors)
        for i in range(2):
            fig.colorbar(mpl.cm.ScalarMappable(norm=norm, cmap=cmap),
                         ax=axs[i], orientation='horizontal', label=r'mixing length $\alpha$')
        Tools.save_figure(fig, 'tracks')

        # draw coordinates
        fig, axs = plt.subplots(2, 1)
        self._draw_curve(pred, data, axs[0], 'star_age', 'model_number', colors)
        self._draw_curve(pred, data, axs[1], 'model_number', 'star_age', colors)
        for i in range(2):
            fig.colorbar(mpl.cm.ScalarMappable(norm=norm, cmap=cmap),
                         ax=axs[i], orientation='vertical', label=r'ML $\alpha$')
        Tools.save_figure(fig, 'coords')

        # draw histories
        for qty in ReduceModel.QTY_LIST[2:]:
            fig, axs = plt.subplots(2, 1)
            self._draw_curve(pred, data, axs[0], 'star_age',     qty, colors)
            self._draw_curve(pred, data, axs[1], 'model_number', qty, colors)
            for i in range(2):
                fig.colorbar(mpl.cm.ScalarMappable(norm=norm, cmap=cmap),
                             ax=axs[i], orientation='vertical', label=r'ML $\alpha$')
            Tools.save_figure(fig, qty.replace('/', '_'))

        Tools.merge_plots(self.outdir, model_name, ['tracks', 'coords'] \
                          + [qty.replace('/', '_') for qty in ReduceModel.QTY_LIST[2:]])

    def clear(self):
        for qty in ReduceModel.QTY_LIST:
            for k in range(len(self.aMLT_list)):
                self.interp_dict[qty][k].clear()
            self.interp_dict[qty].clear()
        self.data_dict.clear()
        self.interp_dict.clear()
        del self.existence, self.data_dict, self.interp_dict

class SynthModel:

    def __init__(self, grid: SynthGrid, **kwargs) -> None:
        self.grid = grid
        self.model_name = f'aMLT={kwargs["aMLT"]:.4f}_{kwargs["mass"]:.2f}M_' \
                          f'Z={kwargs["Z"]:.4f}_FeH={kwargs["FeH"]:+.2f}'

        fpath = grid.indir / f'{self.model_name}.csv'
        self.exists = fpath.exists()
        if not self.exists:
            print(f' > Warning: {self.model_name} does not exist.')
            return
        self.data = pd.read_csv(fpath, index_col=0)

    def clear_data(self):
        if self.exists:
            del self.data


# -*- coding: utf-8 -*-
import time

import click
from click.testing import CliRunner
from dagster_aws.cli.term import Spinner, Term

def test_term():
    def term_helper(term_cmd, prefix, exit_code=0):
        @click.command()
        def fn():
            term_cmd('foo bar')

        runner = CliRunner()
        result = runner.invoke(fn)
        assert result.exit_code == exit_code
        assert result.output == prefix + u'foo bar\n'

    expected = [
        (Term.error, Term.ERROR_PREFIX),
        (Term.info, Term.INFO_PREFIX),
        (Term.success, Term.SUCCESS_PREFIX),
        (Term.waiting, Term.WAITING_PREFIX),
        (Term.warning, Term.WARNING_PREFIX),
    ]
    for term_cmd, prefix in expected:
        term_helper(term_cmd, prefix)

    term_helper(Term.fatal, Term.FATAL_PREFIX, exit_code=1)

def test_spinner(capsys):
    with Spinner():
        time.sleep(0.5)

    captured = capsys.readouterr()
    assert captured.out.encode('unicode-escape').startswith(
        b'\\u280b\\x08\\u2819\\x08\\u2839\\x08\\u2838\\x08'
    )


import logging

import cabby

from events.stix import parse_stix_package, STIXPackage

def collect_indicator_packages(configuration: dict) -> STIXPackage:
    for repository in configuration['repositories']:
        yield from poll_repository(repository)

def poll_repository(repository: dict) -> list:
    logging.debug("Connecting to %s", repository['name'])

    client = cabby.create_client(**repository['client'])
    collections = (c for c in client.get_collections()
                   if c.name not in repository.get('exclusions', ()))

    for collection in collections:
        yield from poll_collection(client, collection.name)

    logging.info("Repository %s exhausted", repository['name'])

def poll_collection(client: cabby.Client11, collection_name: str) -> list:
    packages = 0
    indicators = 0

    logging.debug("Polling from collection %s", collection_name)

    for block in client.poll(collection_name=collection_name):
        package = parse_stix_package(block.content)

        if package is not None:
            packages += 1
            indicators += len(package.indicators)

            yield package

    logging.info("Collection %s: Packages %d - IOCs %d",
                 collection_name, packages, indicators)


from __future__ import division
import scipy.optimize
import numpy as np
import json
import re
import cv2
_BLACK = (0, 0, 0)
_RED = (0, 0, 255)
_BLUE = (255, 0, 0)
_PURPLE = (204, 0, 153)
_ORANGE = (51, 153, 255)
_LBROWN = (0, 153, 230)
keypoint_colors = { '1': _RED, '2': _RED, '3': _RED, '4': _RED, '5': _RED,
                            '6': _ORANGE, '7': _ORANGE, '8': _ORANGE, '9': _ORANGE,
                            '10': _LBROWN, '11': _LBROWN, '12': _LBROWN, '13': _LBROWN,
                            '14': _BLUE, '15': _BLUE, '16': _BLUE, '17': _BLUE,
                            '18': _PURPLE, '19': _PURPLE, '20': _PURPLE, '21': _PURPLE
                            }

def bbox_iou(boxA, boxB):
	# https://www.pyimagesearch.com/2016/11/07/intersection-over-union-iou-for-object-detection/
	# ^^ corrected.

	# Determine the (x, y)-coordinates of the intersection rectangle
	xA = max(boxA[0], boxB[0])
	yA = max(boxA[1], boxB[1])
	xB = min(boxA[2], boxB[2])
	yB = min(boxA[3], boxB[3])

	interW = xB - xA + 1
	interH = yB - yA + 1

	# Correction: reject non-overlapping boxes
	if interW <=0 or interH <=0 :
		return -1.0

	interArea = interW * interH
	boxAArea = (boxA[2] - boxA[0] + 1) * (boxA[3] - boxA[1] + 1)
	boxBArea = (boxB[2] - boxB[0] + 1) * (boxB[3] - boxB[1] + 1)
	iou = interArea / float(boxAArea + boxBArea - interArea)
	return iou

def match_bboxes(bbox_gt, bbox_pred, IOU_THRESH=0.05):
	'''
	Given sets of true and predicted bounding-boxes,
	determine the best possible match.
	Parameters
	----------
	bbox_gt, bbox_pred : N1x4 and N2x4 np array of bboxes [x1,y1,x2,y2].
	  The number of bboxes, N1 and N2, need not be the same.

	Returns
	-------
	(idxs_true, idxs_pred, ious, labels)
		idxs_true, idxs_pred : indices into gt and pred for matches
		ious : corresponding IOU value of each match
		labels: vector of 0/1 values for the list of detections
	'''
	n_true = bbox_gt.shape[0]
	n_pred = bbox_pred.shape[0]
	MAX_DIST = 1.0
	MIN_IOU = 0.0

	# NUM_GT x NUM_PRED
	iou_matrix = np.zeros((n_true, n_pred))
	for i in range(n_true):
		for j in range(n_pred):
			iou_matrix[i, j] = bbox_iou(bbox_gt[i,:], bbox_pred[j,:])

	if n_pred > n_true:
	  # there are more predictions than ground-truth - add dummy rows
	  diff = n_pred - n_true
	  iou_matrix = np.concatenate( (iou_matrix,
									np.full((diff, n_pred), MIN_IOU)),
								  axis=0)

	if n_true > n_pred:
	  # more ground-truth than predictions - add dummy columns
	  diff = n_true - n_pred
	  iou_matrix = np.concatenate( (iou_matrix,
									np.full((n_true, diff), MIN_IOU)),
								  axis=1)

	# call the Hungarian matching
	idxs_true, idxs_pred = scipy.optimize.linear_sum_assignment(1 - iou_matrix)

	if (not idxs_true.size) or (not idxs_pred.size):
		ious = np.array([])
	else:
		ious = iou_matrix[idxs_true, idxs_pred]

	# remove dummy assignments
	sel_pred = idxs_pred<n_pred
	idx_pred_actual = idxs_pred[sel_pred]
	idx_gt_actual = idxs_true[sel_pred]
	ious_actual = iou_matrix[idx_gt_actual, idx_pred_actual]
	sel_valid = (ious_actual > IOU_THRESH)
	label = sel_valid.astype(int)

	return idx_gt_actual[sel_valid], idx_pred_actual[sel_valid], ious_actual[sel_valid], label

def cull_annotations():
	gt = json.load(open("train_validated_keypoints_only.json", "r"))
	boot = json.load(open("hand_keypoints_train_bootstrap.json", "r"))

	# for each image in GT, find entries in boot
	gt_images = gt["images"]
	gt_anns = {}
	boot_anns = {}
	boot_imgs = {}
	max_img_id = 0
	max_ann_id = 0

	for img in gt_images:
		img_id = img["id"]
		# if img_id > max_img_id:
		# 	max_img_id = img_id

		img["id"] = max_img_id + 1
		gt_anns[img["file_name"]] = []

		for ann in gt["annotations"]:
			# if ann["id"] > max_ann_id:
			# 	max_ann_id = ann["id"]

			if ann["image_id"] == img_id:
				ann["image_id"] = img["id"]
				ann["id"] = max_ann_id
				gt_anns[img["file_name"]].append(ann)

				max_ann_id += 1

		max_img_id += 1

	for img in boot["images"]:
		img_id = img["id"]
		boot_imgs[img["file_name"]] = img
		boot_anns[img["file_name"]] = []

		for ann in boot["annotations"]:
			if ann["image_id"] == img_id:
				boot_anns[img["file_name"]].append(ann)

	boot_new_images = []
	count = 0
	total = 0
	imgs_to_remove = set()
	for gt_img in gt_images:
		img_name = gt_img["file_name"]

		vid_id = re.sub("-\d{9}.jpg", "", img_name)
		frame = int(re.search("\d{9}", img_name).group())

		boot_frames = set(["{vid_id}-{frame:09d}.jpg".format(vid_id=vid_id, frame=frame+i) for i in range(-5, 6)])
		boot_frames.remove(img_name)

		# for each entry in boot, match bboxes (x, y, w, h)
		gt_bboxes = []
		for x in gt_anns[img_name]:
			gt_bboxes.append([x["bbox"][0], x["bbox"][1], x["bbox"][0] + x["bbox"][2], x["bbox"][1] + x["bbox"][3]])
		gt_bboxes = np.array(gt_bboxes)

		for f in boot_frames:

			boxes = []
			if f not in boot_anns:
				continue

			# reassign image
			boot_imgs[f]["id"] = max_img_id + 1

			for x in boot_anns[f]:
				boxes.append([x["bbox"][0], x["bbox"][1], x["bbox"][0] + x["bbox"][2], x["bbox"][1] + x["bbox"][3]])
			boxes = np.array(boxes)

			idx_true, idxs_pred, ious, labels = match_bboxes(gt_bboxes, boxes)

			total += 1
			if len(idxs_pred) >= 1:
				count += 1

				# image_debug = cv2.imread("./bootstrap/" + f)

				# find matched boxes + corresponding annotations
				gt_using = [gt_anns[img_name][xi] for xi in idx_true]
				boot_using = [boot_anns[f][xi] for xi in idxs_pred]

				# eliminate keypoints
				remove_idxs = []
				using = []
				for b in range(len(boot_using)):
					gt_ann = gt_using[b]
					boot_ann = boot_using[b]

					boot_ann["image_id"] = max_img_id + 1
					boot_ann["id"] = max_ann_id + 1

					keypoints = boot_ann["keypoints"]

					for i in range(21):
						if gt_ann["keypoints"][i * 3 + 2] == 0:
							keypoints[i * 3] = 0
							keypoints[i * 3 + 1] = 0
							keypoints[i * 3 + 2] = 0
						# else:
						# 	cv2.circle(image_debug, (int(keypoints[i * 3]), int(keypoints[i * 3 + 1])), 4, keypoint_colors[str(i + 1)], -1)
						# 	cv2.putText(image_debug, str(i + 1), (int(keypoints[i * 3] - 4), int(keypoints[i * 3 + 1] - 4)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1, cv2.LINE_AA)

					if sum(keypoints) == 0:
						remove_idxs.append(b)
						continue
					using.append(boot_ann)

					boot_ann["keypoints"] = keypoints

					box = boot_ann["bbox"]
					# cv2.rectangle(image_debug, (int(box[0]), int(box[1])), (int(box[2] + box[0]) , int(box[3] + box[1])), color=(0, 255, 0),thickness=3)
					# cv2.rectangle(image_debug, (int(box[0]), int(box[1])), (int(box[2]) , int(box[3])), color=(0, 255, 0),thickness=3)

					# fix error bbox coco error here # Error is in train _bootstrap
					boot_ann["bbox"] = [box[0], box[1], box[2] - box[0], box[3] - box[1]] # x y w h
					boot_ann["area"] = (box[2] - box[0]) * (box[3] - box[1])

					max_ann_id += 1

				# remove image if no more annotations
				if len(remove_idxs) == len(boot_using):
					imgs_to_remove.add(f)

				# boot_using = [x for i, x in enumerate(boot_using) if i not in remove_idxs]

				# add to new annotations
				boot_new_images.extend(using)

				# save image for debugging
				# cv2.imwrite("./debug_bootstrap_train_filter/debug_" + f, image_debug)
			else:
				# eliminate image
				imgs_to_remove.add(f)

				print(f)

			max_img_id += 1

	print(len(boot_new_images))
	boot["annotations"] = boot_new_images
	for v in gt_anns.values():
		boot["annotations"].extend(v)
	print(len(boot["annotations"]))

	boot["images"] = list(boot_imgs.values())
	boot["images"].extend(gt["images"])
	print(len(boot["images"]), len(imgs_to_remove), len(gt["images"]))

	removed = set()
	imgs_cpy = list(boot["images"])
	for img in imgs_cpy:
		if img["file_name"] in imgs_to_remove:
			boot["images"].remove(img)
			removed.add(img["id"])
			if img in gt["images"]:
				print("overlap")

	print(len(boot["images"]), len(boot["annotations"]))
	# remove annotations on removed images
	cpy = list(boot["annotations"])
	for ann in cpy:
		if ann["image_id"] in removed:
			boot["annotations"].remove(ann)
			print("hi")

	json.dump(boot, open("train_boostrap_filtered_validated_2.json", "w"))

	print(count, total)

	# get rid of unmatched boxes

	# get rid of keypoints that aren't labled, occluded, w/in a certain distance

	# write to image so you can see what they now look like

cull_annotations()


from flask import Flask,render_template,request,redirect,session,flash
app = Flask(__name__)
app.secret_key = 'Farn'

@app.route ('/')
def index():

    return render_template('index.html')

@app.route ('/result', methods=['POST'])
def result():
    if len(request.form['name']) < 1:
        flash("Name cannot be empty!")
        return redirect('/')

    # else:
    #     flash("Success! Your name is {}".format(request.form['name']))
    elif len(request.form['comment']) < 1:
        flash("Comments cannot be empty!")
        return redirect('/')
    elif len(request.form['comment']) >= 120:
        flash("Comments cannot be longer than 120 char.!")
        return redirect('/')
    else:
        flash("Success!")

    return render_template('result.html', name = request.form['name'], location = request.form['location'], language = request.form['language'], comment = request.form['comment'])

app.run(debug = True)


# -*- coding: utf-8 -*-
"""
Created on Tue Sep  7 17:46:55 2021

@author: Administrator
"""
import SimpleITK as sitk
import numpy as np
import os
import cv2
from shutil import copyfile
import random

num=379
lists=['train_002_0000.nii.gz','train_019_0000.nii.gz','train_069_0000.nii.gz','train_101_0000.nii.gz','train_114_0000.nii.gz','train_127_0000.nii.gz','train_150_0000.nii.gz','train_134_0000.nii.gz','train_174_0000.nii.gz','train_195_0000.nii.gz']
for name in lists:
    num=num+1
    copyfile(os.path.join(r'.\data\Raw\TrainingImg',name),os.path.join(r'.\data\Raw\TrainingImg','train_'+str(num)+'_0000.nii.gz'))
    copyfile(os.path.join(r'.\data\Raw\TrainingMask',name.replace('_0000','')),os.path.join(r'.\data\Raw\TrainingMask','train_'+str(num)+'.nii.gz'))
    num=num+1
    img=sitk.ReadImage(os.path.join(r'.\data\Raw\TrainingImg',name))
    mask=sitk.ReadImage(os.path.join(r'.\data\Raw\TrainingMask',name.replace('_0000','')))
    imgarr=sitk.GetArrayFromImage(img)
    maskarr=sitk.GetArrayFromImage(mask)
    imgarr1=imgarr.copy()
    imgarr1[maskarr!=2]=0
    imgarr2=imgarr1.copy()
    imgarr2[imgarr2>50]=0
    for i in range(imgarr2.shape[0]):
        for j in range(imgarr2.shape[1]):
            for k in range(imgarr2.shape[2]):
                if imgarr2[i,j,k]!=0:
                    imgarr2[i,j,k]=imgarr2[i,j,k]+random.randint(100,150)
    imgarr1[imgarr1<=50]=0
    imgarr1=imgarr1+imgarr2
    imgarr[maskarr==2]=0
    imgarr=imgarr+imgarr1
    saveimg=sitk.GetImageFromArray(imgarr)
    saveimg.SetSpacing(img.GetSpacing())
    saveimg.SetDirection(img.GetDirection())
    saveimg.SetOrigin(img.GetOrigin())
    sitk.WriteImage(saveimg,os.path.join(r'.\data\Raw\TrainingImg','train_'+str(num)+'_0000.nii.gz'))
    sitk.WriteImage(mask,os.path.join(r'.\data\Raw\TrainingMask','train_'+str(num)+'.nii.gz'))


from modules import aws_sript, firestorage_code
import os
from flask import Flask, jsonify
from flask import render_template, request, redirect, url_for
from werkzeug.utils import secure_filename
import os, shutil
from flask_cors import CORS
from decorater_file import crossdomain

global app

app = Flask(__name__)
CORS(app)
app.config['SECRET_KEY'] = 'the quick brown fox jumps over the lazy   dog'
app.config['CORS_HEADERS'] = ['Content-Type']

cors = CORS(app, resources={r"/*": {"origins": "*"}})

# Create a directory in a known location to uploaded files to.
uploads_dir = os.path.join(app.instance_path, 'uploads')
if not os.path.exists(uploads_dir):
    os.makedirs(uploads_dir)

# Create a directory in a known location to processed files to.
ml_output_dir = os.path.join(app.instance_path, 'ml_output')
if not os.path.exists(ml_output_dir):
    os.makedirs(ml_output_dir)

@app.route('/upload', methods=['GET', 'POST'])

def upload():
    if request.method == 'POST':
        # save the single "profile" file
        image = request.files['image']
        print(image)

        img_path = os.path.join(uploads_dir, secure_filename(image.filename))

        ml_out_img_path = os.path.join(ml_output_dir, secure_filename(image.filename))
        image.save(img_path)

        print("img_path",img_path)
        print("ml_out_img_path",ml_out_img_path)
        # ML model start
        print("Processing...")
        aws_sript.convert_image(img_path, ml_out_img_path)
        # ML model ends
        print("Image converted successfully")

        img_url = firestorage_code.upload_img(ml_out_img_path)

        print("Image uploaded to firebase storage successfully")

        print("Firebase Image URL ",img_url)

        # cleaning folders
        clean_folders()

        # save each "charts" file

        # for file in request.files.getlist('upload'):
        #     print(file.name)
        #     file.save(os.path.join(uploads_dir, file.name))

        # return redirect(url_for('upload'))
        data = {
            "img_url":img_url
        }
    return jsonify(data),{'Access-Control-Allow-Origin': '*'}

def clean_folders():
        folders = [uploads_dir,ml_output_dir]
        for folder in folders:
            for filename in os.listdir(folder):
                file_path = os.path.join(folder, filename)
                try:
                    if os.path.isfile(file_path) or os.path.islink(file_path):
                        os.unlink(file_path)
                    elif os.path.isdir(file_path):
                        shutil.rmtree(file_path)
                except Exception as e:
                    print('Failed to delete %s. Reason: %s' % (file_path, e))

if __name__ == '__main__':
    app.run(debug=True)


#file used to take screenshot to baseline rectangle mappings off of
import cv2

cam = cv2.VideoCapture(0)

result, image = cam.read()

if result:

    cv2.imshow("img_to_map", image)

    cv2.imwrite("img_to_map.png", image)

    cv2.waitKey(0)
    cv2.destroyWindow("img_to_map")

else:
    print("No image detected. Please! try again")


# -*- coding: utf-8 --

import re
import math
from multiprocessing import cpu_count, freeze_support
from multiprocessing.pool import Pool
import sys

from util import read_text_lines
from util import refine_line
from char2vec import load_model

B = 1
I = 0

'''
1. word2vec 모델 불러오기(from char2vec)
'''

def is_hangul(ch):
    codepoint = ord(ch) if isinstance(ch, str) else int(ch)
    return codepoint >= 0xac00 and codepoint <= 0xd7a3

def is_ascii(ch):
    codepoint = ord(ch) if isinstance(ch, str) else int(ch)
    return codepoint >= 0x20 and codepoint <= 0x7e

def ch2num(ch):
    codepoint = ord(ch) if isinstance(ch, str) else ch
    if is_hangul(ch):
        return codepoint - ord('가') + 256
    elif is_ascii(ch):
        return codepoint
    else:
        return None

def get_features(line_ch, i):
    X = [0 for i in range(6)]

    if i > 2:
        X[0] = ch2num(line_ch[i - 2])

    if i > 1:
        X[1] = ch2num(line_ch[i - 1])

    X[2] = ch2num(line_ch[i])

    if i < len(line_ch) - 1:
        X[3] = ch2num(line_ch[i + 1])

    if i < len(line_ch) - 2:
        X[4] = ch2num(line_ch[i + 2])

    # 문장의 시작 위치 기록
    if i == 0:
        X[5] = 1
    else:
        X[5] = 0

    return X

def raw2corpus(raw_sentence):
    taggeds = []
    text = re.sub(r'(\ )+', ' ', raw_sentence).strip()
    for i in range(len(text)):
        if i == 0:
            taggeds.append('{}/B'.format(text[i]))
        elif text[i] != ' ':
            successor = text[i - 1]
            if successor == ' ':
                taggeds.append('{}/B'.format(text[i]))
            else:
                taggeds.append('{}/I'.format(text[i]))

    return ' '.join(taggeds)

def corpus2sent(line):
    sent = []
    tokens = line.split(' ')
    for token in tokens:
        if '/' not in token:
            continue

        word, tag = token.split('/')
        sent.append((word, tag))

    return sent

# 이건 사용 안함
char2vec_model = load_model(r'./char2vec_Etri_d30.txt')

ngram2vec_models = []
for n in range(1, 4):
    #ngram2vec_models.append(load_model(r'./char2vec_Etri_d30_{}gram.txt'.format(n)))
    #ngram2vec_models.append(load_model(r'./char2vec_ted_d40_{}gram.txt'.format(n)))
    ngram2vec_models.append(load_model(r'./char2vec_MDM001_d40_{}gram.txt'.format(n)))

def char2vec(ch):
    n = len(ch)
    return [float(f) for f in ngram2vec_models[n-1][ch]]

# 조화 평균
def hmean(values):
    top = float(len(values))
    bottom = 0.0
    for v in values:
        top *= v
        bottom += v

    return top / bottom

# 산술 평균
def amean(values):
    s = 0.0
    for v in values:
        s += v

    return v / len(values)

# 기하 평균
def gmean(values):
    m = 1
    for v in values:
        m *= v

    r = m ** (1.0/float(len(values)))
    return r

def index2feature(line, i, offsets):
    '''
    해당 offset에 위치한 글자의 word embedding 벡터를 가져온다.
    offset이 여러 개 있으면 중간값이나 평균(산술, 조화 등)값으로 합쳐서 실험해보기
    * 중간값: 다른 조합인데 같은 걸로 취급될 수 있는 경우가 있으므로 빼기
    * 실험1 --> 산술평균: (a + b + c) / 3
    * 실험2 --> 조화평균: 3*a*b*c / (a + b + c) --> 모두 양수일때만 의미있는 결과 나옴
    * 실험3 --> 기하평균: sqrt3(a * b * c)

    ※ 기하평균 --> 곱해야 하는 값의 평균 구할때 사용(예: 은행 n년간 평균 이자 계산 등)
    '''
    vec = []
    for off in offsets:
        if i + off < 0 or i + off >= len(line):
            return [0.0 for i in range(50)]

        ch, _ = line[i + off]
        vec.append(char2vec_model[ch])

    result = []
    for i in range(len(vec[0])):
        v = []
        for j in range(len(vec)):
            v.append(float(vec[j][i]))

        result.append(amean(v))

    return result

# 다른 논문 참고한 자질에서 2개이상 글자에 해당하는 임베딩은 각 글자의
# 임베딩 정보를 평균낸걸로 만든 자질
def generate_feature(args):
    line = args[0]
    i = args[1]

    feature = []

    feature += index2feature(line, i, [-1])
    feature += index2feature(line, i, [0])
    feature += index2feature(line, i, [1])
    feature += index2feature(line, i, [-2, -1])
    feature += index2feature(line, i, [-1, 0])
    feature += index2feature(line, i, [0, 1])
    feature += index2feature(line, i, [-2, -1, 0])
    feature += index2feature(line, i, [-1, 0, 1])
    feature += index2feature(line, i, [0, 1, 2])

    return feature

# 앞 2글자부터 뒤2글자까지 각 한글자씩의 임베딩 정보를 자질로 사용한 것
def generate_feature2(args):
    line = args[0]
    i = args[1]

    feature = []

    if i >= 2:
        ch, _ = line[i - 2]
        feature += char2vec(ch)
    else:
        feature += [0.0 for i in range(30)]

    if i >= 1:
        ch, _ = line[i - 1]
        feature += char2vec(ch)
    else:
        feature += [0.0 for i in range(30)]

    ch, _ = line[i]
    feature += char2vec(ch)

    if i < len(line) - 1:
        ch, _ = line[i + 1]
        feature += char2vec(ch)
    else:
        feature += [0.0 for i in range(30)]

    if i < len(line) - 2:
        ch, _ = line[i + 2]
        feature += char2vec(ch)
    else:
        feature += [0.0 for i in range(30)]

    return feature

# 타 논문 참고한 자질에서 여러 글자에 해당하는 임베딩 정보를 추가한 자질
def generate_feature3(args):
    line = ''.join([l[0] for l in args[0]])
    i = args[1]
    dim = 40

    feature = []

    # 1-gram
    feature += char2vec(line[i-1]) if i >= 1 else [0.0 for a in range(dim)]
    feature += char2vec(line[i])
    feature += char2vec(line[i+1]) if i < len(line)-1 else [0.0 for a in range(dim)]

    # 2-gram
    feature += char2vec(line[i-2:i]) if i >= 2 else [0.0 for a in range(dim)]
    feature += char2vec(line[i-1:i+1]) if i >= 1 else [0.0 for a in range(dim)]
    feature += char2vec(line[i:i+2]) if i < len(line)-1 else [0.0 for a in range(dim)]

    # 3-gram
    feature += char2vec(line[i-2:i+1]) if i >= 2 else [0.0 for a in range(dim)]
    feature += char2vec(line[i-1:i+2]) if i >= 1 and i < len(line)-1 else [0.0 for a in range(dim)]
    feature += char2vec(line[i:i+3]) if i < len(line)-2 else [0.0 for a in range(dim)]

    return feature

def generate_feature4(args):
    line = ''.join([l[0] for l in args[0]])
    i = args[1]
    dim = 40

    feature = []

    # 1-gram
    feature += char2vec(line[i])
    feature += char2vec(line[i-1]) if i >= 1 else [0.0 for a in range(dim)]
    feature += char2vec(line[i+1]) if i < len(line)-1 else [0.0 for a in range(dim)]

    # 2-gram
    feature += char2vec(line[i-2:i]) if i >= 2 else [0.0 for a in range(dim)]
    feature += char2vec(line[i-1:i+1]) if i >= 1 else [0.0 for a in range(dim)]
    feature += char2vec(line[i:i+2]) if i < len(line)-1 else [0.0 for a in range(dim)]
    feature += char2vec(line[i+1:i+3]) if i < len(line)-2 else [0.0 for a in range(dim)]

    # 3-gram
    feature += char2vec(line[i-2:i+1]) if i >= 2 else [0.0 for a in range(dim)]
    feature += char2vec(line[i-1:i+2]) if i >= 1 and i < len(line)-1 else [0.0 for a in range(dim)]
    feature += char2vec(line[i:i+3]) if i < len(line)-2 else [0.0 for a in range(dim)]
    feature += char2vec(line[i+1:i+4]) if i < len(line)-3 else [0.0 for a in range(dim)]

    return feature

def make_data(pool, fname):
    lines = read_text_lines(fname)
    lines = (refine_line(line) for line in lines)
    corpus = (raw2corpus(line) for line in lines)
    sent = (corpus2sent(line) for line in corpus)

    X = []
    Y = []

    for line in sent:
        X += pool.map(generate_feature4, [(line, i) for i in range(len(line))])
        Y += [(1 if y == 'B' else 0) for _, y in line]

    return X, Y

def make_data_divided(pool, fname):
    lines = read_text_lines(fname)
    lines = (refine_line(line) for line in lines)
    corpus = (raw2corpus(line) for line in lines)
    sent = (corpus2sent(line) for line in corpus)

    line_cnt = 0
    X = []
    Y = []
    for line in sent:
        line_cnt += 1
        x = pool.map(generate_feature4, [(line, i) for i in range(len(line))])
        X += norm_many(pool, x)
        Y += ((1 if y == 'B' else 0) for _, y in line)
        if line_cnt == 100000:
            yield X, Y
            line_cnt = 0
            X = []
            Y = []

    yield X, Y

# todo: 여러 글자의 워드벡터를 더한 것을 고려해서 수정하기
def norm(arr):
    return [round(x*1000, 0) + 10000 for x in arr]

def norm_many(pool, X):
    return list(pool.map(norm, X))

def main():
    for i in range(1, 4):
        char2vec_model = load_model(r'./char2vec_ted_d40_{}gram.txt'.format(i))

        min_v = 0.0
        max_v = 0.0
        for k in char2vec_model.wv.vocab.keys():
            vec = char2vec_model.wv[k]
            for v in vec:
                if v < min_v:
                    min_v = v
                elif v > max_v:
                    max_v = v

        print('#{}: min={}, max={}'.format(i, min_v, max_v))
    #sys.exit(1)

    pool = Pool(processes=cpu_count())
    X, Y = make_data(pool, r'./ted_7_ErasePunc_FullKorean__train.txt')
    print(X[:5])
    print(Y[:5])

if __name__ == '__main__':
    freeze_support()
    main()


from django.core.management.base import BaseCommand

import os
from importlib import import_module
from django.conf import settings
from django.core.management import call_command
from newapp.utils import get_app_template_path, get_app_templates

APP_TEMPLATES = [ x.get('name') for x in get_app_templates() ]

class Command(BaseCommand):
    """
    Example usage:
    python manage.py newapp mambu --template=lite --appsdir=apps
    """
    help = __doc__
    args = '<function arg arg ...>'

    def check_name_conflick(self, name):
        apps_list = next(os.walk(os.path.join(settings.BASE_DIR, "apps")))[1]
        apps_list = apps_list +['admin', 'admindocs', 'auth' ,'contenttypes' ,'flatpages','gis','humanize',
                            'messages','postgres','redirects','sessions','sitemaps','sites','staticfiles'
                            'syndication']
        if name in apps_list:
            return True
        else:
            try:
                import_module(name)
            except ImportError:
                return False
            else:
                return True

    def add_arguments(self, parser):
        parser.add_argument('name', type=str)
        parser.add_argument('--apptype', '-t', dest='apptype', default='lite',
                            help='Application type')
        parser.add_argument('--appdir', '-d', type=str, dest='appdir',  default='/',
                            help='Target directory')

    def handle(self, *args, **options):

        name = options['name']
        if self.check_name_conflick(name):
            self.stdout.write(self.style.ERROR("Sorry, but you can't use %s as name because this name already taken" % name))
            exit()

        apps_type = options['apptype']
        if apps_type not in APP_TEMPLATES:
            self.stdout.write(self.style.ERROR("no template with name %s" % apps_type))
            exit()

        if options['appdir'] == "/":
            app_dir = settings.BASE_DIR
            app_path = os.path.join(settings.BASE_DIR, name)
        else:
            app_dir = options['appdir'].strip("/")
            app_path = os.path.join(settings.BASE_DIR, "%s/%s" % (app_dir, name))

        if os.path.isdir(app_dir):
            os.mkdir(app_path)
        else:
            self.stdout.write(self.style.ERROR("Appdir %s not found" % app_dir))
            exit()

        template_path = get_app_template_path(apps_type)
        call_command("startapp", name, app_path, template=template_path)
        os.unlink(os.path.join(app_path, "desc.txt"))
        self.stdout.write(self.style.SUCCESS("Congratulation apps %s successfuly created" % name))