from pytorch_transformers.modeling_bert import BertConfig
from pytorch_transformers import optimization

import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

from datasets import Datasets
from model import BertForEmotionClassification
from optim import layerwise_decay_optimizer

from sklearn.metrics import classification_report, confusion_matrix
from tqdm import tqdm
import time
import pickle
import utils
import logging
import argparse
import random
import os
import warnings

warnings.filterwarnings('ignore')
logger = utils.get_logger('BERT Classification')
logger.setLevel(logging.INFO)

class ClassificationBatchFunction:
    # batch function for pytorch dataloader
    def __init__(self, max_len, pad_idx, cls_idx, sep_idx):
        self.max_len = max_len
        self.pad_idx = pad_idx
        self.cls_idx = cls_idx
        self.sep_idx = sep_idx

    def __call__(self, batch):
        tokens, label = list(zip(*batch))

        # Get max length from batch
        max_len = min(self.max_len, max([len(i) for i in tokens]))
        tokens = torch.tensor([self.pad([self.cls_idx] + t + [self.sep_idx], max_len) for t in tokens])
        masks = torch.ones_like(tokens).masked_fill(tokens == self.pad_idx, 0)

        return tokens, masks, torch.tensor(label)

    def pad(self, sample, max_len):
        diff = max_len - len(sample)
        if diff > 0:
            sample += [self.pad_idx] * diff
        else:
            sample = sample[-max_len:]
        return sample

def train(args):
    set_seed(args)
    args.train_batch_size = args.train_batch_size*args.n_gpu
    args.eval_batch_size = args.eval_batch_size*args.n_gpu

    # Set device
    if args.device == 'cuda':
        device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
        logger.info('use cuda')
    else:
        device = torch.device('cpu')
        logger.info('use cpu')

    # Set label list for classification
    if args.num_label == 'multi':
        label_list = ['공포', '놀람', '분노', '슬픔', '중립', '행복', '혐오']
    elif args.num_label == 'binary':
        label_list = ['긍정', '부정']
    logger.info('use {} labels for training'.format(len(label_list)))

    # Load pretrained model and model configuration
    pretrained_path = os.path.join('./pretrained_model/', args.pretrained_type)
    if args.pretrained_model_path is None:
        # Use pretrained bert model(etri/skt)
        pretrained_model_path = os.path.join(pretrained_path, 'pytorch_model.bin')
    else:
        # Use further-pretrained bert model
        pretrained_model_path = args.pretrained_model_path
    logger.info('Pretrain Model : {}'.format(pretrained_model_path))
    pretrained = torch.load(pretrained_model_path)

    # weight
    if args.pretrained_type == 'skt' and 'bert.' not in list(pretrained.keys())[0]:
        logger.info('modify parameter names')
        # Change parameter name for consistency
        new_keys_ = ['bert.' + k for k in pretrained.keys()]
        old_values_ = pretrained.values()
        pretrained = {k: v for k, v in zip(new_keys_, old_values_)}

    # bulid model
    bert_config = BertConfig(os.path.join(pretrained_path + '/bert_config.json'))
    bert_config.num_labels = len(label_list)
    model = BertForEmotionClassification(bert_config).to(device)

    # assigning weight
    model.load_state_dict(pretrained, strict=False)

    # Load Datasets
    tr_set = Datasets(file_path=args.train_data_path,
                      label_list=label_list,
                      pretrained_type=args.pretrained_type,
                      max_len=args.max_len)
    dev_set = Datasets(file_path=args.dev_data_path,
                       label_list=label_list,
                       pretrained_type=args.pretrained_type,
                       max_len=args.max_len)

    # Use custom batch function
    collate_fn = ClassificationBatchFunction(args.max_len, tr_set.pad_idx, tr_set.cls_idx, tr_set.sep_idx)

    tr_loader = DataLoader(dataset=tr_set,
                           batch_size=args.train_batch_size,
                           shuffle=True,
                           num_workers=8,
                           pin_memory=True,
                           collate_fn=collate_fn)

    dev_loader = DataLoader(dataset=dev_set,
                            batch_size=args.eval_batch_size,
                            num_workers=8,
                            pin_memory=True,
                            drop_last=False,
                            collate_fn=collate_fn)

    # optimizer
    optimizer = layerwise_decay_optimizer(model=model, lr=args.learning_rate, layerwise_decay=args.layerwise_decay)

    # lr scheduler
    t_total = len(tr_loader) // args.gradient_accumulation_steps * args.epochs
    warmup_steps = int(t_total * args.warmup_percent)
    logger.info('total training steps : {}, lr warmup steps : {}'.format(t_total, warmup_steps))
    # Use gradual warmup and linear decay
    scheduler = optimization.WarmupLinearSchedule(optimizer, warmup_steps=warmup_steps, t_total=t_total)

    # for low-precision training
    if args.fp16:
        try:
            from apex import amp
            logger.info('Use fp16')
        except ImportError:
            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level, verbosity=0)

    # tensorboard setting
    save_path = "./model_saved_finetuning/lr{},batch{},total{},warmup{},len{},{}".format(
        args.learning_rate, args.train_batch_size * args.gradient_accumulation_steps, t_total,
        args.warmup_percent, args.max_len, args.pretrained_type)

    if not os.path.isdir(save_path):
        os.makedirs(save_path)
    writer = SummaryWriter(save_path)

    # Save best model results with resultwriter
    result_writer = utils.ResultWriter("./model_saved_finetuning/results.csv")
    model.zero_grad()

    best_val_loss = 1e+9
    global_step = 0

    train_loss, train_acc, train_f1 = 0, 0, 0
    logging_loss, logging_acc, logging_f1 = 0, 0, 0

    logger.info('***** Training starts *****')
    total_result = []
    for epoch in tqdm(range(args.epochs), desc='epochs'):
        for step, batch in tqdm(enumerate(tr_loader), desc='steps', total=len(tr_loader)):
            model.train()
            x_train, mask_train, y_train = map(lambda x: x.to(device), batch)

            inputs = {
                'input_ids': x_train,
                'attention_mask': mask_train,
                'classification_label': y_train,
            }

            output, loss = model(**inputs)
            y_max = output.max(dim=1)[1]

            cr = classification_report(y_train.tolist(),
                                       y_max.tolist(),
                                       labels=list(range(len(label_list))),
                                       target_names=label_list,
                                       output_dict=True)
            # Get accuracy(micro f1)
            if 'micro avg' not in cr.keys():
                batch_acc = list(cr.items())[len(label_list)][1]
            else:
                # If at least one of labels does not exists in mini-batch, use micro average instead
                batch_acc = cr['micro avg']['f1-score']
            # macro f1
            batch_macro_f1 = cr['macro avg']['f1-score']

            # accumulate measures
            grad_accu = args.gradient_accumulation_steps
            if grad_accu > 1:
                loss /= grad_accu
                batch_acc /= grad_accu
                batch_macro_f1 /= grad_accu

            if args.fp16:
                with amp.scale_loss(loss, optimizer) as scaled_loss:
                    scaled_loss.backward()
            else:
                if args.n_gpu > 1:
                    loss = loss.mean()
                    loss.backward()
                else:
                    loss.backward()

            train_loss += loss.item()
            train_acc += batch_acc
            train_f1 += batch_macro_f1

            if (global_step + 1) % grad_accu == 0:
                if args.fp16:
                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.grad_clip_norm)
                else:
                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip_norm)

                optimizer.step()
                scheduler.step()
                model.zero_grad()
                global_step += 1

                if global_step % args.logging_step == 0:
                    acc_ = (train_acc - logging_acc) / args.logging_step
                    f1_ = (train_f1 - logging_f1) / args.logging_step
                    loss_ = (train_loss - logging_loss) / args.logging_step
                    writer.add_scalars('loss', {'train': loss_}, global_step)
                    writer.add_scalars('acc', {'train': acc_}, global_step)
                    writer.add_scalars('macro_f1', {'train': f1_}, global_step)

                    logger.info('[{}/{}], trn loss : {:.3f}, trn acc : {:.3f}, macro f1 : {:.3f}'.format(
                        global_step, t_total, loss_, acc_, f1_
                    ))
                    logging_acc, logging_f1, logging_loss = train_acc, train_f1, train_loss

                    # Get f1 score for each label
                    f1_results = [(l, r['f1-score']) for i, (l, r) in enumerate(cr.items()) if i < len(label_list)]
                    f1_log = "\n".join(["{} : {}".format(l, f) for l, f in f1_results])
                    logger.info("\n\n***f1-score***\n" + f1_log + "\n\n***confusion matrix***\n{}".format(
                        confusion_matrix(y_train.tolist(), y_max.tolist())))

        # Validation
        val_loss, val_acc, val_macro_f1, _ = evaluate(args, dev_loader, model, device)
        val_result = '[{}/{}] val loss : {:.3f}, val acc : {:.3f}. val macro f1 : {:.3f}'.format(
            global_step, t_total, val_loss, val_acc, val_macro_f1
        )

        writer.add_scalars('loss', {'val': val_loss}, global_step)
        writer.add_scalars('acc', {'val': val_acc}, global_step)
        writer.add_scalars('macro_f1', {'val': val_macro_f1}, global_step)
        logger.info(val_result)
        total_result.append(val_result)

        if val_loss < best_val_loss:
            # Save model checkpoints
            torch.save(model.state_dict(), os.path.join(save_path, 'best_model.bin'))
            torch.save(args, os.path.join(save_path, 'training_args.bin'))
            logger.info('Saving model checkpoint to %s', save_path)
            best_val_loss = val_loss
            best_val_acc = val_acc
            best_val_macro_f1 = val_macro_f1

    # Save results in 'model_saved_finetuning/results.csv'
    results = {
        'val_loss': best_val_loss,
        'val_acc': best_val_acc,
        'val_macro_f1' : best_val_macro_f1,
        'save_dir': save_path,
        'pretrained_path': pretrained_path,
    }
    result_writer.update(args, **results)
    return global_step, loss_, acc_, best_val_loss, best_val_acc, total_result

def evaluate(args, dataloader, model, device, objective='classification'):

    if args.num_label == 'multi':
        label_list = ['공포', '놀람', '분노', '슬픔', '중립', '행복', '혐오']
    elif args.num_label == 'binary':
        label_list = ['긍정', '부정']

    val_loss, val_acc, val_f1 = 0, 0, 0
    total_y = []
    total_y_hat = []

    for val_step, batch in enumerate(dataloader):
        model.eval()

        x_dev, mask_dev, y_dev = map(lambda x: x.to(device), batch)
        total_y += y_dev.tolist()

        inputs = {
            'input_ids': x_dev,
            'attention_mask': mask_dev,
            'classification_label': y_dev,
        }
        with torch.no_grad():
            output, loss = model(**inputs)
            if args.n_gpu > 1:
                loss = loss.mean()
            y_max = output.max(dim=1)[1]
            total_y_hat += y_max.tolist()
            val_loss += loss.item()

    # f1-score 계산
    dev_cr = classification_report(total_y,
                                   total_y_hat,
                                   labels=list(range(len(label_list))),
                                   target_names=label_list,
                                   output_dict=True)

    # Get accuracy(micro f1)
    if 'micro avg' not in dev_cr.keys():
        val_acc = list(dev_cr.items())[len(label_list)][1]
    else:
        # If at least one of labels does not exists in mini-batch, use micro average instead
        val_acc = dev_cr['micro avg']['f1-score']
    # macro f1
    val_macro_f1 = dev_cr['macro avg']['f1-score']

    logger.info('***** Evaluation Results *****')
    f1_results = [(l, r['f1-score']) for i, (l, r) in enumerate(dev_cr.items()) if i < len(label_list)]
    f1_log = "\n".join(["{} : {}".format(l, f) for l, f in f1_results])
    cm = confusion_matrix(total_y, total_y_hat)
    logger.info("\n***f1-score***\n" + f1_log + "\n***confusion matrix***\n{}".format(cm))

    val_loss /= (val_step + 1)
    return val_loss, val_acc, val_macro_f1, (total_y_hat, cm)

def set_seed(args):
    random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed_all(args.seed)

def main():
    parser = argparse.ArgumentParser()

    # Pretrained model Parameters
    parser.add_argument("--pretrained_type", default='etri', type=str,
                        help="type of pretrained model (skt, etri)")
    parser.add_argument("--pretrained_model_path", required=False,
                        help="path of pretrained model (If you wnat to use further-pretrained model)")

    # Train Parameters
    parser.add_argument("--train_batch_size", default=100, type=int,
                        help="batch size")
    parser.add_argument("--eval_batch_size", default=100, type=int,
                        help="batch size for validation")
    parser.add_argument("--layerwise_decay", action="store_true",
                        help="Whether to use layerwise decay")
    parser.add_argument("--learning_rate", default=2e-5, type=float,
                        help="The initial learning rate for Adam")
    parser.add_argument("--epochs", default=50, type=int,
                        help="total epochs")
    parser.add_argument("--gradient_accumulation_steps", default=1, type=int,
                        help="gradient accumulation steps for large batch training")
    parser.add_argument("--warmup_percent", default=0.1, type=float,
                        help="gradient warmup percentage")
    parser.add_argument("--grad_clip_norm", default=1.0, type=float,
                        help="batch size")

    # Other Parameters
    parser.add_argument("--logging_step", default=2, type=int,
                        help="logging step for training loss and acc")
    parser.add_argument("--device", default='cuda', type=str,
                        help="Whether to use cpu or cuda")
    parser.add_argument("--fp16", action="store_true",
                        help="Whether to use nvidia mixed precision training")
    parser.add_argument('--fp16_opt_level', type=str, default='O1',
                        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
                        "See details at https://nvidia.github.io/apex/amp.html")
    parser.add_argument("--seed", default=0, type=int,
                        help="Random seed(default=0)")
    parser.add_argument("--n_gpu", default=1, type=int,
                        help="n_gpu")
    # Data Parameters
    parser.add_argument("--train_data_path", default='../split_datasets_vat/train.pkl', type=str,
                        help="train data path")
    parser.add_argument("--dev_data_path", default='../split_datasets_vat/dev.pkl', type=str,
                        help="dev data path")
    parser.add_argument("--num_label", default='multi', type=str,
                        help="Number of labels in datastes(binary or multi)")
    parser.add_argument("--max_len", default=64, type=int,
                        help="Maximum sequence length")

    args = parser.parse_args()
    set_seed(args)

    t = time.time()
    global_step, train_loss, train_acc, best_val_loss, best_val_acc, total_result = train(args)
    elapsed = time.time() - t

    logger.info('***** Training done *****')
    logger.info('total validation results : \n'+'\n'.join(total_result))
    logger.info('elapsed time: %.3f Hours' % (elapsed / 3600.))
    logger.info('best acc in test: %.4f' % best_val_acc)
    logger.info('best loss in test: %.4f' % best_val_loss)

if __name__ == '__main__':
    main()


import pydantic
from box import ConfigBox
import os
from pathlib import Path

from mlengine.common.logger import logger
from mlengine.common.utils import create_directories
from mlengine.common.exceptions import MissingCriticalFileException
from mlengine.data_read.read import read_csv_file

class StudentDataTypesValidator(pydantic.BaseModel):
    gender: pydantic.StrictStr
    race_ethnicity: pydantic.StrictStr
    parental_level_of_education: pydantic.StrictStr
    lunch: pydantic.StrictStr
    test_preparation_course: pydantic.StrictStr
    math_score: pydantic.StrictInt
    reading_score: pydantic.StrictInt
    writing_score: pydantic.StrictInt

class StudentTransformedDataTypesValidator(pydantic.BaseModel):
    gender: pydantic.StrictStr
    race_ethnicity: pydantic.StrictStr
    parental_level_of_education: pydantic.StrictStr
    lunch: pydantic.StrictStr
    test_preparation_course: pydantic.StrictStr
    math_score: pydantic.StrictInt
    reading_score: pydantic.StrictInt
    writing_score: pydantic.StrictInt
    total_score: pydantic.StrictInt
    average: pydantic.StrictFloat

class StudentDataValidator:
    def __init__(self, config: ConfigBox):
        self.config: ConfigBox = config
        self.data_file: Path = self.config.req_files[0]

    def validate_data(self):
        try:
            df = read_csv_file(self.data_file)
            data_list = [StudentDataTypesValidator(**row) for _, row in df.iterrows()]
            logger.info(f'Successful validation of data file {self.data_file} via Pydantic strict types')
        except Exception as e:
            raise e

class StudentTransformedDataValidator:
    def __init__(self, config: ConfigBox):
        self.config: ConfigBox = config
        self.data_file: Path = self.config.req_files[0]

    def validate_data(self):
        try:
            df = read_csv_file(self.data_file)
            data_list = [StudentDataTypesValidator(**row) for _, row in df.iterrows()]
            logger.info(f'Successful validation of data file {self.data_file} via Pydantic strict types')
        except Exception as e:
            raise e

class FileValidator:
    def __init__(self, config: ConfigBox):
        self.config: ConfigBox = config
        self._create_dirs()

    def _create_dirs(self):
        create_directories([self.config.root_dir])
        if hasattr(self.config, 'models_dir'):
            create_directories([self.config.models_dir])

    def validate_all_files_exist(self):
        try:
            statuses = []
            messages = []

            for file in self.config.req_files:
                validation_status = os.path.isfile(file)
                statuses.append(validation_status)

                msg = f"Validation status: {validation_status} for file: {file}"
                messages.append(msg)

                logger.info(msg)

            status = all(statuses)

            with open(os.path.join(self.config.root_dir, self.config.status_file), 'w') as f:
                for msg in messages:
                    f.write(msg + "\n")
                f.write(f"Overall status: {status}")

            if not status:
                raise MissingCriticalFileException('VLD_EX_001', 'Validation failed for critical file(s). Check the logged status file for more information.')

        except Exception as e:
            raise e


from django.db import models
from django.contrib import admin
try:
    import json
except ImportError:
    import simplejson as json

class Card(models.Model):
    card_id = models.CharField(max_length=16)

    def __unicode__(self):
        return self.card_id

    @classmethod
    def load_json(cls, json_data):
        json_obj = json.loads(json_data)
        card_id = json_obj['card_id']
        cards = Card.objects.filter(card_id=card_id)
        if cards:
            card = cards[0]
        else:
            card = Card()

        for key, value in json_obj.items():
            setattr(card, key, value)

        return card

admin.site.register(Card)


# -*- coding: utf-8 -*-

from odoo import models, fields, api, _
from odoo.exceptions import AccessError, UserError, ValidationError
from odoo.tools.translate import html_translate

import logging

_logger = logging.getLogger(__name__)

class PurchaseOrder(models.Model):
    _name = 'purchase.order'
    _inherit = ['purchase.order', 'x.x_company_organization.org_mixin']

    @api.depends('trans_classify_id')
    def _compute_show_construction(self):
        for rec in self:
            rec.show_construction = True if self.trans_classify_id and self.trans_classify_id.code == 'construction' else False

    @api.depends('order_line', 'order_line.product_id')
    def _compute_is_dropshipping(self):
        for record in self:
            record.is_dropshipping = False
            if record.order_line:
                route_id = self.env.ref('stock_dropshipping.route_drop_shipping', raise_if_not_found=False)
                record.is_dropshipping = True if route_id in record.mapped('product_id').mapped('route_ids') else False

    trans_classify_id = fields.Many2one('x.transaction.classification', "Transaction classification")
    type = fields.Selection([
        ('normal', 'Normal purchase'),
        ('tank_lorry', 'Truck delivery'),
    ], string='Purchase type', copy=False, default='normal')
    desired_delivery = fields.Selection([
        ('full', 'Full payment request'),
        ('separated', 'Can be paid in installments'),
    ], string="Desired delivery", copy=False, default='full')
    date_response = fields.Datetime("Response date", copy=False)
    date_issuance = fields.Date("Issuance date", copy=False, default=fields.Date.context_today)
    jurisdiction_id = fields.Many2one('crm.team', "Jurisdiction")
    dest_address_infor = fields.Html("Direct shipping information", copy=False)
    # Displayed only when the transaction classification item of the slip is "Construction"
    show_construction = fields.Boolean("Show Construction", compute='_compute_show_construction')
    construction_name = fields.Char("Construction name")
    construction_site = fields.Char("Construction site")
    construction_period_start = fields.Date('Scheduled construction period', copy=False)
    construction_period_end = fields.Date('Scheduled construction period (end)', copy=False)
    pres_abs_supplies = fields.Selection([
        ('none', 'Nothing'),
        ('having', 'Yes'),
    ], string="Presence/absence of supplies", copy=False, default='none')
    construction_supplies = fields.Char("Supplies")
    construction_payment_term = fields.Char("Payment terms", readonly=True, default=_("Dealine at the end of the month, payment at the end of the following month according to our regulations"))
    construction_cash = fields.Float("Cash")
    construction_bills = fields.Float("Bills")
    pres_abs_current_theory = fields.Selection([
        ('none', 'No'),
        ('explanation', 'Yes'),
    ], string='Presence/absence of the current theory', copy=False, default='none')
    construction_current_date = fields.Date('Date of the current theory', copy=False)
    construction_current_place = fields.Char('Current theory place', copy=False)
    construction_others = fields.Text('others', copy=False)
    notes_construction_contract = fields.Html("Notes on construction contract", related="company_id.notes_construction_contract")
    estimated_subcontracting_work = fields.Html("Estimated price and estimated period for subcontracting work", related="company_id.estimated_subcontracting_work")

    # Rename fields standard
    name = fields.Char(string="Slip No.")
    origin = fields.Char(string="Reference source")
    partner_id = fields.Many2one('res.partner', string="Supplier")
    user_id = fields.Many2one('res.users', string="Purchasing person")
    create_date = fields.Datetime(string="Create date", default=fields.Datetime.now, copy=False)
    create_uid = fields.Many2one('res.users', string='Slip creator',)
    picking_type_id = fields.Many2one('stock.picking.type', string='Delivery destination',)
    dest_address_id = fields.Many2one('res.partner', string='Direct delivery',)
    fiscal_position_id = fields.Many2one('account.fiscal.position', string='Accounting position',)
    date_planned = fields.Datetime(string="Requested delivery date", copy=False)
    is_dropshipping = fields.Boolean('Is dropship', compute='_compute_is_dropshipping',)

    # TODO: Hide print with state
    @api.model
    def fields_view_get(self, view_id=None, view_type='form', toolbar=False, submenu=False):
        res = super(PurchaseOrder, self).fields_view_get(view_id=view_id, view_type=view_type, toolbar=toolbar,
                                                         submenu=submenu)
        _logger.info('111111111111111 fields_view_get toolbar print: %s', res.get('toolbar', {}).get('print', []))
        return res

    @api.onchange('partner_id')
    def _onchange_partner_id_sgvn(self):
        self.trans_classify_id = self.partner_id.x_transaction_classification and self.partner_id.x_transaction_classification[0].id
        x_organization_id = self.x_organization_id and self.x_organization_id.id
        if self.partner_id and not (self.partner_id.x_organization_id and self.partner_id.x_organization_id.id == x_organization_id):
            return {
                'warning': {
                    'title': _("Alert: Insufficient supplier transaction information"),
                    'message': _("""The purchasing information for [Login User's Organization Name] does not exist for the selected supplier.\nPlease complete the purchase information input / approval of the relevant supplier from the supplier application screen.""")
                },
            }

    @api.onchange('x_organization_id')
    def _onchange_x_organization_id(self):
        x_organization_id = self.x_organization_id and self.x_organization_id.id
        if self.partner_id and not (self.partner_id.x_organization_id and self.partner_id.x_organization_id.id == x_organization_id):
            return {
                'warning': {
                    'title': _("Alert: Insufficient supplier transaction information"),
                    'message': _("""The purchasing information for [Login User's Organization Name] does not exist for the selected supplier.\nPlease complete the purchase information input / approval of the relevant supplier from the supplier application screen.""")
                },
            }

    @api.onchange('construction_cash')
    def _onchange_construction_cash(self):
        if self.construction_cash and self.clamp(self.construction_cash):
            self.construction_bills = 100 - self.construction_cash

    @api.onchange('construction_bills')
    def _onchange_construction_bills(self):
        if self.construction_bills and self.clamp(self.construction_bills):
            self.construction_cash = 100 - self.construction_bills

    @api.constrains("construction_cash", "construction_bills")
    def _check_sum_construction_payment(self):
        for record in self:
            cash = record.construction_cash
            bills = record.construction_bills
            if cash or bills:
                if sum([cash, bills]) != 100 or not(self.clamp(cash) or self.clamp(cash)):
                    raise ValidationError(_('Total payment must be 100%%: Cash %s%% - Bills %s%%' % (cash, bills)))

    # Update file attachment for mail template with trans_classify_id in <![CDATA[]]>
    def action_rfq_send(self):
        res = super(PurchaseOrder, self).action_rfq_send()
        if self.env.context.get('send_rfq', False):
            if self.show_construction:
                res['context'].update({
                    'default_template_id': self.env.ref("x_purchase_quotation.email_template_edi_purchase_construction").id
                })
            else:
                res['context'].update({
                    'default_template_id': self.env.ref("x_purchase_quotation.email_template_edi_purchase").id
                })
        res['context'].update({
            'partner_email_field': 'email_quote_request',
        })
        return res

    def print_quotation(self):
        res = super(PurchaseOrder, self).print_quotation()
        if self.show_construction:
            return self.env.ref('x_purchase_quotation.action_report_estimate_request').report_action(self)
        else:
            return self.env.ref('x_purchase_quotation.action_report_purchasequotation').report_action(self)
        return res

    @api.model
    def clamp(self, number):
        if number:
            if number < 0:
                return False
            elif number > 100:
                return False
            else:
                return number

class PurchaseOrderLine(models.Model):
    _inherit = 'purchase.order.line'

    location_dest_id = fields.Many2one('stock.location', "Delivery destination", domain=[("usage", "in", ["internal", "transit"])])
    # Rename fields standard
    product_uom_id = fields.Many2one('uom.uom', string='Unit',)
    taxes_id = fields.Many2many('account.tax', string='Tax',)

    @api.model
    def default_get(self, fields):
        rec = super(PurchaseOrderLine, self).default_get(fields)
        params = self._context.get('params')
        if params and params.get('id'):
            order_id = self.env['purchase.order'].browse(params.get('id'))
            rec['location_dest_id'] = order_id.x_organization_id.x_purchase_stock_location_id.id if order_id.x_organization_id and order_id.x_organization_id.x_purchase_stock_location_id else False
        else:
            x_organization_id = self.env.user.x_organization_id
            rec['location_dest_id'] = x_organization_id.x_purchase_stock_location_id.id if x_organization_id and x_organization_id.x_purchase_stock_location_id else False
        return rec

    # @api.onchange('order_id', 'order_id.x_organization_id')
    # def _onchange_partner_id_sgvn(self):
    #     self.location_dest_id = self.order_id.x_organization_id.x_purchase_stock_location_id.id if self.order_id.x_organization_id.x_purchase_stock_location_id else False


#!/usr/bin/env python3
# _*_ coding: utf-8 _*_

"""
collect_tweets_by_search.py

트위터 검색 API(https://dev.twitter.com/rest/public/search)를 이용하여 트윗을 수집한다
"""

from twython import Twython
import ujson

CONSUMER_KEY = "YOUR_CONSUMER_KEY"
CONSUMER_SECRET = "YOUR_CONSUMER_SECRET"
ACCESS_TOKEN = "YOUR_ACCESS_TOKEN"
ACCESS_TOKEN_SECRET = "YOUR_ACCESS_TOKEN_SECRET"

twitter = Twython(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)

result = twitter.search(q=u"파이썬")

for status in result["statuses"]:
    print(ujson.dumps(status, ensure_ascii=False))


#!/usr/bin/env python3

# === IMPORTS ===
# Utils
import os
import json

# === FUNCTIONS ===
def obtain_json_data():
    file_directory = os.path.dirname(os.path.abspath(__file__))
    json_path = os.path.join(file_directory, "posts.json")
    with open(json_path, "r") as json_file:
        data = json.load(json_file)
    return data

# === CLASSES ===
class Post:
    def __init__(
        self,
        type,
        id,
        url,
        user,
        totalLikes,
        isBlocked,
        isActive,
        totalViews,
        location,
        date,
    ):
        self.type = type
        self.oid = id
        self.url = url
        self.user = user
        self.totalLikes = totalLikes
        self.isBlocked = isBlocked
        self.isActive = isActive
        self.totalViews = totalViews
        self.location = location
        self.date = date

    def __str__(self):
        return f"{self.oid}: {self.user}"

    def to_dict_custom(self, field):
        return {"id": self.oid, "total": getattr(self, field)}

class Posts:
    def __init__(self):
        data = obtain_json_data()
        self.posts = [
            Post(
                type=item.get("type"),
                id=item.get("id"),
                url=item.get("url"),
                user=item.get("user"),
                totalLikes=item.get("totalLikes"),
                isBlocked=item.get("isBlocked"),
                isActive=item.get("isActive"),
                totalViews=item.get("totalViews"),
                location=item.get("location"),
                date=item.get("date"),
            )
            for item in data
        ]

    def get_top_n_of_total_likes(self, n=None):
        posts_with_likes = [post for post in self.posts if post.totalLikes is not None]
        sorted_objects = sorted(
            posts_with_likes, key=lambda x: getattr(x, "totalLikes"), reverse=True
        )
        result = [post.to_dict_custom("totalLikes") for post in sorted_objects]

        return result[:n]

    def get_top_n_of_total_views(self, n: int):
        posts_with_views = [post for post in self.posts if post.totalViews is not None]
        sorted_objects = sorted(
            posts_with_views, key=lambda x: getattr(x, "totalViews"), reverse=True
        )
        top_list = [post.to_dict_custom("totalViews") for post in sorted_objects[:n]]

        return top_list

    def get_top_n_of_writers(self, n: int):
        posts_with_user = [post for post in self.posts if post.totalViews is not None]
        result = {}
        for post in posts_with_user:
            if post.user in result:
                result[post.user] += 1
            else:
                result[post.user] = 1
        sorted_items = sorted(result.items(), key=lambda x: x[1], reverse=True)
        top_list = [{"id": key, "total": value} for key, value in sorted_items[:n]]

        return top_list


import os

import aiohttp
from aiohttp import ClientResponseError, ServerTimeoutError, TooManyRedirects

from userge import Config, Message, userge

DOGBIN_URL = "https://del.dog/"
NEKOBIN_URL = "https://nekobin.com/"

@userge.on_cmd(
    "paste",
    about={
        "header": "Pastes text or text_file to dogbin",
        "flags": {"-n": "use nekobin"},
        "usage": "{tr}paste [flags] [file_type] [text | reply to msg]",
        "examples": "{tr}paste -py import os",
    },
    del_pre=True,
)
async def paste_(message: Message) -> None:
    """pastes the text directly to dogbin or nekobin"""
    await message.edit("`Processing...`")
    text = message.filtered_input_str
    replied = message.reply_to_message
    use_neko = False
    file_ext = ".txt"
    if not text and replied:
        if replied.document and replied.document.file_size < 2 ** 20 * 10:
            file_ext = os.path.splitext(replied.document.file_name)[1]
            path = await replied.download(Config.DOWN_PATH)
            with open(path, "r") as d_f:
                text = d_f.read()
            os.remove(path)
        elif replied.text:
            text = replied.text
    if not text:
        await message.err("input not found!")
        return
    flags = list(message.flags)
    if "n" in flags:
        use_neko = True
        flags.remove("n")
    if flags and len(flags) == 1:
        file_ext = "." + flags[0]
    await message.edit("`Pasting text ...`")
    async with aiohttp.ClientSession() as ses:
        if use_neko:
            async with ses.post(
                NEKOBIN_URL + "api/documents", json={"content": text}
            ) as resp:
                if resp.status == 201:
                    response = await resp.json()
                    key = response["result"]["key"]
                    final_url = NEKOBIN_URL + key + file_ext
                    reply_text = f"**Nekobin** [URL]({final_url})"
                    await message.edit(reply_text, disable_web_page_preview=True)
                else:
                    await message.err("Failed to reach Nekobin")
        else:
            async with ses.post(
                DOGBIN_URL + "documents", data=text.encode("utf-8")
            ) as resp:
                if resp.status == 200:
                    response = await resp.json()
                    key = response["key"]
                    final_url = DOGBIN_URL + key
                    if response["isUrl"]:
                        reply_text = (
                            f"**Shortened** [URL]({final_url})\n"
                            f"**Dogbin** [URL]({DOGBIN_URL}v/{key})"
                        )
                    else:
                        reply_text = f"**Dogbin** [URL]({final_url}{file_ext})"
                    await message.edit(reply_text, disable_web_page_preview=True)
                else:
                    await message.err("Failed to reach Dogbin")

@userge.on_cmd(
    "getpaste",
    about={
        "header": "Gets the content of a del.dog paste",
        "usage": "{tr}getpaste [del.dog or nekobin link]",
    },
)
async def get_paste_(message: Message):
    """fetches the content of a dogbin or nekobin URL"""
    link = message.input_str
    if not link:
        await message.err("input not found!")
        return
    await message.edit("`Getting paste content...`")
    format_view = f"{DOGBIN_URL}v/"
    if link.startswith(format_view):
        link = link[len(format_view) :]
        raw_link = f"{DOGBIN_URL}raw/{link}"
    elif link.startswith(DOGBIN_URL):
        link = link[len(DOGBIN_URL) :]
        raw_link = f"{DOGBIN_URL}raw/{link}"
    elif link.startswith("del.dog/"):
        link = link[len("del.dog/") :]
        raw_link = f"{DOGBIN_URL}raw/{link}"
    elif link.startswith(NEKOBIN_URL):
        link = link[len(NEKOBIN_URL) :]
        raw_link = f"{NEKOBIN_URL}raw/{link}"
    elif link.startswith("nekobin.com/"):
        link = link[len("nekobin.com/") :]
        raw_link = f"{NEKOBIN_URL}raw/{link}"
    else:
        await message.err("Is that even a paste url?")
        return
    async with aiohttp.ClientSession(raise_for_status=True) as ses:
        try:
            async with ses.get(raw_link) as resp:
                text = await resp.text()
        except ServerTimeoutError as e_r:
            await message.err(f"Request timed out -> {e_r}")
        except TooManyRedirects as e_r:
            await message.err(
                "Request exceeded the configured "
                f"number of maximum redirections -> {e_r}"
            )
        except ClientResponseError as e_r:
            await message.err(f"Request returned an unsuccessful status code -> {e_r}")
        else:
            await message.edit_or_send_as_file(
                "--Fetched Content Successfully!--" f"\n\n**Content** :\n`{text}`"
            )


# -*- coding: utf-8 -*-

from odoo import models, fields, api
from odoo.exceptions import AccessError, UserError, ValidationError
import logging

_logger = logging.getLogger(__name__)

class PurchaseOrder(models.Model):
    _inherit = ['purchase.order']

#     can_send_po = fields.Boolean(compute = '_can_send_po')
    user_access_level = fields.Integer(compute='_compute_user_access', default=0)
    is_approve_visible = fields.Boolean(compute='_is_approve_visible', default=False)

    pre_approved = fields.Float(store=True, default=False)
    final_approved = fields.Float(store=True, default=False)

    # Overriding original action_cancel to ask for approval if user is not assistant nor manager
    def button_cancel(self):
        if not self.env.user.has_group('po_approval.group_purchase_assistant') and self.state not in ('draft'):
            # Getting all assistant/manager users
            all_users = self.env['res.users'].search([('active', '=', True)])
            my_users_group = all_users.filtered(lambda user: user.has_group('po_approval.group_purchase_assistant'))

            partner_ids = []
            for user in my_users_group:
                partner_ids.append(user.partner_id.id)

            _logger.info(f"Action cancel - po approval assistants {partner_ids}")
            view = self.env.ref('log_wizard.log_message_wizard_view')
            wiz = self.env['log.message.wizard'].create({})
            wiz_name = "PO cancellation request"

            ctx = {
                'subject': wiz_name,
                'partner_ids': partner_ids,
                'parent_model': self._name,
                'parent_id': self.id,
            }

            return {
                'name': wiz_name,
                'type': 'ir.actions.act_window',
                'view_type': 'form',
                'view_mode': 'form',
                'res_model': 'log.message.wizard',
                'views': [(view.id, 'form')],
                'view_id': view.id,
                'target': 'new',
                'res_id': wiz.id,
                'context': ctx
            }
        else:
            for order in self:
                for inv in order.invoice_ids:
                    if inv and inv.state not in ('cancel', 'draft'):
                        raise UserError("Unable to cancel this purchase order. You must first cancel the related vendor bills.")

            view = self.env.ref('log_wizard.log_message_wizard_view')
            wiz = self.env['log.message.wizard'].create({})
            wiz_name = "RFQ cancellation reason"

            ctx = {
                'subject': wiz_name,
                'to_state': 'cancel',
                'parent_model': self._name,
                'parent_id': self.id,
            }

            return {
                'name': wiz_name,
                'type': 'ir.actions.act_window',
                'view_type': 'form',
                'view_mode': 'form',
                'res_model': 'log.message.wizard',
                'views': [(view.id, 'form')],
                'view_id': view.id,
                'target': 'new',
                'res_id': wiz.id,
                'context': ctx
            }

    @api.depends('user_id')
    def _compute_user_access(self):
        # If user is manager - access level = 2
        # If user is assistant - access level = 1
        # If user is user - access level = 0
        _logger.info("_compute_user_access")
        if self.env.user.has_group('purchase.group_purchase_manager'):
            self.user_access_level = 2
            _logger.info("level 2")
        elif self.env.user.has_group('po_approval.group_purchase_assistant'):
            self.user_access_level = 1
            _logger.info("level 1")
        else:
            self.user_access_level = 0
            _logger.info("level 0")

#     @api.depends('user_access_level')
#     def _can_send_po(self):
#         _logger.info("_can_send_po")

#         user_access_level = self.user_access_level
#         env_lock_conf_po = self.company_id.po_lock == 'lock'

#         _logger.info(f"user level -> {user_access_level}")
#         _logger.info(f"lock conf PO -> {env_lock_conf_po}")
#         if env_lock_conf_po:
#             if user_access_level > 0 and self.state in ('done', 'purchase'):
#                 self.can_send_po = True
#                 _logger.info(f"can send -> True")
#             else:
#                 self.can_send_po = False
#                 _logger.info(f"can send -> False")
#         else:
#             self.can_send_po = True
#             _logger.info(f"can send -> True")

    @api.depends('user_id')
    def _is_approve_visible(self):

        if self.env.user.has_group('purchase.group_purchase_manager'):
            is_user_manager = True
            is_user_assistant = False
        else:
            is_user_manager = False
            if self.env.user.has_group('po_approval.group_purchase_assistant'):
                is_user_assistant = True
            else:
                is_user_assistant = False

        if self.state in ('to approve'):
            if (is_user_assistant and not self.pre_approved) or (is_user_manager):
                self.is_approve_visible = True
            else:
                self.is_approve_visible = False
        else:
            self.is_approve_visible = False

    # Overriding original method to allow two-people approval
    def button_approve(self, force=False, override=False):
        # all_users = self.env['res.users'].search([('active', '=', True)])

        if self.env.user.has_group('purchase.group_purchase_manager') or override:
            self.write({'pre_approved':True, 'final_approved':True})
            self.write({'state': 'purchase', 'date_approve': fields.Datetime.now()})
            self.filtered(lambda p: p.company_id.po_lock == 'lock').write({'state': 'done'})
            return {}
        else:
            self.pre_approved = True
            po_number = self.name

            msg = f"
The Purchase Order {po_number} needs final approval.
"
            msg += f"
Vendor: {self.partner_id.name}
"
            try:
                msg += f"
End Customer: {self.end_customer_id.name}
"
            except:
                pass

            all_users = self.env['res.users'].search([('active', '=', True)])

            my_users_group = all_users.filtered(lambda user: user.has_group('purchase.group_purchase_manager'))

            partner_ids = []
            for user in my_users_group:
                partner_ids.append(user.partner_id.id)

            self.message_post(
                subject='Purchase Order pending for Approval',
                body=msg,
                partner_ids=tuple(partner_ids),
            )
            return {}


from __future__ import absolute_import

from uuid import uuid4

from sentry.models import (
    Event,
    Group,
    GroupAssignee,
    GroupHash,
    GroupMeta,
    GroupRedirect,
    GroupStatus,
    ScheduledDeletion,
    UserReport,
)
from sentry.tasks.deletion import run_deletion
from sentry.testutils import TestCase
from sentry.testutils.helpers.datetime import iso_format, before_now

class DeleteGroupTest(TestCase):
    def test_simple(self):
        key = "key"
        value = "value"

        event_id = "a" * 32
        project = self.create_project()
        event = self.store_event(
            data={
                "event_id": event_id,
                "tags": {key: value},
                "timestamp": iso_format(before_now(minutes=1)),
            },
            project_id=project.id,
        )
        group = event.group
        group.update(status=GroupStatus.PENDING_DELETION)

        project = self.create_project()
        group = self.create_group(project=project)
        event = self.create_event(group=group)

        UserReport.objects.create(group_id=group.id, project_id=event.project_id, name="Jane Doe")

        GroupAssignee.objects.create(group=group, project=project, user=self.user)
        GroupHash.objects.create(project=project, group=group, hash=uuid4().hex)
        GroupMeta.objects.create(group=group, key="foo", value="bar")
        GroupRedirect.objects.create(group_id=group.id, previous_group_id=1)

        deletion = ScheduledDeletion.schedule(group, days=0)
        deletion.update(in_progress=True)

        with self.tasks():
            run_deletion(deletion.id)

        assert not Event.objects.filter(id=event.id).exists()
        assert not UserReport.objects.filter(group_id=group.id).exists()
        assert not GroupRedirect.objects.filter(group_id=group.id).exists()
        assert not GroupHash.objects.filter(group_id=group.id).exists()
        assert not Group.objects.filter(id=group.id).exists()


# coding=utf-8

import json
import urllib
import urllib2
import os

class ShouHuiJianZhu(object):
    def __init__(self):
        pass

    def gain_response(self, base_url, headers):
        # url = 'http://image.so.com/j?q=%E6%89%8B%E7%BB%98%E5%BB%BA%E7%AD%91&src=srp' \
        #       '&correct=%E6%89%8B%E7%BB%98%E5%BB%BA%E7%AD%91&pn=60&ch=&sn=96&cn=0&gn=0&kn=36'
        page = 0
        for i in range(10):
            page += 60
            params = {
                'q': '手绘建筑',
                'src': 'srp',
                'correct': '手绘建筑',
                'pn': page,
                'ch': '',
                'cn': '0',
                'gn': '0',
                'kn': '36'
            }
            data = urllib.urlencode(params)
            url = base_url + data
            request = urllib2.Request(url, headers=headers)
            response = urllib2.urlopen(request).read()
            json_data = json.loads(response)
            # print(json_data)
            list = json_data.get('list')
            for image in list:
                image_url = image.get('thumb')
                self.download_image(image_url)

    def download_image(self, image_url):
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'}
        request = urllib2.Request(image_url, headers=headers)
        response = urllib2.urlopen(request).read()
        filename = image_url[-10:]
        # os.makedirs('images')
        # folder = '/images'
        # if not os.path.exists(folder):
        #     os.makedirs(folder)
        with open(filename, 'wb')as f:
            f.write(response)

    def main(self):
        base_url = 'http://image.so.com/j?'
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'}
        self.gain_response(base_url, headers)

if __name__ == '__main__':
    jz = ShouHuiJianZhu()
    jz.main()


import torchvision.models as models
import torch
import torch.nn as nn
import torch.optim as optim
from torchvision import transforms
from transformers import DeiTForImageClassification
import database.dataset as dataset
import numpy as np
import time
from database.loss import MarginLoss, ProxyNCA_prob, NormSoftmax

from argparse import ArgumentParser, ArgumentTypeError

class Model(nn.Module):
    def __init__(self, model='densenet', eval=True, batch_size=32, num_features=128,
                 name='weights', use_dr=True, device='cuda:0', freeze=False):
        super(Model, self).__init__()
        if model == 'densenet':
            self.forward_function = self.forward_conv
            self.conv_net = models.densenet121(pretrained=True).to(device=device)
        elif model == 'resnet':
            self.forward_function = self.forward_conv
            self.conv_net = models.resnet50(pretrained=True).to(device=device)
        elif model == 'transformer':
            self.forward_function = self.forward_transformer
            self.model = DeiTForImageClassification.from_pretrained('facebook/deit-base-distilled-patch16-224').to(device=device)

        if freeze and model != 'transformer':
            for param in self.conv_net.parameters():
                param.requires_grad = False
        elif freeze:
            for param in self.model.parameters():
                param.requires_grad = False

        if use_dr and model != 'transformer':
            out_features = 4096
            if model == 'densenet':
                self.conv_net.classifier = nn.Linear(1024, out_features).to(device=device)
            elif model == 'resnet':
                self.conv_net.fc = nn.Linear(2048, out_features).to(device=device)

            self.first_conv1 = nn.Conv2d(3, 96, kernel_size=8, padding=1, stride=16).to(device=device)
            self.first_conv2 = nn.MaxPool2d(3, 4, 1).to(device=device)

            self.second_conv1 = nn.Conv2d(3, 96, kernel_size=7, padding=4, stride=32).to(device=device)
            self.second_conv2 = nn.MaxPool2d(7, 2, 3).to(device=device)

            self.linear = nn.Linear(7168, num_features).to(device=device)
            self.use_dr = True
        else:
            if model == 'densenet':
                self.conv_net.classifier = nn.Linear(1024, num_features).to(device=device)
            elif model == 'resnet':
                self.conv_net.fc = nn.Linear(2048, num_features).to(device=device)
            elif model == 'transformer':
                self.model.classifier = torch.nn.Linear(768, num_features).to(device=device)
                for module in filter(lambda m: type(m) == nn.LayerNorm, self.model.modules()):
                    module.eval()
                    module.train = lambda _: None
            self.use_dr = False

        self.num_features = num_features
        self.norm = nn.functional.normalize

        self.transformer = model == 'transformer'

        self.name = name
        self.device = device

        if eval == True:
            self.load_state_dict(torch.load(name))
            self.eval()
            self.eval = True
        else:
            self.train()
            self.eval = False
            self.batch_size = batch_size

    def forward_conv(self, input):
        tensor1 = self.conv_net(input)

        tensor1 = self.norm(tensor1)

        if self.use_dr:
            tensor2 = self.first_conv1(input)
            tensor2 = self.first_conv2(tensor2)
            tensor2 = torch.flatten(tensor2, start_dim=1)

            tensor3 = self.second_conv1(input)
            tensor3 = self.second_conv2(tensor3)
            tensor3 = torch.flatten(tensor3, start_dim=1)

            tensor4 = self.norm(torch.cat((tensor2, tensor3), 1))

            return self.norm(self.linear(torch.cat((tensor1, tensor4), 1)))

        return tensor1

    def forward_transformer(self, input):
        return self.norm(self.model(input).logits, 1)

    def forward(self, input):
        return self.forward_function(input)

    def train_epochs(self, dir, epochs, sched, loss, generalise, lr, decay, beta_lr, gamma, lr_proxies):
        data = dataset.TrainingDataset(dir, 2, generalise, self.transformer)
        print('Size of dataset', data.__len__())

        if loss == 'margin':
            loss_function = MarginLoss(n_classes=len(data.classes))

            to_optim = [{'params':self.parameters(),'lr':lr,'weight_decay':decay},
                        {'params':loss_function.parameters(), 'lr':beta_lr, 'weight_decay':0}]

            optimizer = torch.optim.Adam(to_optim)
        elif loss == 'proxy_nca_pp':
            loss_function = ProxyNCA_prob(len(data.classes), self.num_features, 3, device)

            to_optim = [
                {'params':self.parameters(), 'weight_decay':0},
                {'params':loss_function.parameters(), 'lr': lr_proxies},
            ]

            optimizer = torch.optim.Adam(to_optim, lr=lr, eps=1)
        elif loss == 'softmax':
            loss_function = NormSoftmax(0.05, len(data.classes), self.num_features, lr_proxies, self.device)

            to_optim = [
                {'params':self.parameters(),'lr':lr,'weight_decay':decay},
                {'params':loss_function.parameters(),'lr':lr_proxies,'weight_decay':decay}
            ]

            optimizer = torch.optim.Adam(to_optim)

        if sched == 'exponential':
            scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=gamma)
        elif sched == 'step':
            scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[epochs//2, epochs],
                                                            gamma=gamma)

        loader = torch.utils.data.DataLoader(data, batch_size=self.batch_size,
                                             shuffle=True, num_workers=12,
                                             pin_memory=True)

        loss_list = []
        try:
            for epoch in range(epochs):
                start_time = time.time()

                for i, (labels, images) in enumerate(loader):
                    images_gpu = images.to(device=self.device)
                    labels = labels.to(device=self.device)

                    if not self.transformer:
                        out = self.forward(images_gpu)
                    else:
                        out = self.forward(images_gpu.view(-1, 3, 224, 224))

                    loss = loss_function(out, labels)

                    optimizer.zero_grad(set_to_none=True)
                    loss.backward()
                    optimizer.step()

                    loss_list.append(loss.item())

                print("epoch {}, loss = {}, time {}".format(epoch, np.mean(loss_list),
                                                            time.time() - start_time))

                print("\n----------------------------------------------------------------\n")
                if sched != None:
                    scheduler.step()

                torch.save(self.state_dict(), self.name)

        except KeyboardInterrupt:
            print("Interrupted")

    def train_dr(self, data, num_epochs, lr):
        data = dataset.DRDataset(data)
        print('Size of dataset', data.__len__())

        loader = torch.utils.data.DataLoader(data, batch_size=self.batch_size,
                                             shuffle=True, num_workers=12,
                                             pin_memory=True)
        loss_function = torch.nn.TripletMarginLoss()
        optimizer = torch.optim.Adam(self.parameters(), lr=lr)
        loss_list = []
        try:
            for epoch in range(num_epochs):
                start_time = time.time()

                for i, (image0, image1, image2) in enumerate(loader):
                    image0 = image0.to(device='cuda:0')
                    image1 = image1.to(device='cuda:0')
                    image2 = image2.to(device='cuda:0')

                    out0 = self.forward(image0).cpu()
                    out1 = self.forward(image1).cpu()
                    out2 = self.forward(image2).cpu()

                    loss = loss_function(out0, out1, out2)

                    optimizer.zero_grad(set_to_none=True)
                    loss.backward()
                    optimizer.step()

                    loss_list.append(loss.item())

                print("epoch {}, batch {}, loss = {}".format(epoch, i,
                                                             np.mean(loss_list)))
                loss_list.clear()
                print("time for epoch {}".format(time.time()- start_time))

                torch.save(self.state_dict(), self.name)

                if (epoch + 1) % 4:
                    lr /= 2
                    for param in optimizer.param_groups:
                        param['lr'] = lr
        except KeyboardInterrupt:
            print("Interrupted")

if __name__ == "__main__":
    parser = ArgumentParser()

    parser.add_argument(
        '--num_features',
        type=int,
        help='number of features to use',
        default=128
    )

    parser.add_argument(
        '--batch_size',
        type=int,
        default=32
    )

    parser.add_argument(
        '--model',
        help='feature extractor to use',
        default='densenet'
    )

    parser.add_argument(
        '--weights',
        default='weights'
    )

    parser.add_argument(
        '--training_data',
    )

    parser.add_argument(
        '--dr_model',
        action="store_true"
    )

    parser.add_argument(
        '--num_epochs',
        type=int,
        default=5
    )

    parser.add_argument(
        '--scheduler',
        default=None,
        help='<exponential, step>'
    )

    parser.add_argument(
        '--gpu_id',
        default=0,
        type=int
    )

    parser.add_argument(
        '--loss',
        default='margin',
        help='<margin, proxy_nca_pp, softmax, deep_ranking>'
    )

    parser.add_argument(
        '--freeze',
        action='store_true'
    )

    parser.add_argument(
        '--generalise',
        action='store_true',
        help='train on only half the classes of images'
    )

    parser.add_argument(
        '--lr',
        default=0.0001,
        type=float
    )

    parser.add_argument(
        '--decay',
        default=0.0004,
        type=float
    )

    parser.add_argument(
        '--beta_lr',
        default=0.0005,
        type=float
    )

    parser.add_argument(
        '--gamma',
        default=0.3,
        type=float
    )

    parser.add_argument(
        '--lr_proxies',
        default=0.00001,
        type=float
    )

    args = parser.parse_args()

    if args.gpu_id >= 0:
        device = 'cuda:' + str(args.gpu_id)
    else:
        device = 'cpu'

    m = Model(model=args.model, eval=False, batch_size=args.batch_size,
              num_features=args.num_features, name=args.weights,
              use_dr=args.dr_model, device=device, freeze=args.freeze)

    if args.loss == 'deep_ranking':
        m.train_dr(args.training_data, args.num_epochs, args.lr)
    else:
        m.train_epochs(args.training_data, args.num_epochs, args.scheduler, args.loss, args.generalise,
                       args.lr, args.decay, args.beta_lr, args.gamma, args.lr_proxies)


import random

from sklearn.linear_model import LogisticRegression

from metal.utils import convert_labels
from metal.metrics import metric_score

class LogisticRegressionWrapper(object):
    """A wrapper around scikit-learn's LogisticRegression class

    The wrapper is necessary both to convert labels from categorical to one-zero
    and to match the interface expected by snorkel-metal's ModelTuners.
    """
    def __init__(self, C=1.0, penalty='l2', seed=None):
        if seed:
            random.seed(seed)
        self.model = LogisticRegression(C=C, penalty='l2')

    def train(self, X, Y, X_dev=None, Y_dev=None, **kwargs):
        Y_bin = convert_labels(Y, 'categorical', 'onezero')
        self.model.fit(X, Y_bin)

    def predict(self, X):
        return self.model.predict(X)

    def score(self, X, Y, metric='f1', verbose=True):
        Y = convert_labels(Y, 'categorical', 'onezero')
        Y_p = self.predict(X)

        metric_list = metric if isinstance(metric, list) else [metric]
        scores = []
        for metric in metric_list:
            score = metric_score(Y, Y_p, metric)
            scores.append(score)
            if verbose:
                print(f"{metric.capitalize()}: {score:.3f}")

        if isinstance(scores, list) and len(scores) == 1:
            return scores[0]
        else:
            return scores


from django import template
from django.template.defaultfilters import stringfilter

register = template.Library()

@register.filter
@stringfilter
def formatar_preco(preco):
    preco = preco[:-3]
    preco = preco[:-2] + ',' + preco[-2:]
    return preco

register.filter('formatar_preco', formatar_preco)


from django.contrib import admin
from .models import *

# Register your models here.
class CustomUserAdmin(admin.ModelAdmin):
    list_display = ('phone_number', 'joined', 'email', 'is_superuser')
    date_hierarchy = 'joined'
    list_filter = ('joined',    )
    search_fields = ('email', 'phone_number')

    fieldsets = (
        (None, {
            'fields': (
                'email', 'password'
            ),
        }),
        ('User status', {
            'fields': (
                'is_staff', 'is_superuser', 'is_active'
            ),
        }),
    )

admin.site.register(CustomUser, CustomUserAdmin)
admin.site.register(UserOTP)


import lightning as pl
import torch
import transformers
from transformers import AdamW, T5ForConditionalGeneration, T5TokenizerFast as T5Tokenizer
from model import SummaryModel

tokenizer = T5Tokenizer.from_pretrained("t5-base")

checkpoint="base-checkpoint.ckpt"
model = SummaryModel()
model=model.load_from_checkpoint(checkpoint,map_location="cpu")

def generate_summary(question):

    inputs_encoding =  tokenizer(
        question,
        add_special_tokens=True,
        max_length= 512,
        padding = 'max_length',
        truncation='only_first',
        return_attention_mask=True,
        return_tensors="pt"
        )

    generate_ids = model.model.generate(
        input_ids = inputs_encoding["input_ids"],
        attention_mask = inputs_encoding["attention_mask"],
        max_length = 128,
        num_beams = 4,
        num_return_sequences = 1,
        no_repeat_ngram_size=2,
        early_stopping=True,
        )

    preds = [
        tokenizer.decode(gen_id,
        skip_special_tokens=True,
        clean_up_tokenization_spaces=True)
        for gen_id in generate_ids
    ]

    return "".join(preds)


import sys
import random
import datetime

import numpy as np
import torch
import wandb

from rppg.loss import loss_fn
from rppg.models import get_model
from rppg.optim import optimizer
from rppg.config import get_config
from rppg.dataset_loader import (dataset_loader, dataset_split, data_loader)
from rppg.preprocessing.dataset_preprocess import check_preprocessed_data
from rppg.run import run
from rppg.utils.test_utils import save_single_result

SEED = 0

# for Reproducible model
torch.manual_seed(SEED)
torch.cuda.manual_seed(SEED)
torch.cuda.manual_seed_all(SEED)  # if use multi-GPU
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(SEED)
random.seed(SEED)

generator = torch.Generator()
generator.manual_seed(SEED)

if __name__ == "__main__":

    cfg = get_config("configs/base_config.yaml")
    # preprocess_cfg = get_config("rppg/configs/preprocess.yaml")
    result_save_path = 'result/csv/'

    check_preprocessed_data(cfg)

    datasets = dataset_loader(fit_cfg=cfg.fit, dataset_path=cfg.dataset_path)

    data_loaders = data_loader(datasets=datasets, fit_cfg=cfg.fit)

    model = get_model(cfg.fit)

    if cfg.wandb.flag and cfg.fit.train_flag:
        wandb.init(project=cfg.wandb.project_name,
                   entity=cfg.wandb.entity,
                   name=cfg.fit.model + "/" +
                        cfg.fit.train.dataset + "/" +
                        cfg.fit.test.dataset + "/" +
                        str(cfg.fit.img_size) + "/" +
                        str(cfg.fit.test.batch_size // cfg.fit.train.fs) + "/" +
                        datetime.datetime.now().strftime('%m-%d%H:%M:%S'))
        wandb.config = {
            "learning_rate": cfg.fit.train.learning_rate,
            "epochs": cfg.fit.train.epochs,
            "train_batch_size": cfg.fit.train.batch_size,
            "test_batch_size": cfg.fit.test.batch_size
        }
        wandb.watch(model, log="all", log_freq=10)

    opt = None
    criterion = None
    lr_sch = None
    if cfg.fit.train_flag:
        opt = optimizer(
            model_params=model.parameters(),
            learning_rate=cfg.fit.train.learning_rate,
            optim=cfg.fit.train.optimizer)
        criterion = loss_fn(loss_name=cfg.fit.train.loss)
        lr_sch = torch.optim.lr_scheduler.OneCycleLR(
            opt, max_lr=cfg.fit.train.learning_rate, epochs=cfg.fit.train.epochs,
            steps_per_epoch=len(datasets[0]))
    test_result = run(model, False, opt, lr_sch, criterion, cfg, data_loaders)

    save_single_result(result_save_path, test_result, cfg.fit)

    sys.exit(0)


import os
import platform
import sys
try:
	import termios
	import tty
except ImportError:
	import msvcrt

from .ansi_escapes import *
from .colorama import just_fix_windows_console

if platform.system() == "Windows":
	just_fix_windows_console()

class Shell():
	offset = 0
	line = 0

	def __init__(self, stdin = sys.stdin, stdout = sys.stdout):
		self.stdin = stdin
		self.stdout = stdout
		self.eof_when_enter = False
		self.interactables = []

	def read(self, count = 1):
		return self.stdin.read(count)

	def readline(self, count = 1):
		return self.stdin.readline(count)

	def input_raw(self, count = 1):
		try:
			fd = self.stdin.fileno()
			term_attrs = termios.tcgetattr(fd)
		except NameError:
			pass
		try:
			try:
				tty.setraw(fd)
				key = self.stdin.read(count)
			except NameError:
				key = msvcrt.getwch()
				count -= 1
				while count > 0:
					key += msvcrt.getwch()
					count -= 1
		finally:
			try:
				termios.tcsetattr(fd, termios.TCSADRAIN, term_attrs)
			except NameError:
				pass
		return key

	def input(self, count = 1):
		key = self.input_raw(count)
		if key == "\x03" or key == "\x1a": # Ctrl+C or Ctrl+Z
			raise KeyboardInterrupt()
		return key

	def inputline(self, count = 1):
		buffer = ""
		while count > 0:
			key = self.input()
			if ord(key) in {10, 13}: # Enter
				if count > 1:
					buffer += "\n"
				count -= 1
			else:
				buffer += key
		return buffer

	def write(self, value):
		self.stdout.write(value)
		value = str(value)
		self.line += value.count("\n")
		try:
			where = value.rindex("\n") + 1
			self.offset = len(value[where:])
		except ValueError:
			self.offset += len(value)

	def up(self, count):
		self.write(cursor_up(count))

	def down(self, count):
		self.write(cursor_down(count))

	def right(self, count):
		self.write(cursor_forward(count))

	def left(self, count):
		self.write(cursor_backward(count))

	def clear(self):
		if self.line > 0:
			self.stdout.write(erase_lines(self.line + 1))
			self.line = 0
		self.offset = 0

	def get_interactable(self, criteria):
		try:
			return self.interactables[criteria]
		except IndexError:
			pass
		except TypeError:
			pass
		for interactable in self.interactables:
			if interactable.key == criteria:
				return interactable
		raise ValueError(criteria)

	def observe(self, key):
		if self.eof_when_enter and ord(key) in {10, 13}: # Enter
			raise EOFError()
		return False

	def draw(self, interactable):
		return interactable.render(self, self.offset, self.line)

	def touch(self, interactable, key):
		return interactable.observe_key(key)

	def render(self):
		self.clear()
		self.write("\n")
		for interactable in self.interactables:
			self.draw(interactable)

	def enter(self):
		self.hide_cursor()
		self.render()

	def loop(self):
		self.enter()
		while True:
			try:
				key = self.input(1)
				observed = False
				for interactable in self.interactables:
					observed = self.touch(interactable, key) or observed
				if not observed:
					observed = self.observe(key)
				if observed:
					self.render()
			except EOFError:
				break
			except KeyboardInterrupt as err:
				self.leave()
				raise err
		self.leave()

	def leave(self):
		self.show_cursor()

	def hide_cursor(self):
		self.write(CURSOR_HIDE)

	def show_cursor(self):
		self.write(CURSOR_SHOW)

	class Interactable():
		def __init__(self, key):
			self.key = key

		def observe_key(self, what):
			return False

		def render(self, shell, offset, line):
			pass

		def lines(self, shell):
			return 0

class InteractiveShell(Shell):
	page_buffer_offset = 0
	global_buffer_offset = 0
	page = 1

	def __init__(self, stdin = sys.stdin, stdout = sys.stdout, infinite_scroll = False, lines_per_page = 6, implicit_page_indicator = False):
		Shell.__init__(self, stdin, stdout)
		self.infinite_scroll = infinite_scroll
		self.lines_per_page = lines_per_page
		self.implicit_page_indicator = implicit_page_indicator

	def observe(self, raw):
		observed = Shell.observe(self, raw)
		if raw != "\x1b" and raw != "\xe0" and raw != "\x00":
			return observed
		key = self.input_raw(1)
		if raw == "\xe0" or raw == "\x00": # Windows
			if key == "M": # Right
				self.turn_forward()
			elif key == "K": # Left
				self.turn_backward()
			else:
				return observed
			return True
		if key != "[": # Unix
			return observed
		joy = self.input_raw(1)
		if joy == "C": # Right
			self.turn_forward()
		elif joy == "D": # Left
			self.turn_backward()
		else:
			return observed
		return True

	def turn_forward(self):
		if self.global_buffer_offset + self.page_buffer_offset >= len(self.interactables):
			if self.infinite_scroll:
				self.global_buffer_offset = self.page_buffer_offset = 0
				self.page = 1
			return

		self.global_buffer_offset += self.page_buffer_offset
		self.page_buffer_offset = 0
		self.page += 1

	def turn_backward(self):
		index = self.global_buffer_offset
		if index == 0:
			if self.infinite_scroll:
				page = 1
				page_occupied_lines = 0
				page_buffer_offset = 0

				while index < len(self.interactables):
					lines = self.interactables[index].lines(self)
					if lines > self.lines_per_page or lines < 0:
						raise BufferError()

					if page_occupied_lines + lines > self.lines_per_page:
						page_occupied_lines = page_buffer_offset = 0
						page += 1
						continue

					page_occupied_lines += lines
					page_buffer_offset += 1
					index += 1

				self.global_buffer_offset = index - page_buffer_offset
				self.page_buffer_offset = 0
				self.page = page
			return

		page_occupied_lines = 0
		while index > 0:
			lines = self.interactables[index - 1].lines(self)
			if lines > self.lines_per_page or lines < 0:
				raise BufferError()

			if page_occupied_lines + lines > self.lines_per_page:
				break

			page_occupied_lines += lines
			index -= 1

		self.global_buffer_offset = index
		self.page_buffer_offset = 0
		self.page -= 1

	def draw(self, interactable, page, page_occupied_lines):
		if isinstance(interactable, InteractiveShell.Interactable):
			return interactable.render(self, self.offset, self.line, page, self.page_buffer_offset, page_occupied_lines)
		return Shell.draw(self, interactable)

	def write_implicit_indicator(self):
		if self.implicit_page_indicator and (self.global_buffer_offset > 0 or self.global_buffer_offset + self.page_buffer_offset < len(self.interactables)):
			self.write(
				"\n" * (self.lines_per_page + 1 - self.line) +
				(".." if self.global_buffer_offset > 0 else " " * 2)
				+ " " * 45 +
				(".." if self.global_buffer_offset + self.page_buffer_offset < len(self.interactables) else " " * 2)
				+ "\n"
			)

	def render(self):
		self.clear()
		if len(self.interactables) == 0:
			return
		if self.global_buffer_offset >= len(self.interactables):
			raise IndexError()
		self.write("\n")

		page_occupied_lines = 0
		self.page_buffer_offset = 0

		while self.global_buffer_offset + self.page_buffer_offset < len(self.interactables):
			interactable = self.interactables[self.global_buffer_offset + self.page_buffer_offset]
			lines = interactable.lines(self)
			if lines > self.lines_per_page or lines < 0:
				raise BufferError()
			if page_occupied_lines + lines > self.lines_per_page:
				break
			self.draw(interactable, self.page, page_occupied_lines)
			page_occupied_lines += lines
			self.page_buffer_offset += 1

		self.write_implicit_indicator()

	def enter(self):
		self.global_buffer_offset = self.page_buffer_offset = 0
		self.page = 1
		Shell.enter(self)

	def leave(self):
		self.clear()
		Shell.leave(self)

	class Interactable(Shell.Interactable):
		def __init__(self, key):
			Shell.Interactable.__init__(self, key)

		def render(self, shell, offset, line, page = 0, index = -1, lines_before = -1):
			pass

class SelectiveShell(InteractiveShell):
	page_cursor_offset = -1
	pending_hover_offset = 0
	blocked_in_page = False

	def __init__(self, stdin = sys.stdin, stdout = sys.stdout, infinite_scroll = False, lines_per_page = 6, implicit_page_indicator = False):
		InteractiveShell.__init__(self, stdin, stdout, infinite_scroll, lines_per_page, implicit_page_indicator)
		self.eof_when_enter = True

	def turn_backward(self):
		if not self.blocked_in_page:
			InteractiveShell.turn_backward(self)
		self.page_cursor_offset += 1
		self.pending_hover_offset = -2

	def turn_forward(self):
		if not self.blocked_in_page:
			InteractiveShell.turn_forward(self)
		self.page_cursor_offset -= 1
		self.pending_hover_offset = 2

	def turn_up(self):
		if self.page_cursor_offset > 0 or self.infinite_scroll or self.global_buffer_offset > 0:
			self.pending_hover_offset = -1 if not self.blocked_in_page else -2

	def turn_down(self):
		if (self.page_cursor_offset < self.page_buffer_offset and self.which() < len(self.interactables) - 1) or self.infinite_scroll:
			self.pending_hover_offset = 1 if not self.blocked_in_page else 2

	def hover_previous(self):
		cursor_offset = min(self.page_cursor_offset, self.page_buffer_offset) - 1
		while cursor_offset >= 0:
			try:
				interactable = self.interactables[self.global_buffer_offset + cursor_offset]
			except IndexError:
				cursor_offset -= 1
				continue
			if isinstance(interactable, SelectiveShell.Selectable) and interactable.hoverable():
				self.page_cursor_offset = cursor_offset
				return True
			cursor_offset -= 1
		return False

	def hover_next(self):
		cursor_offset = max(self.page_cursor_offset, -1) + 1
		while cursor_offset < self.page_buffer_offset:
			try:
				interactable = self.interactables[self.global_buffer_offset + cursor_offset]
			except IndexError:
				cursor_offset += 1
				continue
			if isinstance(interactable, SelectiveShell.Selectable) and interactable.hoverable():
				self.page_cursor_offset = cursor_offset
				return True
			cursor_offset += 1
		return False

	def touch(self, interactable, key):
		if isinstance(interactable, SelectiveShell.Selectable):
			try:
				return interactable.observe_key(key, self.interactables.index(interactable) == self.which())
			except ValueError:
				pass
		return InteractiveShell.touch(self, interactable, key)

	def render(self):
		InteractiveShell.render(self)

		# Changes cursor location to previous one if possible
		if self.pending_hover_offset != 0:
			if self.pending_hover_offset <= -1:
				if not self.hover_previous():

					if self.pending_hover_offset == -1:
						if self.infinite_scroll or self.global_buffer_offset > 0:
							self.page_cursor_offset = self.lines_per_page
							InteractiveShell.turn_backward(self)
							InteractiveShell.render(self)
							if not self.hover_previous():
								self.page_cursor_offset = -1

					elif not self.hover_next() and not self.hovered():
						self.page_cursor_offset = -1

				InteractiveShell.render(self)

			elif self.pending_hover_offset >= 1:
				if not self.hover_next():

					if self.pending_hover_offset == 1:
						self.page_cursor_offset = -1
						InteractiveShell.turn_forward(self)
						InteractiveShell.render(self)
						if not self.hover_next():
							self.page_cursor_offset = -1

					elif not self.hover_previous() and not self.hovered():
						self.page_cursor_offset = -1

				InteractiveShell.render(self)

			self.pending_hover_offset = 0

	def observe(self, raw):
		if self.eof_when_enter and ord(raw) in {10, 13}: # Enter
			if self.which() == -1:
				return False
			raise EOFError()
		observed = Shell.observe(self, raw)
		if raw != "\x1b" and raw != "\xe0" and raw != "\x00":
			return observed
		key = self.input_raw(1)
		if raw == "\xe0" or raw == "\x00": # Windows
			if key == "H": # Up
				self.turn_up()
			elif key == "P": # Down
				self.turn_down()
			elif key == "M": # Right
				self.turn_forward()
			elif key == "K": # Left
				self.turn_backward()
			else:
				return observed
			return True
		if key != "[":
			return observed
		joy = self.input_raw(1) # Unix
		if joy == "A": # Up
			self.turn_up()
		elif joy == "B": # Down
			self.turn_down()
		elif joy == "C": # Right
			self.turn_forward()
		elif joy == "D": # Left
			self.turn_backward()
		else:
			return observed
		return True

	def draw(self, interactable, page, page_occupied_lines):
		if isinstance(interactable, SelectiveShell.Selectable):
			return interactable.render(self, self.offset, self.line, page, self.page_buffer_offset, page_occupied_lines, self.page_cursor_offset == self.page_buffer_offset)
		return InteractiveShell.draw(self, interactable, page, page_occupied_lines)

	def enter(self):
		self.page_cursor_offset = -1
		self.pending_hover_offset = 2
		InteractiveShell.enter(self)

	def hovered(self):
		interactable = self.get_interactable(self.which())
		return interactable.hoverable() if isinstance(interactable, SelectiveShell.Selectable) else False

	def which(self):
		return self.global_buffer_offset + self.page_cursor_offset if self.page_cursor_offset != -1 else -1

	def what(self):
		try:
			return self.interactables[self.which()].key
		except IndexError:
			return None

	class Selectable(InteractiveShell.Interactable):
		def __init__(self, key):
			InteractiveShell.Interactable.__init__(self, key)

		def render(self, shell, offset, line, page = 0, index = -1, lines_before = -1, at_cursor = None):
			pass

		def hoverable(self):
			return True

		def placeholder(self):
			pass

		def observe_key(self, what, at_cursor = None):
			return False

class Separator(Shell.Interactable):
	def __init__(self, key = "separator", size = 1):
		Shell.Interactable.__init__(self, key)
		self.size = size

	def render(self, shell, offset, line):
		shell.write("\n" * self.size)

	def lines(self, shell):
		return self.size

class Notice(Shell.Interactable):
	def __init__(self, key, text = None):
		Shell.Interactable.__init__(self, key)
		self.text = text if text is not None else key

	def render(self, shell, offset, line):
		shell.write(str(self.text) + "\n")

	def lines(self, shell):
		return str(self.text).count("\n") + 1

class Entry(SelectiveShell.Selectable):
	def __init__(self, key, text = None, arrow = "> "):
		SelectiveShell.Selectable.__init__(self, key)
		self.text = text if text is not None else key
		self.arrow = arrow

	def get_arrow(self, at_cursor = None):
		return "" if at_cursor is None else \
			str(self.arrow) if at_cursor else " " * len(self.arrow)

	def render(self, shell, offset, line, page = 0, index = -1, lines_before = -1, at_cursor = None):
		shell.write(self.get_arrow(at_cursor) + str(self.text) + "\n")

	def placeholder(self):
		return str(self.text).partition("\n")[0]

	def lines(self, shell):
		return str(self.text).count("\n") + 1

class Switch(Entry):
	def __init__(self, key, text = None, checked = False, arrow = "> ", checked_arrow = "* ", hover_arrow = ">*"):
		Entry.__init__(self, key, text, arrow)
		self.checked_arrow = checked_arrow
		self.checked = checked
		self.hover_arrow = hover_arrow

	def get_arrow(self, at_cursor = None):
		return str(self.hover_arrow) if at_cursor and self.checked else \
			str(self.arrow) if at_cursor else str(self.checked_arrow) if self.checked else " " * len(self.arrow)

	def observe_key(self, what, at_cursor = None):
		if at_cursor and ord(what) in {10, 13}:
			self.checked = not self.checked
			return True
		return Entry.observe_key(self, what)

class Input(Entry):
	def __init__(self, key, hint = None, text = "", arrow = "> ", template = None, maximum_length = 40):
		Entry.__init__(self, key, text, arrow)
		self.hovered = False
		self.hint = hint
		self.template = template
		self.maximum_length = maximum_length

	def render(self, shell, offset, line, page = 0, index = -1, lines_before = -1, at_cursor = None):
		shell.write(self.get_arrow(at_cursor) + (str(self.hint) if self.hint is not None else "") +
			("" if self.hovered else "\x1b[2m") + (self.text if len(self.text) > 0 or self.hovered else \
				"..." if self.template is None else self.template) + ("\x1b[7m " if self.hovered else "") + "\x1b[0m\n")

	def read(self):
		return self.template if not self.hovered and len(self.text) == 0 and self.template is not None else self.text

	def observe_key(self, what, at_cursor = None):
		if at_cursor:
			if ord(what) in {10, 13}:
				self.hovered = not self.hovered
				return True
			if self.hovered:
				if what in ("\x7f", "\x08"):
					if len(self.text) > 0:
						self.text = self.text[::-1][1:][::-1]
					else:
						self.hovered = False
				elif what.isprintable() and len(self.text + what) <= self.maximum_length:
					self.text += what
				return True
		return Entry.observe_key(self, what)

class Progress(Shell.Interactable):
	def __init__(self, key = "progress", progress = 0, weight = 49, text = None):
		Shell.Interactable.__init__(self, key)
		self.progress = progress
		self.weight = weight
		self.text = text

	def render(self, shell, offset, line):
		text = (str(self.text) if self.text is not None else str(int(self.progress * 100)) + "%").center(self.weight)
		size = int(self.weight * self.progress)
		shell.write("\x1b[7m" + text[:size] + "\x1b[2m" + text[size:self.weight] + "\x1b[0m\n")

	def seek(self, progress, text = None):
		self.progress = progress
		if text is not None:
			self.text = text

	def lines(self, shell):
		return (str(self.text).count("\n") if self.text is not None else 0) + 1

class Interrupt(InteractiveShell.Interactable):
	def __init__(self, key = "interrupt", occupied_page = True):
		InteractiveShell.Interactable.__init__(self, key)
		self.ocuppied_page = occupied_page

	def render(self, shell, offset, line, page = 0, index = -1, lines_before = -1):
		raise EOFError()

	def lines(self, shell):
		return shell.lines_per_page if self.ocuppied_page else 0

class Debugger(SelectiveShell.Selectable):
	def __init__(self, key = "debugger"):
		SelectiveShell.Selectable.__init__(self, key)

	def render(self, shell, offset, line, page = 0, index = -1, lines_before = -1, at_cursor = None):
		shell.write(f"Page {page}:{shell.global_buffer_offset}, offset {offset}/{shell.page_buffer_offset}, cursor {shell.page_cursor_offset}\n")

	def hoverable(self):
		return False

def select_prompt(prompt = None, *variants, fallback = None, what_not_which = False):
	if prompt is not None:
		print(prompt, end="")
	shell = SelectiveShell(infinite_scroll=True, implicit_page_indicator=True)
	for variant in variants:
		shell.interactables.append(Entry(variant))
	try:
		shell.loop()
		result = shell.which()
	except KeyboardInterrupt:
		result = fallback
	try:
		interactable = shell.get_interactable(result)
	except ValueError:
		return None
	try:
		print((prompt + " " if prompt is not None else "") + "\x1b[2m" + interactable.placeholder() + "\x1b[0m")
	except ValueError:
		pass
	return result if not what_not_which else interactable.key if interactable is not None else None

if __name__ == "__main__":
	shell = Shell()
	while True:
		try:
			key = shell.input(1)
		except KeyboardInterrupt:
			break
		print(ord(key), ": ", str(key.encode("unicode-escape"))[2:][::-1][1:][::-1].replace("\\\\", "\\"), sep="")


import math
import numpy as np
import scipy.optimize as optimize

import mlak.LinearAlgebra as la
import mlak.MathTools as mt
import mlak.ModelAnalyzer as ma

def apply_topology( topology, theta_ ):
	theta = []
	offset = 0
	for i in range( 1, len( topology ) ):
		rows = topology[i]
		cols = topology[i - 1] + 1
		s = rows * cols
		t = np.array( theta_[offset:offset + s] )
		t.shape = ( rows, cols )
		theta.append( t )
		offset += s
	return theta

def flatten_nn( theta ):
	return np.concatenate( list( map( lambda x : x.flatten(), theta ) ) )

def randomize_weights( topology, theta ):
	offset = 0
	ss = math.sqrt( 6 )
	for i in range( 1, len( topology ) ):
		rows = topology[i]
		cols = topology[i - 1] + 1
		s = rows * cols
		epsilon = ss / math.sqrt( rows + cols )
		theta[offset:offset + s] = np.random.rand( s ) * epsilon * 2 - epsilon
		offset += s

def compute_cost( theta, *args ):
	X, y, topology, classCount, Lambda, debug = args
	theta = apply_topology( topology, theta )

	for l in range( len( theta ) ):
		X = la.add_ones_column( X )
		X = np.dot( X, theta[l].T )
		X = mt.sigmoid( X )

	m = len( y )
	yp = np.zeros( ( m, classCount ) )
	yp[np.arange( m ), y.flatten()] = 1
	cost = np.sum( -( yp * mt.log( X ) + ( 1 - yp ) * mt.log( 1 - X ) ) )
	r = 0
	for t in theta:
		r += np.sum( t[:,1:] ** 2 )
	cost /= m
	cost += Lambda * r / ( m * 2 )
	if debug: # pragma: no cover
		print( "cost = {}                                \r".format( cost ), end = "" )
	return cost

def activation_derivative( a ):
	return a * ( 1 - a )

def compute_grad( theta, *args ):
	X, y, topology, classCount, Lambda, debug = args
	theta = apply_topology( topology, theta )
	thetaT = []
	for t in theta:
		thetaT.append( t.T )
	gradient = []
	for t in theta:
		gradient.append( np.zeros( t.shape ) )
	activation = []
	for t in thetaT:
		X = la.add_ones_column( X )
		activation.append( X )
		X = np.dot( X, t )
		X = mt.sigmoid( X )
	m = len( y )
	yp = np.zeros( ( m, classCount ) )
	yp[np.arange( m ), y.flatten()] = 1
	delta = ( X - yp ).T
	for g, t, a in zip( reversed( gradient ), reversed( thetaT ), reversed( activation ) ):
		g += np.dot( delta, a )
		delta = np.dot( t, delta ) * activation_derivative( a.T )
		delta = delta[1:]
	for g, t in zip( gradient, theta ):
		t[:, 0] = 0
		g += t * Lambda
		g /= m
	return flatten_nn( gradient )

def predict_one_vs_all( X, topoTheta ):
	topology, theta = topoTheta
	theta = apply_topology( topology, theta )
	y = []
	for i, x in enumerate( X ):
		for l in range( len( theta ) ):
			x = np.concatenate( ( [1], x ) )
			x = np.dot( theta[l], x )
			x = mt.sigmoid( x )
		y.append( x )
	return np.array( y )

class NeuralNetworkSolver:
	def __initial_theta( shaper, y, solution, nnTopology = None, **kwArgs ):
		if solution is None:
			shaper.learn_labels( y )
			topology = list( map( int, nnTopology.split( "," ) ) ) if nnTopology else []
			topology = [shaper.feature_count()] + topology + [shaper.class_count()]
			s = 0
			for i in range( 1, len( topology ) ):
				s += topology[i] * ( topology[i - 1] + 1 )
			theta = np.zeros( s )
			randomize_weights( topology, theta )
			return topology, theta
		return solution.model()

	def type( self_ ):
		return ma.SolverType.CLASSIFIER

	def train( self_, X, y, solution = None, Lambda = 0, iterations = 50, **kwArgs ):
		shaper = solution.shaper() if solution else ma.DataShaper( X, **kwArgs )
		debug = kwArgs.get( "debug", False )
		topology, theta = NeuralNetworkSolver.__initial_theta( shaper, y, solution, **kwArgs )
		y = shaper.map_labels( y )
		X = shaper.conform( X, addOnes = False )
		theta = optimize.fmin_cg(
			compute_cost,
			theta, fprime = compute_grad,
			args = ( X, y, topology, shaper.class_count(), Lambda, debug ),
			maxiter = iterations,
			disp = False
		)
		return ma.Solution( model = ( topology, theta ), shaper = shaper )

	def verify( self_, solution, X, y ):
		accuracy = np.mean( 1.0 * ( y.flatten() == self_.predict( solution, X ) ) )
		return 1 - accuracy

	def predict( self_, solution, X ):
		X = solution.shaper().conform( X, addOnes = False )
		yp = predict_one_vs_all( X, solution.model() )
		yp = np.argmax( yp, axis = 1 )
		return solution.shaper().labels( yp )

	def examine( self_, solution, X ):
		X = solution.shaper().conform( X, addOnes = False )
		yp = predict_one_vs_all( X, solution.model() )
		return ma.label_confidence( solution.shaper(), yp )


import logging

from pyramid.view import view_config
from pyramid.httpexceptions import HTTPUnprocessableEntity, HTTPNotFound

from ziggurat_foundations.models.services.user import UserService
from appenlight.lib.utils import permission_tuple_to_dict
from appenlight.models.services.config import ConfigService
from appenlight.models.group import Group
from appenlight.models.services.group import GroupService
from appenlight.models.user import User
from appenlight.models import DBSession
from appenlight import forms
from webob.multidict import MultiDict

log = logging.getLogger(__name__)

_ = str

@view_config(
    route_name="groups_no_id",
    renderer="json",
    request_method="GET",
    permission="authenticated",
)
def groups_list(request):
    """
    Returns groups list
    """
    groups = Group.all().order_by(Group.group_name)
    list_groups = ConfigService.by_key_and_section(
        "list_groups_to_non_admins", "global"
    )
    if list_groups.value or request.has_permission("root_administration"):
        return [g.get_dict() for g in groups]
    else:
        return []

@view_config(
    route_name="groups_no_id",
    renderer="json",
    request_method="POST",
    permission="root_administration",
)
def groups_create(request):
    """
    Returns groups list
    """
    form = forms.GroupCreateForm(
        MultiDict(request.safe_json_body or {}), csrf_context=request
    )
    if form.validate():
        log.info("registering group")
        group = Group()
        # insert new group here
        DBSession.add(group)
        form.populate_obj(group)
        request.session.flash(_("Group created"))
        DBSession.flush()
        return group.get_dict(include_perms=True)
    else:
        return HTTPUnprocessableEntity(body=form.errors_json)

@view_config(
    route_name="groups",
    renderer="json",
    request_method="DELETE",
    permission="root_administration",
)
def groups_DELETE(request):
    """
    Removes a groups permanently from db
    """
    msg = _("You cannot remove administrator group from the system")
    group = GroupService.by_id(request.matchdict.get("group_id"))
    if group:
        if group.id == 1:
            request.session.flash(msg, "warning")
        else:
            DBSession.delete(group)
            request.session.flash(_("Group removed"))
            return True
    request.response.status = 422
    return False

@view_config(
    route_name="groups",
    renderer="json",
    request_method="GET",
    permission="root_administration",
)
@view_config(
    route_name="groups",
    renderer="json",
    request_method="PATCH",
    permission="root_administration",
)
def group_update(request):
    """
    Updates group object
    """
    group = GroupService.by_id(request.matchdict.get("group_id"))
    if not group:
        return HTTPNotFound()

    if request.method == "PATCH":
        form = forms.GroupCreateForm(
            MultiDict(request.unsafe_json_body), csrf_context=request
        )
        form._modified_group = group
        if form.validate():
            form.populate_obj(group)
        else:
            return HTTPUnprocessableEntity(body=form.errors_json)
    return group.get_dict(include_perms=True)

@view_config(
    route_name="groups_property",
    match_param="key=resource_permissions",
    renderer="json",
    permission="root_administration",
)
def groups_resource_permissions_list(request):
    """
    Get list of permissions assigned to specific resources
    """
    group = GroupService.by_id(request.matchdict.get("group_id"))
    if not group:
        return HTTPNotFound()
    return [
        permission_tuple_to_dict(perm)
        for perm in GroupService.resources_with_possible_perms(group)
    ]

@view_config(
    route_name="groups_property",
    match_param="key=users",
    request_method="GET",
    renderer="json",
    permission="root_administration",
)
def groups_users_list(request):
    """
    Get list of permissions assigned to specific resources
    """
    group = GroupService.by_id(request.matchdict.get("group_id"))
    if not group:
        return HTTPNotFound()
    props = [
        "user_name",
        "id",
        "first_name",
        "last_name",
        "email",
        "last_login_date",
        "status",
    ]
    users_dicts = []
    for user in group.users:
        u_dict = user.get_dict(include_keys=props)
        u_dict["gravatar_url"] = UserService.gravatar_url(user, s=20)
        users_dicts.append(u_dict)
    return users_dicts

@view_config(
    route_name="groups_property",
    match_param="key=users",
    request_method="DELETE",
    renderer="json",
    permission="root_administration",
)
def groups_users_remove(request):
    """
    Get list of permissions assigned to specific resources
    """
    group = GroupService.by_id(request.matchdict.get("group_id"))
    user = UserService.by_user_name(request.GET.get("user_name"))
    if not group or not user:
        return HTTPNotFound()
    if len(group.users) > 1:
        group.users.remove(user)
        msg = "User removed from group"
        request.session.flash(msg)
        group.member_count = group.users_dynamic.count()
        return True
    msg = "Administrator group needs to contain at least one user"
    request.session.flash(msg, "warning")
    return False

@view_config(
    route_name="groups_property",
    match_param="key=users",
    request_method="POST",
    renderer="json",
    permission="root_administration",
)
def groups_users_add(request):
    """
    Get list of permissions assigned to specific resources
    """
    group = GroupService.by_id(request.matchdict.get("group_id"))
    user = UserService.by_user_name(request.unsafe_json_body.get("user_name"))
    if not user:
        user = UserService.by_email(request.unsafe_json_body.get("user_name"))

    if not group or not user:
        return HTTPNotFound()
    if user not in group.users:
        group.users.append(user)
        group.member_count = group.users_dynamic.count()
    props = [
        "user_name",
        "id",
        "first_name",
        "last_name",
        "email",
        "last_login_date",
        "status",
    ]
    u_dict = user.get_dict(include_keys=props)
    u_dict["gravatar_url"] = UserService.gravatar_url(user, s=20)
    return u_dict