import os
import numpy as np
from utils.datasets import VOCDetection, MultiscaleConcatDataset
from eval import evaluate
from models import Darknet
from utils.utils import *
from utils.plot import draw_image_batch_with_targets
from tensorboardX import SummaryWriter
import json
from torch.utils.data import DataLoader, ConcatDataset
import torch
from tqdm import tqdm
import logging
import argparse
import shutil
import sys

def get_experiment_name(config):
    batch_size = config["train"]["batch_size"]
    grad_accum = config["train"]["gradient_accumulations"]
    learning_rate = config["train"]["learning_rate"]

    augment = "aug__" if config["train"]["augment"] else ""
    grad_clipping = "grad_clip__" if config["train"]["gradient_clipping"] else ""
    frozen_extractor = "frozen__" if config["train"]["freeze_feature_extractor"] else ""

    return f"exp__b_{batch_size}_grad_{grad_accum}" \
        + f"__lr_{learning_rate:.1e}".replace('.', ',') + \
        f"__{augment}{frozen_extractor}{grad_clipping}"

def get_grad_norm(model):
    total_norm = 0
    for p in model.parameters():
        if not p.requires_grad:
            continue
        param_norm = p.grad.data.norm(2)
        total_norm += param_norm.item() ** 2
    total_norm = total_norm ** (1. / 2)

    return total_norm

def prepare_dataloaders(config):

    train_datasets = list()
    for dataset_config in config['train']["datasets"]:

        train_dataset = VOCDetection(img_dir=dataset_config["image_folder"],
                                     annotation_dir=dataset_config["annot_folder"],
                                     cache_dir=dataset_config["cache_dir"],
                                     split_file=dataset_config['split_file'],
                                     img_size=config['model']['input_size'],
                                     filter_labels=config['model']['labels'],
                                     multiscale=True,
                                     augment=config['train']['augment'])
        train_dataset.name = dataset_config.get('name')
        train_datasets.append(train_dataset)

    train_concat_dataset = MultiscaleConcatDataset(train_datasets)
    train_loader = DataLoader(train_concat_dataset,
                              batch_size=config["train"]["batch_size"],
                              collate_fn=train_concat_dataset.collate_fn,
                              shuffle=True)

    if not config['val']['validate']:
        return train_loader

    else:
        val_datasets, val_loader_dict = list(), dict()
        for i, dataset_config in enumerate(config['val']["datasets"]):
            val_dataset = VOCDetection(img_dir=dataset_config["image_folder"],
                                       annotation_dir=dataset_config["annot_folder"],
                                       cache_dir=dataset_config["cache_dir"],
                                       split_file=dataset_config['split_file'],
                                       img_size=config['model']['input_size'],
                                       filter_labels=config['model']['labels'],
                                       multiscale=False,
                                       augment=False)
            val_dataset.name = dataset_config.get('name')
            val_datasets.append(val_dataset)

            val_concat_dataset = MultiscaleConcatDataset(val_datasets)
            val_loader = DataLoader(val_dataset,
                                    batch_size=config["val"]["batch_size"],
                                    collate_fn=val_dataset.collate_fn,
                                    shuffle=True)
            dataset_name = val_dataset.name if val_dataset.name else f"Dataset #{i}"
            val_loader_dict[dataset_name] = val_loader
        val_concat_loader = DataLoader(val_concat_dataset,
                            batch_size=config["val"]["batch_size"],
                            collate_fn=val_concat_dataset.collate_fn,
                            shuffle=True)

        return train_loader, val_concat_loader, val_loader_dict

# TODO: move somewhere
metrics = [
    "grid_size",
    "loss",
    "x",
    "y",
    "w",
    "h",
    "conf",
    "cls",
    "cls_acc",
    "recall50",
    "recall75",
    "precision",
    "conf_obj",
    "conf_noobj",
]

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--config_file", default="config/runs/config.json")
    parser.add_argument("--output_dir" , default='output')
    args = parser.parse_args()

    with open(args.config_file) as config_buffer:
        config = json.loads(config_buffer.read())

    exp_name = get_experiment_name(config)
    print(f"Experiment name: {exp_name}")
    out_dir = os.path.join(args.output_dir, exp_name)
    if os.path.exists(out_dir):
        print("experiment dir already exists! Removing...")
        shutil.rmtree(out_dir)

    os.makedirs(out_dir)

    log_dir = f"{out_dir}/logs"
    checkpoint_dir = f"{out_dir}/checkpoints"
    if not os.path.exists(checkpoint_dir):
        os.makedirs(checkpoint_dir)
    tb_logger = SummaryWriter(log_dir)

    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                        datefmt='%m/%d/%Y %H:%M:%S',
                        handlers=[
                            logging.FileHandler(f"{out_dir}/log.log"),
                            logging.StreamHandler(sys.stdout)
                        ],
                        level=logging.INFO)

    logger = logging.getLogger(__name__)
    logging.info("New session")

    seed = config["train"]["seed"]
    if seed > 0:
        np.random.seed(seed)
        torch.manual_seed(seed)

    ###############################
    #   Prepare data loaders
    ###############################
    print("Loading datasets...")
    if config['val']['validate']:
        train_loader, val_concat_loader, val_loader_dict = prepare_dataloaders(config)
    else:
        train_loader = prepare_dataloaders(config)
    print("Loaded!")
    if config["train"]["debug"]:
        image_batch, target = next(iter(train_loader))
        draw_image_batch_with_targets(image_batch[:4], target, cols=2)

        if config['val']['validate']:
            val_image_batch, val_target = next(iter(val_concat_loader))
            draw_image_batch_with_targets(val_image_batch[:4], val_target, cols=2)

    ###############################
    #   Construct the model
    ###############################

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    model = Darknet(config["model"]["config"]).to(device)
    model.apply(weights_init_normal)
    print("Model initialized!")

    if config["train"]["freeze_feature_extractor"]:
        model.freeze_feature_extractor()

    print(f"Trainable params: {get_trainable_params_num(model):,}")

    # If specified we start from checkpoint
    if config["model"]["pretrained_weights"]:
        if config["model"]["pretrained_weights"].endswith(".pth"):
            model.load_state_dict(torch.load(config["model"]["pretrained_weights"]))
        else:
            model.load_darknet_weights(config["model"]["pretrained_weights"])
            print("Pretrained weights loaded!")

    optimizer = torch.optim.Adam(model.parameters(),
                                 lr=config["train"]["learning_rate"])

    ###############################
    #   Training
    ###############################
    batches_done = 0
    grad_accumulations = config["train"]["gradient_accumulations"]
    save_every = config["train"]["save_every"]

    if config["val"]["validate"]:
        val_iterator = iter(val_concat_loader)

    for epoch in range(config["train"]["nb_epochs"]):

        effective_loss = 0
        loss_history = torch.zeros(len(train_loader))
        logger.info(f"Epoch {epoch} started!")
        bar = tqdm(train_loader)
        for i, (image_batch, bboxes) in enumerate(bar):
            model.train()
            image_batch = image_batch.to(device)
            bboxes = bboxes.to(device)

            loss, outputs = model(image_batch, bboxes)
            effective_loss += loss.item()
            loss_history[i] = loss.item()

            loss.backward()

            if i % grad_accumulations == 0:
                # Accumulates gradient before each step
                optimizer.step()

                if config["train"]["gradient_clipping"]:
                    torch.nn.utils.clip_grad_norm_(model.parameters(), 5)
                grad_norm = get_grad_norm(model)

                optimizer.zero_grad()
                if config["val"]["validate"]:
                    model.eval()

                    try:
                        val_image_batch, val_bboxes = next(val_iterator)
                    except StopIteration:
                        val_iterator = iter(val_concat_loader)
                        val_image_batch, val_bboxes = next(val_iterator)
                    val_image_batch = val_image_batch.to(device)
                    val_bboxes = val_bboxes.to(device)
                    with torch.no_grad():
                        val_loss, val_outputs = model(val_image_batch, val_bboxes)

                    tb_logger.add_scalar("loss/validation", val_loss, batches_done)

                bar.set_description(f"Loss: {effective_loss / grad_accumulations:.6f}")

                batches_done += 1

                # Tensorboard logging
                for metric_name in metrics:
                    metric_dict = {}
                    for j, yolo_layer in enumerate(model.yolo_layers):
                        metric_dict[f"yolo_{j}"] = yolo_layer.metrics[metric_name]

                    if metric_name == 'loss':
                        metric_dict["overall"] = loss.item()

                    tb_logger.add_scalars(metric_name, metric_dict, batches_done)
                tb_logger.add_scalar("grad_norm", grad_norm, batches_done)
                tb_logger.add_scalar("loss/effective_loss", effective_loss, batches_done)

                effective_loss = 0

                # save model
                if save_every > 0 and batches_done % save_every == 0:
                    torch.save(model.state_dict(), f"{checkpoint_dir}/yolov3_{batches_done}.pth")

        epoch_loss = loss_history.mean()
        print(f"Epoch loss: {epoch_loss}")
        tb_logger.add_scalar("epoch_loss", epoch_loss, epoch)

        if config["val"]["validate"]:
            result_dict = evaluate(model, val_loader_dict, config["val"])
            for name, results in result_dict.items():
                output_str = f"{name} evaluation results:\n" \
                    f"precision-{results['precision']},\n" \
                    f"recall-{results['recall']},\n" \
                    f"AP-{results['AP']},\n" \
                    f"F1-{results['F1']},\n" \
                    f"ap_class-{results['AP_class']}"
                logging.info(output_str)
                print(output_str)

                tb_logger.add_scalar(f"val_precision/{name}", results['precision'], epoch)
                tb_logger.add_scalar(f"val_recall/{name}", results['recall'], epoch)
                tb_logger.add_scalar(f"val_F1/{name}", results['F1'], epoch)
                tb_logger.add_scalar(f"val_AP/{name}", results['AP'], epoch)

        # save model
        torch.save(model.state_dict(), f"{checkpoint_dir}/yolov3_epoch_{epoch}.pth")

def log_model_validation(model, val_loader, val_config, tb_logger, epoch_num):
    precision, recall, AP, f1, ap_class = evaluate(model, val_loader, val_config)
    output_str = f"Evaluating results: precision-{precision}, recall-{recall}, AP-{AP}, F1-{f1}, ap_class-{ap_class}"
    logging.info(output_str)
    print(output_str)

    tb_logger.add_scalar("val_precision", precision, epoch)
    tb_logger.add_scalar("val_recall", recall, epoch)
    tb_logger.add_scalar("val_F1", f1, epoch)
    tb_logger.add_scalar("val_AP", AP, epoch)

if __name__ == '__main__':
    main()


import json

# Author: Chris Petrich
# Date: Apr 2018

# massively reduced version of the original version
#   maintaining original interface of save()

def _to_json(dct):
    """convert keys to str, py2 and py3"""
    if isinstance(dct, dict):
        return {str(key).replace(' ',''):dct[key] for key in dct}
    return dct

def _str2inttuple(s):
    """parse string as tuple of integers"""
    return tuple([int(v) for v in s.split('(',1)[1].split(')',1)[0].split(',')])

def _from_json(dct):
    """convert string keys back to tuples"""
    if not isinstance(dct, dict): return dct
    for key in dct:
        break
    if isinstance(key, str) and key.startswith('('):
        return {_str2inttuple(key): dct[key] for key in dct}
    return dct

def save(filename, head_module, data_dict):
    """Outputs only data_dict"""
    # NB: zipping this up could reduce space to 30%.
    # remove .py and add .json
    filename = filename[:-3] if filename.lower().endswith('.py') else filename
    filename = filename if filename.lower().endswith('.json') else filename+'.json'
    with open(filename,'w') as f:
        json.dump({outer_key:_to_json(data_dict[outer_key]) for outer_key in data_dict}, f, separators=(',', ':'))

def load(filename):
    # remove .py and add .json
    filename = filename[:-3] if filename.lower().endswith('.py') else filename
    filename = filename if filename.lower().endswith('.json') else filename+'.json'
    with open(filename) as f:
        data=json.load(f)
    return {'data':{key:_from_json(data[key]) for key in data}}


#import FaceLandmarksExtractor

import numpy as np
import dlib
import torch
import keras
from keras import backend as K
from keras import layers as KL
import math
import os
import time
import code

#channels_first BatchNorm2D https://github.com/keras-team/keras/issues/10648

class TorchBatchNorm2D(keras.engine.Layer):
    def __init__(self, axis=-1, momentum=0.99, epsilon=1e-3, **kwargs):
        super(TorchBatchNorm2D, self).__init__(**kwargs)
        self.supports_masking = True
        self.axis = axis
        self.momentum = momentum
        self.epsilon = epsilon

    def build(self, input_shape):
        dim = input_shape[self.axis]
        if dim is None:
            raise ValueError('Axis ' + str(self.axis) + ' of '
                             'input tensor should have a defined dimension '
                             'but the layer received an input with shape ' +
                             str(input_shape) + '.')
        shape = (dim,)
        self.gamma = self.add_weight(shape=shape, name='gamma', initializer='ones', regularizer=None, constraint=None)
        self.beta = self.add_weight(shape=shape, name='beta', initializer='zeros', regularizer=None, constraint=None)
        self.moving_mean = self.add_weight(shape=shape, name='moving_mean', initializer='zeros', trainable=False)
        self.moving_variance = self.add_weight(shape=shape, name='moving_variance', initializer='ones', trainable=False)
        self.built = True

    def call(self, inputs, training=None):
        input_shape = K.int_shape(inputs)

        broadcast_shape = [1] * len(input_shape)
        broadcast_shape[self.axis] = input_shape[self.axis]

        broadcast_moving_mean = K.reshape(self.moving_mean, broadcast_shape)
        broadcast_moving_variance = K.reshape(self.moving_variance, broadcast_shape)
        broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
        broadcast_beta = K.reshape(self.beta, broadcast_shape)
        invstd = K.ones (shape=broadcast_shape, dtype='float32') / K.sqrt(broadcast_moving_variance + K.constant(self.epsilon, dtype='float32'))

        return (inputs - broadcast_moving_mean) * invstd * broadcast_gamma + broadcast_beta

    def get_config(self):
        config = { 'axis': self.axis, 'momentum': self.momentum, 'epsilon': self.epsilon }
        base_config = super(TorchBatchNorm2D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

def t2kw_conv2d (src):
    if src.bias is not None:
        return [ np.moveaxis(src.weight.data.cpu().numpy(), [0,1,2,3], [3,2,0,1]), src.bias.data.cpu().numpy() ]
    else:
        return [ np.moveaxis(src.weight.data.cpu().numpy(), [0,1,2,3], [3,2,0,1])]

def t2kw_bn2d(src):
    return [ src.weight.data.cpu().numpy(), src.bias.data.cpu().numpy(), src.running_mean.cpu().numpy(), src.running_var.cpu().numpy() ]

import face_alignment
fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._2D,enable_cuda=False,enable_cudnn=False,use_cnn_face_detector=True).face_alignemnt_net
fa.eval()

def KerasConvBlock(in_planes, out_planes, input, srctorch):
    out1 = TorchBatchNorm2D(axis=1,  momentum=0.1, epsilon=1e-05, weights=t2kw_bn2d(srctorch.bn1) )(input)
    out1 = KL.Activation( keras.backend.relu ) (out1)
    out1 = KL.ZeroPadding2D(padding=(1, 1), data_format='channels_first')(out1)
    out1 = KL.convolutional.Conv2D( int(out_planes/2), kernel_size=3, strides=1, data_format='channels_first', padding='valid', use_bias = False, weights=t2kw_conv2d(srctorch.conv1) ) (out1)

    out2 = TorchBatchNorm2D(axis=1,  momentum=0.1, epsilon=1e-05, weights=t2kw_bn2d(srctorch.bn2) )(out1)
    out2 = KL.Activation( keras.backend.relu ) (out2)
    out2 = KL.ZeroPadding2D(padding=(1, 1), data_format='channels_first')(out2)
    out2 = KL.convolutional.Conv2D( int(out_planes/4), kernel_size=3, strides=1, data_format='channels_first', padding='valid', use_bias = False, weights=t2kw_conv2d(srctorch.conv2) ) (out2)

    out3 = TorchBatchNorm2D(axis=1,  momentum=0.1, epsilon=1e-05, weights=t2kw_bn2d(srctorch.bn3) )(out2)
    out3 = KL.Activation( keras.backend.relu ) (out3)
    out3 = KL.ZeroPadding2D(padding=(1, 1), data_format='channels_first')(out3)
    out3 = KL.convolutional.Conv2D( int(out_planes/4), kernel_size=3, strides=1, data_format='channels_first', padding='valid', use_bias = False, weights=t2kw_conv2d(srctorch.conv3) ) (out3)

    out3 = KL.Concatenate(axis=1)([out1, out2, out3])

    if in_planes != out_planes:
        downsample = TorchBatchNorm2D(axis=1,  momentum=0.1, epsilon=1e-05, weights=t2kw_bn2d(srctorch.downsample[0]) )(input)
        downsample = KL.Activation( keras.backend.relu ) (downsample)
        downsample = KL.convolutional.Conv2D( out_planes, kernel_size=1, strides=1, data_format='channels_first', padding='valid', use_bias = False, weights=t2kw_conv2d(srctorch.downsample[2]) ) (downsample)
        out3 = KL.add ( [out3, downsample] )
    else:
        out3 = KL.add ( [out3, input] )

    return out3

def KerasHourGlass (depth, input, srctorch):

    up1 = KerasConvBlock(256, 256, input, srctorch._modules['b1_%d' % (depth)])

    low1 = KL.AveragePooling2D (pool_size=2, strides=2, data_format='channels_first', padding='valid' )(input)
    low1 = KerasConvBlock (256, 256, low1, srctorch._modules['b2_%d' % (depth)])

    if depth > 1:
        low2 = KerasHourGlass (depth-1, low1, srctorch)
    else:
        low2 = KerasConvBlock(256, 256, low1, srctorch._modules['b2_plus_%d' % (depth)])

    low3 = KerasConvBlock(256, 256, low2, srctorch._modules['b3_%d' % (depth)])

    up2 = KL.UpSampling2D(size=2, data_format='channels_first') (low3)
    return KL.add ( [up1, up2] )

model_path = os.path.join( r'D:\DFLBuild\test\2DFAN-4.h5' )
if os.path.exists (model_path):
    t = time.time()
    model = keras.models.load_model (model_path, custom_objects={'TorchBatchNorm2D': TorchBatchNorm2D} )
    print ('load takes = %f' %( time.time() - t ) )
else:
    _input = keras.layers.Input ( shape=(3, 256,256) )
    x = KL.ZeroPadding2D(padding=(3, 3), data_format='channels_first')(_input)
    x = KL.convolutional.Conv2D( 64, kernel_size=7, strides=2, data_format='channels_first', padding='valid', weights=t2kw_conv2d(fa.conv1) ) (x)

    x = TorchBatchNorm2D(axis=1,  momentum=0.1, epsilon=1e-05, weights=t2kw_bn2d(fa.bn1) )(x)
    x = KL.Activation( keras.backend.relu ) (x)

    x = KerasConvBlock (64, 128, x, fa.conv2)
    x = KL.AveragePooling2D (pool_size=2, strides=2, data_format='channels_first', padding='valid' ) (x)
    x = KerasConvBlock (128, 128, x, fa.conv3)
    x = KerasConvBlock (128, 256, x, fa.conv4)

    outputs = []
    previous = x
    for i in range(4):
        ll = KerasHourGlass (4, previous, fa._modules['m%d' % (i) ])
        ll = KerasConvBlock (256,256, ll, fa._modules['top_m_%d' % (i)])

        ll = KL.convolutional.Conv2D(256, kernel_size=1, strides=1, data_format='channels_first', padding='valid', weights=t2kw_conv2d( fa._modules['conv_last%d' % (i)] ) ) (ll)
        ll = TorchBatchNorm2D(axis=1,  momentum=0.1, epsilon=1e-05, weights=t2kw_bn2d( fa._modules['bn_end%d' % (i)] ) )(ll)
        ll = KL.Activation( keras.backend.relu ) (ll)

        tmp_out = KL.convolutional.Conv2D(68, kernel_size=1, strides=1, data_format='channels_first', padding='valid', weights=t2kw_conv2d( fa._modules['l%d' % (i)] ) ) (ll)
        outputs.append(tmp_out)
        if i < 4 - 1:
            ll = KL.convolutional.Conv2D(256, kernel_size=1, strides=1, data_format='channels_first', padding='valid', weights=t2kw_conv2d( fa._modules['bl%d' % (i)] ) ) (ll)
            previous = KL.add ( [previous, ll, KL.convolutional.Conv2D(256, kernel_size=1, strides=1, data_format='channels_first', padding='valid', weights=t2kw_conv2d( fa._modules['al%d' % (i)] ) ) (tmp_out) ] )

    model = keras.models.Model (_input, outputs)
    model.compile ( loss='mse', optimizer='adam' )
    model.save (model_path)
    model.save_weights ( os.path.join( os.path.dirname(__file__) , 'weights.h5') )

    model_short = keras.models.Model (_input, outputs[0])
    model_short.compile ( loss='mse', optimizer='adam' )
    model_short.save ( os.path.join( os.path.dirname(__file__) , "2DFAN-4_light.h5" ) )
    model_short.save_weights ( os.path.join( os.path.dirname(__file__) , '_light_weights.h5') )

def transform(point, center, scale, resolution, invert=False):
    _pt = torch.ones(3)
    _pt[0] = point[0]
    _pt[1] = point[1]

    h = 200.0 * scale
    t = torch.eye(3)
    t[0, 0] = resolution / h
    t[1, 1] = resolution / h
    t[0, 2] = resolution * (-center[0] / h + 0.5)
    t[1, 2] = resolution * (-center[1] / h + 0.5)

    if invert:
        t = torch.inverse(t)

    new_point = (torch.matmul(t, _pt))[0:2]

    return new_point.int()

def get_preds_fromhm(hm, center=None, scale=None):
    max, idx = torch.max(  hm.view(hm.size(0), hm.size(1), hm.size(2) * hm.size(3)), 2)
    idx += 1
    preds = idx.view(idx.size(0), idx.size(1), 1).repeat(1, 1, 2).float()
    preds[..., 0].apply_(lambda x: (x - 1) % hm.size(3) + 1)
    preds[..., 1].add_(-1).div_(hm.size(2)).floor_().add_(1)

    for i in range(preds.size(0)):
        for j in range(preds.size(1)):
            hm_ = hm[i, j, :]
            pX, pY = int(preds[i, j, 0]) - 1, int(preds[i, j, 1]) - 1
            if pX > 0 and pX < 63 and pY > 0 and pY < 63:
                diff = torch.FloatTensor(
                    [hm_[pY, pX + 1] - hm_[pY, pX - 1],
                     hm_[pY + 1, pX] - hm_[pY - 1, pX]])
                preds[i, j].add_(diff.sign_().mul_(.25))

    preds.add_(-.5)

    preds_orig = torch.zeros(preds.size())
    if center is not None and scale is not None:
        for i in range(hm.size(0)):
            for j in range(hm.size(1)):
                preds_orig[i, j] = transform(
                    preds[i, j], center, scale, hm.size(2), True)

    return preds, preds_orig

def get_preds_fromhm2(a, center=None, scale=None):
    b = a.reshape ( (a.shape[0], a.shape[1]*a.shape[2]) )
    c = b.argmax(1).reshape ( (a.shape[0], 1) ).repeat(2, axis=1).astype(np.float)
    c[:,0] %= a.shape[2]
    c[:,1] = np.apply_along_axis ( lambda x: np.floor(x / a.shape[2]), 0, c[:,1] )

    for i in range(a.shape[0]):
        pX, pY = int(c[i,0]), int(c[i,1])
        if pX > 0 and pX < 63 and pY > 0 and pY < 63:
            diff = np.array ( [a[i,pY,pX+1]-a[i,pY,pX-1], a[i,pY+1,pX]-a[i,pY-1,pX]] )
            c[i] += np.sign(diff)*0.25

    c += 0.5
    result = np.empty ( (a.shape[0],2), dtype=np.int )
    if center is not None and scale is not None:
        for i in range(a.shape[0]):
            pt = np.array ( [c[i][0], c[i][1], 1.0] )
            h = 200.0 * scale
            m = np.eye(3)
            m[0,0] = a.shape[2] / h
            m[1,1] = a.shape[2] / h
            m[0,2] = a.shape[2] * ( -center[0] / h + 0.5 )
            m[1,2] = a.shape[2] * ( -center[1] / h + 0.5 )
            m = np.linalg.inv(m)
            result[i] = np.matmul (m, pt)[0:2].astype( np.int )
    return result

rnd_data = np.random.rand (3, 256,256).astype(np.float32)
#rnd_data = np.random.random_integers (2, size=(3, 256,256)).astype(np.float32)
#rnd_data = np.array ( [[[1]*256]*256]*3 , dtype=np.float32 )
input_data = np.array ([rnd_data])

fa_out_tensor = fa( torch.autograd.Variable( torch.from_numpy(input_data), volatile=True) )[-1].data.cpu()
fa_out = fa_out_tensor.numpy()

t = time.time()
m_out = model.predict ( input_data )[-1]
print ('predict takes = %f' %( time.time() - t ) )
t = time.time()

#fa_base_out = fa_base(torch.autograd.Variable( torch.from_numpy(input_data), volatile=True))[0].data.cpu().numpy()

print ( 'shapes = %s , %s , equal == %s ' % (fa_out.shape, m_out.shape, (fa_out.shape == m_out.shape) ) )
print ( 'allclose == %s' %  ( np.allclose(fa_out, m_out) ) )
print ( 'total abs diff outputs = %f' % ( np.sum ( np.abs(np.ndarray.flatten(fa_out-m_out))) ))

###
d = dlib.rectangle(156,364,424,765)

center = torch.FloatTensor(
                    [d.right() - (d.right() - d.left()) / 2.0, d.bottom() -
                     (d.bottom() - d.top()) / 2.0])
center[1] = center[1] - (d.bottom() - d.top()) * 0.12
scale = (d.right() - d.left() + d.bottom() - d.top()) / 195.0
pts, pts_img = get_preds_fromhm (fa_out_tensor, center, scale)
pts_img = pts_img.view(68, 2).numpy()

###

m_pts_img = get_preds_fromhm2 (m_out[0], center, scale)

print ('pts1 == pts2 == %s' % ( np.array_equal(pts_img, m_pts_img) ) )
import code
code.interact(local=dict(globals(), **locals()))

#print ( np.array_equal (fa_out, m_out) ) #>>> False
#code.interact(local=dict(globals(), **locals()))

#code.interact(local=locals())

#code.interact(local=locals())

###
#fa.conv1.weight = torch.nn.Parameter( torch.from_numpy ( np.array( [[[[1.0]*7]*7]*3]*64, dtype=np.float32) ) )
#fa.conv1.bias = torch.nn.Parameter( torch.from_numpy ( np.array( [1.0]*64, dtype=np.float32 ) ) )
#model.layers[2].set_weights( [ np.array( [[[[1.0]*64]*3]*7]*7, dtype=np.float32), np.array( [1.0]*64, dtype=np.float32 ) ] )

#b = np.array( [1.0]*64, dtype=np.float32 )
#b = np.random.rand (64).astype(np.float32)
#w = np.array( [[[[1.0]*7]*7]*3]*64, dtype=np.float32)
#w = np.random.rand (64, 3, 7, 7).astype(np.float32)
#s = w #fa_base.conv1.weight.data.cpu().numpy() #64x3x7x7
#d = np.moveaxis(s, [0,1,2,3], [3,2,0,1] )

#fa.conv1.weight = torch.nn.Parameter( torch.from_numpy ( w ) )
#fa.conv1.bias = torch.nn.Parameter( torch.from_numpy ( b ) )
#model.layers[2].set_weights( [np.transpose(w), b] )
#model.layers[2].set_weights( [d, b] )
'''
for i in range(0,64):
    for j in range(0,128):
        b = np.array_equal (fa_out[i,j], m_out[i,j])
        if b == False:
            print ( '%d %d == False' %(i,j) ) #>>> False
'''

'''
input = -2.7966828
gamma = 0.7640695571899414
beta = 0.22801123559474945
moving_mean = 0.12693816423416138
moving_variance = 0.10409101098775864
epsilon = 0.0 #0.00001

print ( gamma * (input - moving_mean) / math.sqrt(moving_variance + epsilon) + beta )
print ( (input - moving_mean) * (1.0 / math.sqrt(moving_variance) + epsilon)*gamma + beta   )
'''
#code.interact(local=dict(globals(), **locals()))
'''
conv_64_128 = x
conv_64_128 = TorchBatchNorm2D(axis=1,  momentum=0.1, epsilon=1e-05, weights=t2kw_bn2d(fa.conv2.bn1) )(conv_64_128)
conv_64_128 = KL.Activation( keras.backend.relu ) (conv_64_128)
conv_64_128 = KL.ZeroPadding2D(padding=(1, 1), data_format='channels_first')(conv_64_128)
conv_64_128 = KL.convolutional.Conv2D( 64, kernel_size=3, strides=1, data_format='channels_first', padding='valid', use_bias = False, weights=t2kw_conv2d(fa.conv2.conv1) ) (conv_64_128)
conv_64_128 = TorchBatchNorm2D(axis=1,  momentum=0.1, epsilon=1e-05, weights=t2kw_bn2d(fa.conv2.bn2) )(conv_64_128)
conv_64_128 = KL.Activation( keras.backend.relu ) (conv_64_128)
'''
#
#
#keras result = gamma * (input - moving_mean) / sqrt(moving_variance + epsilon) + beta
#
# (input - mean / scale_factor) / sqrt(var / scale_factor + eps)
#
#input = -3.0322433
#
#gamma = 0.1859646
#beta = -0.17041835
#moving_mean = -3.0345056
#moving_variance = 8.773307
#epsilon = 0.00001
#
#result = - 0.17027631
#
# fa result = 1.930317


from django.contrib.admin import AdminSite

class TestPlatAdminSite(AdminSite):

    def get_app_list(self, request):
        """
        Return a sorted list of all the installed apps that have been
        registered in this site.
        """
        app_ordering = {
            '测试平台': 1,
            'Celery结果': 2,
            '周期性任务': 3,
            '认证和授权': 4,
            '内容类型': 5
        }
        app_dict = self._build_app_dict(request)

        # Sort the apps alphabetically.
        app_list = sorted(app_dict.values(), key=lambda x: app_ordering[x["name"]])

        model_ordering = {
            '测试项目': 1,
            '接口定义': 2,
            '接口执行履历': 3,
            '用例': 4,
            '用例执行履历': 5,
            '用例套件': 6,
            '用例套件执行履历': 7,
            '测试批次(报告)': 8,
            '用例接口': 9,
            '项目成员': 10,
            '部署环境': 11,
        }
        # Sort the models alphabetically within each app.
        for app in app_list:
            if app['name'] == '测试平台':
                app["models"].sort(key=lambda x: model_ordering[x["name"]])
            else:
                app["models"].sort(key=lambda x: x["name"])
        return app_list


import cdsapi
import xarray as xr
import zarr
import numcodecs
import os
from huggingface_hub import HfApi
api = HfApi()
files = api.list_repo_files("openclimatefix/era5-reanalysis", repo_type="dataset")
data_files = [file for file in files if file.startswith("data/")]
c = cdsapi.Client()

for year in range(2022,2015,-1):
    for month in range(1,13):
        files = api.list_repo_files("openclimatefix/era5-reanalysis", repo_type="dataset")
        data_files = [file for file in files if file.startswith("data/")]
        for day in range(1,31):
            data_files = [file for file in files if file.startswith("data/")]
            if f"data/air/{year}/{str(month).zfill(2)}/{year}_{str(month).zfill(2)}_{str(day).zfill(2)}.zarr.zip" in data_files:
                continue
            try:
                c.retrieve(
                    'reanalysis-era5-pressure-levels',
                    {
                        'product_type': 'reanalysis',
                        'variable': [
                            'divergence', 'fraction_of_cloud_cover', 'geopotential',
                            'ozone_mass_mixing_ratio', 'potential_vorticity', 'relative_humidity',
                            'specific_cloud_ice_water_content', 'specific_cloud_liquid_water_content', 'specific_humidity',
                            'specific_rain_water_content', 'specific_snow_water_content', 'temperature',
                            'u_component_of_wind', 'v_component_of_wind', 'vertical_velocity',
                            'vorticity',
                        ],
                        'pressure_level': [
                            '1', '2', '3',
                            '5', '7', '10',
                            '20', '30', '50',
                            '70', '100', '125',
                            '150', '175', '200',
                            '225', '250', '300',
                            '350', '400', '450',
                            '500', '550', '600',
                            '650', '700', '750',
                            '775', '800', '825',
                            '850', '875', '900',
                            '925', '950', '975',
                            '1000',
                        ],
                        'year': str(year),
                        'month': str(month).zfill(2),
                        'day': str(day).zfill(2),
                        'time': [
                            '00:00', '01:00', '02:00',
                            '03:00', '04:00', '05:00',
                            '06:00', '07:00', '08:00',
                            '09:00', '10:00', '11:00',
                            '12:00', '13:00', '14:00',
                            '15:00', '16:00', '17:00',
                            '18:00', '19:00', '20:00',
                            '21:00', '22:00', '23:00',
                        ],
                        'format': 'netcdf',
                    },
                    f'download_air.nc')
                data = xr.open_dataset("/home/jacob/Downloads/download_air.nc", engine="netcdf4")
                print(data)
                encoding = {var: {"compressor": numcodecs.get_codec(dict(id="zlib", level=5))} for var in data.data_vars}
                if f"data/air/{year}/{str(month).zfill(2)}/{year}_{str(month).zfill(2)}_{str(day).zfill(2)}_{str(i).zfill(2)}.zarr.zip" in data_files:
                    continue
                d = data.chunk({"time": 1})
                with zarr.ZipStore(f'/run/media/jacob/Elements/{year}_{str(month).zfill(2)}_{str(day).zfill(2)}.zarr.zip', mode='w') as store:
                    d.to_zarr(store, encoding=encoding, compute=True)
                api.upload_file(
                    path_or_fileobj=f'/run/media/jacob/Elements/{year}_{str(month).zfill(2)}_{str(day).zfill(2)}.zarr.zip',
                    path_in_repo=f"data/air/{year}/{str(month).zfill(2)}/{year}{str(month).zfill(2)}{str(day).zfill(2)}.zarr.zip",
                    repo_id="openclimatefix/era5-reanalysis",
                    repo_type="dataset",
                    )
                os.remove(
                    f'/run/media/jacob/Elements/{year}_{str(month).zfill(2)}_{str(day).zfill(2)}.zarr.zip')
            except:
                continue


from __future__ import absolute_import, division, print_function, unicode_literals

import math, pprint, signal, sys, io
from datetime import datetime
from threading import Event

from flent import runners, transformers

from flent.util import classname

import collections

def new(settings):
    cname = classname(settings.AGGREGATOR, "Aggregator")
    if not cname in globals():
        raise RuntimeError("Aggregator not found: '%s'" % settings.AGGREGATOR)
    try:
        agg = globals()[cname](settings)
        for s in list(settings.DATA_SETS.items()):
            agg.add_instance(*s)
        return agg
    except Exception as e:
        raise RuntimeError("Error loading %s: %s." % (cname, e))

class GracefulShutdown(Exception):
    pass

def handle_usr1(signal, frame):
    raise GracefulShutdown()

class Aggregator(object):
    """Basic aggregator. Runs all jobs and returns their result."""

    def __init__(self, settings):
        self.instances = {}
        self.threads = {}
        self.settings = settings
        self.failed_runners = 0
        if self.settings.LOG_FILE is None:
            self.logfile = None
        else:
            self.logfile = io.open(self.settings.LOG_FILE, "at")

        self.postprocessors = []

    def add_instance(self, name, config):
        instance = dict(config)

        if not 'delay' in instance:
            instance['delay'] = 0

        instance['runner'] = runners.get(instance['runner'])
        instance['start_event'] = None
        instance['kill_event'] = None
        instance['finish_event'] = Event()

        if 'data_transform' in config:
            instance['transformers'] = []
            for t in [i.strip() for i in config['data_transform'].split(',')]:
                if hasattr(transformers, t):
                    instance['transformers'].append(getattr(transformers, t))

        self.instances[name] = instance

    def aggregate(self):
        raise NotImplementedError()

    def create_watchdog(self, name, instance):
        watchdog = runners.get('timer')("Timeout watchdog for '%s'" % name, None,
                                        start_event = instance['start_event'],
                                        kill_event = instance['kill_event'],
                                        finish_event = Event(),
                                        timeout = instance['kill_timeout'])
        instance['kill_event'] = watchdog.finish_event
        return watchdog

    def collect(self):
        """Create a ProcessRunner thread for each instance and start them. Wait
        for the threads to exit, then collect the results."""

        if self.logfile:
            self.logfile.write("Start run at %s\n" % datetime.now())
        signal.signal(signal.SIGUSR1, handle_usr1)

        result = {}
        metadata = {}
        raw_values = {}
        try:
            for n,i in list(self.instances.items()):
                watchdog = None
                if 'run_after' in i:
                    i['start_event'] = self.instances[i['run_after']]['finish_event']
                if 'kill_after' in i:
                    i['kill_event'] = self.instances[i['kill_after']]['finish_event']
                if 'kill_timeout' in i and i['kill_timeout']:
                    watchdog = self.create_watchdog(n, i)
                self.threads[n] = i['runner'](n, self.settings, **i)
                self.threads[n].start()
                if watchdog is not None:
                    self.threads[watchdog.name] = watchdog
                    watchdog.start()
            shutting_down = False
            for n,t in list(self.threads.items()):
                while t.isAlive():
                    try:
                        t.join(1)
                    except GracefulShutdown:
                        if not shutting_down:
                            sys.stderr.write("SIGUSR1 received; initiating graceful shutdown. "
                                             "This may take a while...\n")
                            self.kill_runners(graceful=True)
                            shutting_down = True
                        else:
                            sys.stderr.write("Already initiated graceful shutdown. Patience, please...\n")
                self._log(n,t)
                if t.result is None:
                    continue
                elif isinstance(t.result, collections.Callable):
                    # If the result is callable, the runner is really a
                    # post-processor (Avg etc), and should be run as such (by the
                    # postprocess() method)
                    self.postprocessors.append(t.result)
                elif hasattr(t.result, 'keys'):
                    if not t.result:
                        self.failed_runners += 1
                    for k in t.result.keys():
                        key = "%s::%s" % (n,k)
                        result[key] = t.result[k]
                        if key in self.instances and 'transformers' in self.instances[key]:
                            for tr in self.instances[key]['transformers']:
                                result[key] = tr(result[key])
                else:
                    if not t.result:
                        self.failed_runners += 1
                    result[n] = t.result
                    if 'transformers' in self.instances[n]:
                        for tr in self.instances[n]['transformers']:
                            result[n] = tr(result[n])

                if hasattr(t, 'metadata'):
                    metadata[n] = t.metadata
                    if 'transformers' in self.instances[n]:
                        for tr in self.instances[n]['transformers']:
                            for k,v in metadata[n].items():
                                try:
                                    metadata[n][k] = tr(v)
                                except:
                                    pass
                if hasattr(t, 'raw_values'):
                    raw_values[n] = t.raw_values
        except KeyboardInterrupt:
            self.kill_runners()
            raise

        if self.logfile is not None:
            self.logfile.write("Raw aggregated data:\n")
            formatted = pprint.pformat(result)
            if hasattr(formatted, 'decode'):
                formatted = formatted.decode()
            self.logfile.write(formatted)
        signal.signal(signal.SIGUSR1, signal.SIG_DFL)
        return result,metadata,raw_values

    def kill_runners(self, graceful=False):
        for t in list(self.threads.values()):
            t.kill(graceful)

    def postprocess(self, result):
        for p in self.postprocessors:
            result = p(result)
        return result

    def _log(self, name, runner):
        if self.logfile is None:
            return
        self.logfile.write("Runner: %s - %s\n" % (name, runner.__class__.__name__))
        self.logfile.write("Command: %s\nReturncode: %d\n" % (runner.command, runner.returncode))
        self.logfile.write("Program stdout:\n")
        self.logfile.write("  " + "\n  ".join(runner.out.splitlines()) + "\n")
        self.logfile.write("Program stderr:\n")
        self.logfile.write("  " + "\n  ".join(runner.err.splitlines()) + "\n")

class IterationAggregator(Aggregator):
    """Iteration aggregator. Runs the jobs multiple times and aggregates the
    results. Assumes each job outputs one value."""

    def __init__(self, *args, **kwargs):
        Aggregator.__init__(self, *args, **kwargs)
        self.iterations = self.settings.ITERATIONS

    def aggregate(self, results):
        for i in range(self.iterations):
            data,metadata,raw_values = self.collect()
            results.meta('SERIES_META', metadata)
            results.meta('FAILED_RUNNERS', self.failed_runners)
            results.raw_values = raw_values
            if i == 0:
                results.create_series(data.keys())
            results.append_datapoint(i, data)
        return results

class TimeseriesAggregator(Aggregator):
    """Time series aggregator. Runs the jobs (which are all assumed to output a
    series of timed entries) and combines the times onto a single timeline,
    aligning values to the same time steps (interpolating values as necessary).
    Assumes each job outputs a list of pairs (time, value) where the times and
    values are floating point values."""

    def __init__(self, *args, **kwargs):
        Aggregator.__init__(self, *args, **kwargs)
        self.step = self.settings.STEP_SIZE
        self.max_distance = self.step * 5.0

    def aggregate(self, results):
        measurements,metadata,raw_values = self.collect()
        if not measurements:
            raise RuntimeError("No data to aggregate. Run with -L and check log file to investigate.")
        results.meta('SERIES_META', metadata)
        results.meta('FAILED_RUNNERS', self.failed_runners)
        results.raw_values = raw_values
        results.create_series(list(measurements.keys()))

        # We start steps at the minimum time value, and do as many steps as are
        # necessary to get past the maximum time value with the selected step
        # size
        first_times = [i[0][0] for i in list(measurements.values()) if i and i[0]]
        last_times = [i[-1][0] for i in list(measurements.values()) if i and i[-1]]
        if not (first_times and last_times):
            raise RuntimeError("No data to aggregate. Run with -L and check log file to investigate.")
        t_0 = min(first_times)
        t_max = max(last_times)
        steps = int(math.ceil((t_max-t_0)/self.step))

        results.meta('T0', datetime.utcfromtimestamp(t_0))

        time_labels = []

        for s in range(steps):
            time_label = self.step*s
            t = t_0 + self.step*s

            # for each step we need to find the interpolated measurement value
            # at time t by interpolating between the nearest measurements before
            # and after t
            result = {}
            # n is the name of this measurement (from the config), r is the list
            # of measurement pairs (time,value)
            for n,r in list(measurements.items()):
                max_dist = self.max_distance
                last = False
                if not r:
                    continue
                t_prev = v_prev = None
                t_next = v_next = None

                for i in range(len(r)):
                    if r[i][0] > t:
                        if i > 0:
                            t_prev,v_prev = r[i-1]
                        else:
                            # minimum interpolation distance on first entry to
                            # avoid multiple interpolations to the same value
                            max_dist = self.step*0.5
                        t_next,v_next = r[i]
                        break
                if t_next is None:
                    t_next,v_next = r[-1]
                    last = True
                if abs(t-t_next) <= max_dist:
                    if t_prev is None:
                        # The first/last data point for this measurement is after the
                        # current t. Don't interpolate, just use the value.
                        if last and results.last_datapoint(n) in (v_next,None):
                            # Avoid repeating last interpolation
                            result[n] = None
                        else:
                            result[n] = v_next
                    else:
                        # We found the previous and next values; interpolate between
                        # them. We assume that the rate of change dv/dt is constant
                        # in the interval, and so can be calculated as
                        # (v_next-v_prev)/(t_next-t_prev). Then the value v_t at t
                        # can be calculated as v_t=v_prev + dv/dt*(t-t_prev)

                        dv_dt = (v_next-v_prev)/(t_next-t_prev)
                        result[n] = v_prev + dv_dt*(t-t_prev)
                else:
                    # Interpolation distance is too long; don't use the value.
                    result[n] = None

            results.append_datapoint(time_label, result)

        return results


import cx_Oracle
import time

def test_network_speed(connection_string, sql):
    try:
        connection = cx_Oracle.connect(connection_string)
        cursor = connection.cursor()

        start_time = time.time()
        cursor.execute(sql)
        data = cursor.fetchall()
        end_time = time.time()

        cursor.close()
        connection.close()

        elapsed_time = end_time - start_time
        return True, elapsed_time
    except Exception as e:
        return False, str(e)

if __name__ == "__main__":
    # 替换为您的数据库连接字符串和要执行的SELECT语句
    connection_string = "udms/admin@192.168.1.15:1521/demodb"
    sql = "SELECT * FROM (SELECT * FROM defect_region) WHERE ROWNUM <= 100000"

    success, elapsed_time = test_network_speed(connection_string, sql)

    if success:
        print(f"查询成功！执行时间：{elapsed_time:.4f} 秒")
    else:
        print(f"查询失败：{elapsed_time}")

    sql = "SELECT * FROM (SELECT * FROM edc_raw) WHERE ROWNUM <= 100000"

    success, elapsed_time = test_network_speed(connection_string, sql)

    if success:
        print(f"查询成功！执行时间：{elapsed_time:.4f} 秒")
    else:
        print(f"查询失败：{elapsed_time}")


import nltk
from nltk import word_tokenize
from nltk import FreqDist
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
import random
import sys

stopWords = set(stopwords.words('english'))#setting stop words

if len(sys.argv)<2:#checking for sysarg
	print("No file name provided")
	exit()

fileName = sys.argv[1]

f = open(fileName)#open and read file
yourList = f.read()

xx = word_tokenize(yourList)#tokenize
y = len(set(xx))#how many unique strings
z = len(xx)#how many tokens
g = y/z#calc lexical diversity
g= round(g,2)#round lexical diversity to two digits
print("Lexical diversity is %d", g)

def preprocessing(x):#preprocessing method
	lemmatizer = WordNetLemmatizer()
	zz = []
	for t in x:#make new list zz with lowercase, alpha, in stopwords, and length<5
		if t.isalpha() and t.islower() and t not in stopWords and len(t)>5:
			tt = lemmatizer.lemmatize(t)#lemmatize words whichc fit the criteria
			zz.append(tt)
	kk = list(set(zz))#only unique words
	acount = len(kk)#how many unique words
	kk = nltk.pos_tag(kk)#pos tag the unique words and get only the nouns
	#print(kk[0:20])
	Nlist = []
	hList = ["NN","NNS","NNP","NNPS"]
	for x in kk:
		if x[1] in hList:
			Nlist.append(x)
	dcount = len(Nlist)#how many nounds

	superList = []
	for x in Nlist:
		superList.append(x[0])#elimate pos, and only keep word

	print("Number of tokens from step A is "+ str(acount) + " and nouns from step D is " + str(dcount))
	return((zz,superList))#return as tuple
(a,b) = preprocessing(xx) #call preprocessing and unpack tuple

fdist = FreqDist(a)# find the frequency of the nouns
dicty = {}
for one, two in fdist.items():
	if one in b:
		dicty[one] = two#dict with keys being the unique nouns, and the values being the frequency
sorted_values = sorted(dicty.values())#sort and reverse the dict
sorted_values = sorted_values[::-1]
sorted_dicty = {}
for i in sorted_values:
    for k in dicty.keys():
        if dicty[k] == i:
            sorted_dicty[k] = dicty[k]
keys = sorted_dicty.keys()
keys = list(keys)
dictList = []
x = 0
while x<50:#print top 50 nouns
	dictList.append(keys[x])
	print(str(x+1) + ". " + str(keys[x]) + ": " + str(dicty[keys[x]]))
	x+=1

actualWord = random.choice(dictList)#get random word from top 50
placeholder = len(actualWord)*'_'#set placeholder with underscores

score = 5
guess = ""

while score > 0 and guess != "!" and "_" in placeholder:
	print(placeholder + ", score: " + str(score))#print status
	guess = input("Enter your guess: \n")
	hit = 0 #toggle to see if guess was sucessfull
	if guess != "!":#if they made a valid guess
		for x in range(len(actualWord)):#if the guess is correct, then replace all instances of that letter in the placeholder
			if guess == actualWord[x]:
				s = list(placeholder)
				s[x] = guess
				placeholder = "".join(s)
				hit = 1
	else:#messages to player after each round
		print("Game exited")
	if hit == 0:
		print("Sorry, guess again")
		score = score-1
	else:
		print("Right!!")
		score = score+1

print("Word was " + actualWord + ", score: " + str(score))


import os
import json
import tabulate as tbl
import tkinter.messagebox as mb
from constants import *

def add_to_basket(store_articles: 'list[list]', basket: 'list[list]', article_to_add: str):
    price = 0
    article_name = ''
    # Removing the article from the the store
    for article in store_articles:
        if article[0] == article_to_add:
            price = article[3]
            article_name = article[1]
            article[2] -= 1
            if article[2] == 0:
                store_articles.remove(article)

    # adding the article to the basket
    is_already_there = False
    for article in basket:
        if article[0] == article_to_add:
            article[2] += 1
            is_already_there = True
    if not is_already_there:
        basket.append([article_to_add, article_name, 1, price, price*1])

def remove_from_basket(store_articles: 'list[list]', basket: 'list[list]', article_to_remove: str):

    price = 0
    article_name = ''
    # removing article to the basket
    for article in basket:
        if article[0] == article_to_remove:
            price = article[3]
            article_name = article[1]
            article[2] -= 1
            if (article[2] == 0):
                basket.remove(article)

    # HERE: restoring the article to the store
    is_already_there = False
    for article in store_articles:
        if article[0] == article_to_remove:
            article[2] += 1
            is_already_there = True
    if not is_already_there:
        store_articles.append(
            [article_to_remove, article_name, 1, price])

def validate_purchase(basket: 'list[list]', name_of_the_client: str) -> str:
    clients: list[dict] = json.load(open('./data/clients.json', 'r'))
    # TODO: check if the client is already in the database
    # if true, check the number of visists and the articles puchases
    # if the number of purchases is >2 for an article, we do a reduction

    client: dict = {}
    # find the rig client:
    is_already_exist = False
    for cl in clients:
        if cl["name"] == name_of_the_client:
            is_already_exist = True
            cl['visits'] += 1
            client = cl
            cl['purchases'].append(basket)

    if not is_already_exist:
        client = {
            'name': name_of_the_client,
            'visits': 1,
            'purchases': [basket]
        }
        clients.append(client)

    "saving the client in the clients file"
    with open('./data/clients.json', 'w') as client_file:
        json.dump(clients, client_file)

    # Now we can process the bill by modifiying the basket content

    for new_purchase in basket:

        # calculate the total price
        new_purchase[4] = new_purchase[2] * \
            new_purchase[3]
        hasReduction = False
        if client['visits'] > 2:
            for old_purchases in list(client['purchases'][:-1]):

                for old_purchase in old_purchases:
                    if old_purchase[0] == new_purchase[0] and old_purchase[2] >= 2:
                        # we have to accord the reduction
                        hasReduction = True
                        new_purchase[4] *= 0.98
        if hasReduction:
            mb.showinfo(
                "Reductions😍", f"Felicitations vous beneficiez d'une reduction de 2% sur l'article {new_purchase[1]}")

    # Data that will be contained in a bill
    bill = {
        'client_name': name_of_the_client,
        'bill_path': generate_bill_path(name_of_the_client, client['visits']),
        'purchases': basket
    }
    # savaing the new bill in the invoices file, and
    invoices = json.load(open('./data/invoices.json', 'r'))
    invoices.append(bill)
    with open('./data/invoices.json', 'w') as invoices_file:
        json.dump(invoices, invoices_file)

    # generate the bill_file
    bill_writter(bill)
    basket = []
    return bill['bill_path']

def bill_writter(bill: dict) -> None:
    basket: list = bill['purchases']
    count, total = 0, 0
    for article in basket:
        count += article[2]
        total += article[4]
    basket.append(['TOTAL', '', count, '', total])
    headers = ['Code', 'Article',  'Quantité',
               'Prix Unitaire', 'Prix total']
    with open(f'{bill["bill_path"]}', 'w', encoding="utf-8") as bill_file:
        table = tbl.tabulate(basket, headers=headers,
                             tablefmt='fancy_grid', showindex=True)
        name = str(bill['client_name']).title()
        bill_file.write(f"Nom du client: {name}\n{table}\n\nMerci d'avoir fait vos achats chez {SHOP_NAME}")

def generate_bill_path(client_name: str, visits_nbr: int) -> str:
    base = './data/bills/'+client_name
    if os.path.exists(base):
        return base+f'/{client_name}{visits_nbr-1}.txt'
    os.mkdir(base)
    return base+f'/{client_name}.txt'


from selenium import webdriver
from selenium.common.exceptions import JavascriptException, ElementNotInteractableException
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.options import Options
import time
import bs4

def obtain_cookies_and_thread_meta(browser, keyword):
    browser.get('https://hackforums.net/member.php?action=login')

    time.sleep(7)    # wait for the possible identity check to finish

    username_elem = browser.find_element_by_name('username')
    username_elem.send_keys('Napoleon')

    password_elem = browser.find_element_by_name('password')
    password_elem.send_keys('LillyTheDog1!')
    password_elem.send_keys(Keys.RETURN)

    time.sleep(5)    # have to wait for the login cookies to be transferred

    cookies = __extract_login_cookies(browser.get_cookies())

    browser.get('https://hackforums.net/search.php')
    # searchElem = browser.find_elements_by_name('keywords')

    num_replies_elem = browser.find_element_by_name("numreplies")
    num_replies_elem.send_keys('30')

    search_elems = browser.find_elements_by_name('keywords')
    for search_elem in search_elems:
        try:
            search_elem.send_keys(keyword)
            search_elem.send_keys(Keys.RETURN)
            break    # do not attempt to find the others
        except ElementNotInteractableException:
            continue

    threads = []
    threads.append(__extract_threads_on_page(browser))
    while __next_page(browser):
        threads.append(__extract_threads_on_page(browser))

    threads = [item for sublist in threads for item in sublist]
    return (cookies, threads)

def __extract_threads_on_page(browser):
    time.sleep(3)    # wait for page to load

    # 1st page
    page_soup = bs4.BeautifulSoup(browser.page_source, 'html.parser')
    current_rows = page_soup.find_all("tr", {"class" : "inline_row"})

    threads_meta = []
    for row in current_rows:
        threads_meta.append(__get_thread_meta(row))

    return threads_meta

def __next_page(browser):
    try:
        next_button = browser.find_element_by_class_name('pagination_next')
        next_button.click()
        return True
    except:
        return False    # next button cannot be found or clicked, means on the last page

def __get_thread_meta(row_soup):
    info = row_soup.find_next('a', href=True)
    tid = info['href'].split('&')[0].split('=')[1]
    title = info.get_text()

    author = row_soup.find_next('div', {"class" : "author smalltext"}).get_text()

    reply_count = row_soup.find_next('td', {"class" : "trow1 mobile-remove"}).get_text()
    views = row_soup.find_next('td', {"class" : "trow1 mobile-remove"}).get_text()

    return (tid, author, title, reply_count, views)

def __extract_login_cookies(cookies_list):
    cookie_string = ''

    for cookie in cookies_list:
        cookie_string += cookie['name']
        cookie_string += '='
        cookie_string += cookie['value']
        cookie_string += '; '

    return cookie_string[:-2]    # remove the final '; '

if __name__ == '__main__':
    print('testing')
    # cookies = obtain_cookies_and_thread_meta('fortnite')
    # print(cookies)


from __future__ import print_function
import os.path
from os import path
import asyncio
import platform
import math
import sys
import re

from bleak import BleakClient, BleakScanner
from bleak.exc import BleakError

header = """#####################################################################
    ------------------------BLE OTA update---------------------
    Arduino code @ https://github.com/fbiego/ESP32_BLE_OTA_Arduino
#####################################################################"""

UART_SERVICE_UUID = "c78cfcf9-cdb1-4962-b69d-ac1c380c0001"
UART_RX_CHAR_UUID = "eac38323-5642-48ca-9d34-a316c2dc0002"
UART_TX_CHAR_UUID = "eac38323-5642-48ca-9d34-a316c2dc0003"

PART = 16000
MTU = 250

end = True
clt = None
fileBytes = None
total = 0
sent = 0
#device = None

def get_bytes_from_file(filename):
    print("Reading from: ", filename)
    return open(filename, "rb").read()

async def start_ota():
    device = None
    while device == None:
        devices = await BleakScanner.discover()
        for d in devices:
            if d.name == "InFlow":
                print("Connecting to InFlow")
                device = d

    #device = await BleakScanner.find_device_by_address(ble_address, timeout=20.0)
    disconnected_event = asyncio.Event()

    def handle_disconnect(_: BleakClient):
        global disconnect
        disconnect = False
        print(": Device disconnected")
        exit()
        disconnected_event.set()

    async def handle_rx(_: int, data: bytearray):
        print("handle rx: " + str(data))
        if (data[0] == 0xAA):
            print("Transfer mode:", data[1])
            printProgressBar(0, total, prefix = 'Progress:', suffix = 'Complete', length = 50)
            if data[1] == 1:
                for x in range(0, fileParts):
                    await send_part(x, fileBytes, clt)
                    printProgressBar(x + 1, total, prefix = 'Progress:', suffix = 'Complete', length = 50)
            else:
                await send_part(0, fileBytes, clt)

        if (data[0] == 0xF1):
            nxt = int.from_bytes(bytearray([data[1], data[2]]), "big")
            await send_part(nxt, fileBytes, clt)
            printProgressBar(nxt + 1, total, prefix = 'Progress:', suffix = 'Complete', length = 50)
        if (data[0] == 0xF2):
            ins = 'Installing firmware'
            #print("Installing firmware")
        if (data[0] == 0x0F):
            result = bytearray([])
            for s in range(1, len(data)):
                result.append(data[s])
            print("OTA result: ", str(result, 'utf-8'))
            global end
            end = False
        #print("received:", data)

    async def send_part(position: int, data: bytearray, client: BleakClient):
        start = (position * PART)
        end = (position + 1) * PART
        if len(data) < end:
            end = len(data)
        parts = (end - start) / MTU
        for i in range(0, int(parts)):
            toSend = bytearray()
            for y in range(0, MTU):
                toSend.append(data[(position*PART)+(MTU * i) + y])
            await send_data(client, toSend, False)
        if (end - start)%MTU != 0:
            rem = (end - start)%MTU
            toSend = bytearray()#([int(parts)])
            for y in range(0, rem):
                toSend.append(data[(position*PART)+(MTU * int(parts)) + y])
            await send_data(client, toSend, False)
        #update = bytearray([int((end - start)/256), int((end - start) % 256), int(position/256), int(position % 256) ])
        #await send_data(client, update, True)

    async def send_data(client: BleakClient, data: bytearray, response: bool):
        global sent
        print("sending " + str(len(data)))
        sent = sent + len(data)
        print("sent " + str(sent) + " total")
        await client.write_gatt_char(UART_TX_CHAR_UUID, data, response)
        await asyncio.sleep(0.2)

    if not device:
        print("-----------Failed--------------")
        print(f"Device with address {ble_address} could not be found.")
        return
        #raise BleakError(f"A device with address {ble_address} could not be found.")
    async with BleakClient(device, disconnected_callback=handle_disconnect) as client:
        #await client.start_notify(UART_RX_CHAR_UUID, handle_rx)
        await asyncio.sleep(1.0)

        #await send_data(client, bytearray([0xFD]), False)

        global fileBytes
        fileBytes = get_bytes_from_file("firmware.bin")
        global clt
        clt = client
        fileParts = math.ceil(len(fileBytes) / PART)
        fileLen = len(fileBytes)
        fileSize = bytearray([fileLen >>  24 & 0xFF, fileLen >>  16 & 0xFF, fileLen >>  8 & 0xFF, fileLen & 0xFF])
        #await send_data(client, fileSize, False)
        print("file size: " + str(fileLen))
        global total
        total = fileParts
        otaInfo = bytearray([int(fileParts/256), int(fileParts%256), int(MTU / 256), int(MTU%256) ])
        #await send_data(client, otaInfo, False)
        print("file parts: " + str(fileParts))
        for x in range(0, fileParts):
            await send_part(x, fileBytes, clt)

        print("All data sent")
        while end:
            await asyncio.sleep(1.0)
        print("Waiting for disconnect... ", end="")
        await disconnected_event.wait()
        print("-----------Complete--------------")

"""
ble_address = (
    "98:CD:AC:D3:6B:E2"
    if platform.system() != "Darwin"
    else "B9EA5233-37EF-4DD6-87A8-2A875E821C46"
)
"""

def isValidAddress(str):

    # Regex to check valid
    # MAC address
    regex = ("^([0-9A-Fa-f]{2}[:-])" +
             "{5}([0-9A-Fa-f]{2})|" +
             "([0-9a-fA-F]{4}\\." +
             "[0-9a-fA-F]{4}\\." +
             "[0-9a-fA-F]{4}){17}$")
    regex2 = "^[{]?[0-9a-fA-F]{8}" + "-([0-9a-fA-F]{4}-)" + "{3}[0-9a-fA-F]{12}[}]?$"

    # Compile the ReGex
    p = re.compile(regex)
    q = re.compile(regex2)

    # If the string is empty
    # return false
    if (str == None):
        return False

    # Return if the string
    # matched the ReGex
    if(re.search(p, str) and len(str) == 17):
        return True
    else:
        if (re.search(q, str) and len(str) == 36):
            return True
        else:
            return False

if __name__ == "__main__":
    print(header)
    print("Trying to start OTA update")
    asyncio.run(start_ota())


# -*- coding: utf-8 -*-
# @Time    : 2022/9/6 9:18 AM
# @Author  : Hush
# @Email   : crush@tju.edu.cn

import numpy as np
import matplotlib.pyplot as plt

class LinearRegression:
    def __init__(self, data, data_test):
        self.m = data.shape[0]
        self.cols = data.shape[1] + 1
        self.w = np.zeros(self.cols)
        self.b = np.ones(self.m).reshape(self.m, 1)
        self.lr = 0.000016
        self.train_X = np.hstack([self.b, data])

        self.m_test = data_test.shape[0]
        self.b_test = np.ones(self.m_test).reshape(self.m_test, 1)
        self.test_X = np.hstack([self.b_test, data_test])

    def fit(self, X, Y, X_test, Y_test, epochs):
        i = 0
        loss_record = []
        mae_record = []
        mae_test_record = []
        for i in range(epochs):
            ## predict
            y_hat = self.predict(X)

            ## compute loss
            loss = (np.sum(np.square(Y - y_hat)) / (2 * self.m))
            loss_record.append(loss)

            ## compute gradiant
            grad = (np.dot(X.T, (y_hat - Y)))

            ## update weight----gradient_descent
            self.w = self.w - self.lr * grad

            mae = (np.sum(np.abs(y_hat - Y)) / self.m)
            mae_record.append(mae)

            ## test each epoch
            mae_test = self.test(X_test, Y_test)
            mae_test_record.append(mae_test)
        self.plot_history(loss_record, mae_record, mae_test_record, epochs)
        return loss_record, mae_record, mae_test_record

    def predict(self, X):
        return np.dot(X, self.w)

    def test(self, X, Y):
        mae_test = (np.sum(np.abs(self.predict(X) - Y)) / self.m_test)
        return mae_test

    def plot_history(self, loss_record, acc_record, acc_test_record, epochs):
        plt.figure(figsize=(10, 5))
        plt.subplot(2, 2, 1)
        plt.xlabel('Epoch')
        plt.ylabel('loss')
        plt.plot(range(epochs), loss_record, label='loss')
        plt.legend()
        plt.subplot(2, 2, 2)
        plt.xlabel('Epoch')
        plt.ylabel('mae_train')
        plt.plot(range(epochs), acc_record, label='acc_train')
        plt.subplot(2, 1, 2)
        plt.xlabel('Epoch')
        plt.ylabel('mae_test')
        plt.plot(range(epochs), acc_test_record, label='acc_test', color='red')
        plt.show()


#!/usr/bin/env python3
import sys
import glob
import json
import cv2
import numpy as np

from dataset import resize_image, scale_image, get_name_with_uid
from eval import predict
import config as cfg

def main(picture):
    face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
    img = cv2.imread(picture)

    pic_w, pic_h, pic_d = img.shape

    # img = scale_image(img, 1.5)

    # cv2.imshow('source', img)
    # cv2.waitKey(0)
    # cv2.destroyAllWindows()

    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    faces = face_cascade.detectMultiScale(
        gray,
        scaleFactor = 1.2,
        minNeighbors = 5,
        minSize = (10, 10),
        flags = cv2.CASCADE_SCALE_IMAGE
    )

    face = np.array([], dtype = 'uint8')

    print(len(faces))

    if len(faces) > 0:
        print('Reading model...')
        recognizer = cv2.face.LBPHFaceRecognizer_create()
        recognizer.read(cfg.MODEL_PATH)

    for (x, y, w, h) in faces:
        cv2.rectangle(img, (x - 2, y - 2), (x + w + 2, y + h + 5), (255, 0, 0), 5)
        predict_id, percent = predict(recognizer, gray[y : y + h, x : x + w])
        name = get_name_with_uid(predict_id)
        cv2.putText(img, name, (x, y + h), 0, 0.5, (255, 255, 0), 1, cv2.LINE_AA)

    cv2.imwrite('image.jpg', img)
    # cv2.waitKey(0)
    # cv2.destroyAllWindows()

def show_video():
    face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
    video = cv2.VideoCapture(0)

    recognizer = cv2.face.LBPHFaceRecognizer_create()
    recognizer.read(cfg.MODEL_PATH)

    while True:
        ret, img = video.read()
        if not ret:
            break
        img = cv2.flip(img, 1)
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        faces = face_cascade.detectMultiScale(
            gray,
            scaleFactor = 1.2,
            minNeighbors = 5,
            minSize = (100, 100),
            flags = cv2.CASCADE_SCALE_IMAGE
        )

        for (x, y, w, h) in faces:
            cv2.rectangle(img, (x - 2, y - 2), (x + w + 2, y + h + 5), (255, 255, 0), 2)
            predict_id, percent = predict(recognizer, gray[y : y + h, x : x + w])
            name = get_name_with_uid(predict_id)
            cv2.putText(img, name, (x, y + h), 0, 0.5, (255, 255, 255), 1, cv2.LINE_AA)

        cv2.imshow('video', img)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

if __name__ == '__main__':
    if len(sys.argv) == 2:
        main(sys.argv[1])
    elif len(sys.argv) == 1:
        show_video()
    else:
        print("ERROR")


from __future__ import annotations
from typing import Iterator, Mapping, Any, List, Optional, Callable

from spil import Sid, Getter, Finder
from spil.conf import default_path_config, path_data_suffix  # type: ignore

class GetByFinder(Getter):
    """
    Some Getters first use a Finder to retrieve the Sids,
    then, in a second distinct step, retrieve the data.
    For example, GetFromPaths which stores data in json "sidecar" files.

    This "GetByFinder" is an abstract Getter, implementing shared methods for Getters that use a Finder.

    These Getters can inherit GetByFinder, and:
    - override the constructor and define self.finder
    - or pass it to the constructor (inherited from GetByFinder)
    - and implement get_data()
    """

    def __init__(self, finder: Finder):
        self.finder = finder

    def get(
        self,
        search_sid: str | Sid,
        attributes: Optional[List[str]] = None,
        sid_encode: Callable = str,
    ) -> Iterator[Mapping[str, Any]]:
        """
        See Getter.
        """
        for sid in self.finder.find(search_sid=search_sid, as_sid=True):
            data = self.get_data(sid, attributes=attributes, sid_encode=sid_encode) or {}
            yield data

    def do_get(
        self,
        search_sids: List[Sid],
        attributes: Optional[List[str]] = None,
        sid_encode: Callable = str,
    ) -> Iterator[Mapping[str, Any]]:
        """
        See Getter.
        """
        for sid in self.finder.do_find(search_sids=search_sids, as_sid=True):
            data = self.get_data(sid, attributes=attributes, sid_encode=sid_encode) or {}
            yield data

    def get_data(
        self, sid: str | Sid, attributes: Optional[List[str]] = None, sid_encode: Callable = str
    ) -> Mapping[str, Any]:
        """
        See Getter.
        """
        raise NotImplementedError("To be used, GetByFinder should be inherited, and get_data overriden.")

    def __str__(self):
        return f'[spil.{self.__class__.__name__} -- Finder: "{self.finder}"]'

if __name__ == "__main__":
    from spil import FindInPaths

    finder = FindInPaths()
    print(GetByFinder(finder))


import numpy as np, pylab as pl, pandas as pd
from PIL import Image, ImageTk
from scipy.ndimage.interpolation import zoom as szoom
from matplotlib import animation
import sys, operator, os, threading, time, tifffile

from .roi import ROI, ROIView
from .images import Tiff, AVI, HDF5
from .series import Series
from .config import *

if PF_pyver == 3:
    import tkinter as tk
elif PF_pyver == 2:
    import Tkinter as tk

try:
    import pyqtgraph as pg
except ImportError:
    pg = None

class Movie(np.ndarray):
    """An object for *in-memory* storage of n sequential 2d images along with a sampling rate

    Parameters
    ----------
    data : array-like, str-like
        input data, see below
    Ts : [optional] float
        sampling period
    filename : str
        path to filename for data

    Input data can be supplied in multiple ways:
    (1) as a numpy array of shape (n,y,x)
    (2) a string [or list thereof] corresponding to a file path to a tiff/avi/hdf5

    The data in Movie objects is stored following the standard pyfluo convention in which the 0th axis corresponds to time. For example, movie[0] corresponds to the movie frame at time 0.
    """

    __array_priority__ = 1. #ensures that ufuncs return ROI class instead of np.ndarrays
    _custom_attrs = ['Ts', 'filename']

    def __new__(cls, data, Ts=1, filename=[], **kwargs):
        assert isinstance(data, PF_list_types+PF_str_types), 'Movie data not supplied in proper format.'

        tf = None
        # if data is filenames
        if type(data) in PF_str_types:
            filename = data
            suf = os.path.splitext(filename)[-1]

            if suf == '.avi':
                data = AVI(data, **kwargs).data
            elif suf in ['.h5','.hdf5']:
                hs = HDF5(data, **kwargs)
                data = h.data
                Ts = h.Ts #overwrites any supplied Ts with hdf5 file's stored time info
            elif suf in ['.tif','.tiff']:
                tf = Tiff(data)
                data = tf.data
                Ts = tf.Ts

        # convert to a np array (either of strings or data itself)
        data = np.asarray(data)
        obj = data.view(cls)

        obj.Ts = Ts
        obj.filename = filename
        obj.tif_obj = tf

        return obj

    def __array_finalize__(self, obj):
        if obj is None:
            return

        for ca in Movie._custom_attrs:
            setattr(self, ca, getattr(obj, ca, None))

    def project(self, axis=0, method=np.mean, show=False, roi=None, backend=pl, **kwargs):
        """Flatten/project the movie data across one or many axes

        Parameters
        ----------
        axis : int, list
            axis/axes over which to flatten
        method : def
            function to apply across the specified axes
        show : bool
            display the result (if 2d, as image; if 1d, as trace)
        roi : pyfluo.ROI
            roi to display
        backend : module
            module used for interactive display (only matplotlib currently supported)

        Returns
        -------
        The projected image
        """
        if method == None:
            method = np.mean
        pro = np.apply_over_axes(method,self,axes=axis).squeeze()

        if show:
            ax = pl.gca()
            ax.margins(0.)
            if pro.ndim == 2:
                pl.imshow(pro, cmap=pl.cm.Greys_r, **kwargs)
                if roi is not None:
                    roi.show()

        return pro
    def play(self, method='opencv', **kwargs):
        """Play the movie

        Parameters
        ----------
        method : str
            'pyqtgraph', 'tkinter', 'opencv'

        """
        if method == 'opencv':
            return play_mov(self)
        elif method == 'tkinter' or (method == 'pyqtgraph' and pg is None):
            print('Try installing PyQtGraph for another good interactive media player.')
            return MoviePlayer(self, Ts=self.Ts, **kwargs)
        elif method == 'pyqtgraph':
            im = pg.image(self)
            im.fps = int(np.round(1/self.Ts))
            im.play()
            pg.QtGui.QApplication.exec_()
            return im

    def select_roi(self, *args, **kwargs):
        """Select ROI using ROIView

        Parameters
        ----------
        projection_method : def
            'method' parameter for Movie.project, used in display
        *args, **kwargs
            those accepted by pyfluo.roi.ROIView
        """
        zp = self.project(show=False, method=kwargs.pop('projection_method',np.mean))
        rv = ROIView(zp)
        return rv
    def extract(self, roi):
        """Extract a time series consisting of one value for each movie frame, attained by performing an operation over the regions of interest (ROI) supplied

        Parameters
        ----------
        roi : pyfluo.ROI
            the rois over which to extract data

        Returns
        -------
        pyfluo.Series object, with multiple columns corresponding to multiple ROIs

        Note: my gut says np.einsum should be able to further optimize this, yet all benchmarks I've run so far say this is the fastest
        """
        if roi.ndim == 3:
            flatshape = (len(roi),-1)
            roi_norm = roi.astype(float) / roi.sum(axis=(1,2))[:,None,None]
        elif roi.ndim == 2:
            flatshape = -1
            roi_norm = roi.astype(float) / roi.sum()
        roi_norm = roi_norm.reshape(flatshape)
        if self.ndim == 2:
            self_flat = np.array([self])
        else:
            self_flat = self
        self_flat = self_flat.reshape((len(self_flat),-1)).T
        dp = (roi_norm.dot(self_flat)).T

        return Series(dp, Ts=self.Ts)

    def resize(self, factor, order=0):
        """Resize movie using scipy.ndimage.zoom

        Parameters
        ----------
        factor : float, tuple, list
            multiplying factor for dimensions (y,x), ex. 0.5 will downsample by 2. If number, it is used for all dimensions
        order : int
            order of interpolation (0=nearest, 1=bilinear, 2=cubic)
        """
        if isinstance(factor, PF_numeric_types):
            factor = [factor,factor]
        elif isinstance(factor, PF_list_types):
            factor = list(factor)
        else:
            raise Exception('factor parameter not understood')
        res = szoom(self, [1]+factor, order=order)
        res = self.__class__(res, Ts=self.Ts)
        for ca in self._custom_attrs:
            res.__setattr__(ca, self.__getattribute__(ca))
        return res

    def resample(self, n=4):
        """Downsample movie by taking mean of every n frames
        """
        if n==1:
            return self

        if len(self)%n == 0:
            working = self
        else:
            working = self[:-(len(self)%n)]

        result = np.nanmean(working.reshape((-1,n,working.shape[1],working.shape[2])), axis=1)
        result.Ts = self.Ts*n
        return result

    def save(self, filename, fmt=None, codec='MJPG', fps=None, minmax=(0,300)):
        """Save movie for playback in video player

        Note that this function is intended for saving the movie in an avi/tiff-like format. Saving for further analysis should be performed using pyfluo's io functions.

        Parameters
        ----------
        filename : str
            destination file name, with or without extension
        fmt : str
            format to save movie, specified as file extension, ex. 'avi', 'mp4', 'tif'
        codec : int
            opencv fourcc code, as chars to be fed to cv2.FOURCC ex. 'IYUV'
        fps : int
            frames per second. Defaults to obj.fs
            applies to only to formats tht store time info, like avi
        minmax : tuple
            min and max pixels values corresponding to black and white. clipping occurs on either end
            if None: min and max value of movie are used
        """
        if fps == None:
            fps = 1/self.Ts

        if '.' in filename:
            filename, fmt = os.path.splitext(filename)
            fmt = fmt.strip('.')

        filename += '.'+fmt

        if fmt in ['tif','tiff']:
            tifffile.imsave(filename, data=np.asarray(self))
        elif fmt in ['mp4', 'avi']:
            if CV_VERSION >= 3:
                codec = cv2.VideoWriter_fourcc(*codec)
            elif CV_VERSION == 2:
                codec = cv2.cv.FOURCC(*codec)

            if minmax is None:
                minn,maxx = np.min(self),np.max(self)
            else:
                minn,maxx = minmax
            data = np.clip(self, minn, maxx)
            data = data - minn
            data = data / maxx-minn
            data = (255*data).astype(np.uint8)
            y,x = data[0].shape
            vw = cv2.VideoWriter(filename, codec, fps, (x,y), isColor=True)
            for d in data:
                vw.write(cv2.cvtColor(d, cv2.COLOR_GRAY2BGR))
            vw.release()

###################
def play_mov(data, loop=True, fps=None, minmax=(0,300), scale=1, show_time=True, fontsize=1, rolling_mean=1, generator_fxn=None, **kwargs):
    """An opencv-based movie playing function, with keyboard controls

    This works, but has the following disadvantages:
    -requires opencv
    -is blocking

    Parameters
    ----------
    data : see description
        any iterable object where the iteration axis represents time and items of the iterable are 2d matrices, and which has the following methods/properties:
            Ts : sample rate
            shape : (z,y,x)
            min() : minimum value
            max() : maximum value
    loop : bool
        repeat playback upon finishing
    fps : float
        playback rate in frames per second. Defaults to 1/Ts
    scale : float
        scaling factor to resize playback images
    show_time : bool
        show time on image
    fontsize : float
        to display time
    rolling_mean : int
        number of frames to avg in display
    generator_fxn : str
        name of function to call to retrieve generator for object. if None, object is considered iterable
        if using this parameter, the data parameter should be an object that implements generator_fxn

    Playback controls (window must have focus):
        'f' : faster
        's' : slower
        'p' : pause / continue
        'r' : reverse
        '=' : zoom in
        '-' : zoom out
        'q' : quit

    """
    if fps==None:
        fps = 1/data.Ts
    fpms = fps / 1000.

    title = 'p / q / f / s / r / b / d / = / -'
    cv2.namedWindow(title)
    minn,maxx = minmax
    #minn,maxx = data.min(),data.max() # not needed anymore with new contrast method

    if generator_fxn is not None:
        gfunc = getattr(data,generator_fxn)
        dataiter = gfunc()
    else:
        gfunc = None
        dataiter = data

    global current_idx
    global oper_idx
    oper_idx = 0 # add/subtract
    current_idx = 0

    def fnext(d):
        global current_idx
        global oper_idx
        opfunc = [operator.add, operator.sub][oper_idx]
        current_idx = opfunc(current_idx, rolling_mean)
        try:
            return next(d)
        except TypeError:
            if current_idx>=len(data):
                current_idx = len(data)-1
            return data[current_idx]

    wait = 1./fpms
    paused = False
    while True:
        if not paused:

            # get frame
            size = tuple((scale*np.array(data.shape)[-1:0:-1]).astype(int))
            font_size = scale * fontsize * min(data[0].shape)/450.
            t = data.Ts*current_idx

            fr = np.mean([fnext(dataiter) for k in range(rolling_mean)], axis=0)

            fr = np.clip(fr, minn, maxx)
            fr -= minn
            fr /= maxx-minn
            fr = cv2.resize(fr,size)
            if show_time:
                cv2.putText(fr, '{:0.3f}'.format(t), (5,int(30*font_size)), cv2.FONT_HERSHEY_SIMPLEX, font_size, (120,100,80), thickness=1)
            cv2.imshow(title,fr)

            # update indices
            if current_idx+rolling_mean >= data.shape[0]:
                if not loop:
                    break
                elif loop:
                    current_idx = 0
                    if gfunc is not None:
                        dataiter = gfunc()

        # user input
        k=cv2.waitKey(int(wait))
        if k == ord('q'):
            break
        elif k == ord('p'):
            paused = not paused
        elif k == ord('f'):
            wait = wait/1.5
            wait = max([wait, 1])
        elif k == ord('s'):
            wait = wait*1.5
        elif k == ord('b'):
            minn -= 20 #brighter
        elif k == ord('d'):
            minn += 20 #dimmer
        elif k == ord('r'):
            oper_idx = int(not oper_idx)
        elif k == ord('='):
            scale = scale * 1.5
        elif k == ord('-'):
            scale = scale / 1.5

    cv2.destroyWindow(title); cv2.waitKey(1)

class MoviePlayer(tk.Frame):
    """A movie player built on Tkinter with the goal of circumventing the need for blocking in the opencv implementation

    This is able to play movies at high framerate, but lacks some of the convenience live controls
    """

    def __init__(self, data, Ts=1, zoom=None):
        # Data handling
        self.data = data

        if self.data.dtype!=np.uint8 or self.data.min()!=0 or self.data.max()!=255:
            minn,maxx = self.data.min(),self.data.max()
            self.data = (255*((self.data-minn)/(maxx-minn))).astype(np.uint8)

        if zoom is not None:
            size = tuple((zoom*np.array(self.data.shape)[-1:0:-1]).astype(int))
            self.data = np.array([cv2.resize(i,size) for i in self.data])

        self.n, self.h, self.w = self.data.shape
        self.imshape = self.data.shape[1:][::-1]
        self.i = 0
        self.Ts = Ts

        if tk._default_root is not None:
            self.root = tk._default_root
            self.master = tk.Toplevel(self.root)
            self.made_root = False
        else:
            self.root = tk.Tk()
            self.master = self.root
            self.made_root = True

        window_size = [self.w, self.h]
        tk.Frame.__init__(self, master=self.master, width=window_size[0], height=window_size[1])
        self.bind("<Destroy>", self.end)

        # Tk setup
        self.canvas = tk.Canvas(self, width=self.w, height=self.h)
        # Populate frame
        self.canvas.place(relx=0,rely=0,x=0,y=0)
        self.pack()

        # Bindings
        self.root.bind('f', self.faster)
        self.root.bind('s', self.slower)
        self.root.bind('r', self.reverse)

        # Run events
        self._inc = 1
        self.recall = None
        self.ended = False
        self.t0 = time.clock()
        self.update_interval = 1 # seconds
        self.after(0, self.step)
        self.mainloop()

    def faster(self, *args):
        self.Ts /= 1.5

    def slower(self, *args):
        self.Ts *= 1.5

    def reverse(self, *args):
        self._inc = -self._inc

    def inc(self):
        n = self._inc
        self.i += n
        over = self.n - self.i
        if over <= 0:
            self.i = abs(over)
        if self.i < 0:
            self.i = self.n + self.i

    def step(self):
        if self.ended:
            return
        self.im = Image.fromarray(self.data[self.i])
        self.inc()
        self.photo = ImageTk.PhotoImage(image=self.im) # 1-3 msec
        self.canvas.create_image(0, 0, image=self.photo, anchor=tk.NW) # 1-5 msec
        if time.clock() - self.t0 > self.update_interval:
            self.update() # 20-40 msec
        self.recall = self.after(int(self.Ts*1000),self.step)

    def end(self, *args):
        self.ended = True
        if self.made_root:
            self.root.quit()
        else:
            self.master.destroy()
            self.root.quit() # no clue why this doesnt kill mpl windows but its okay


import os
import sys
import csv
import json
import pathlib
from progressbar import ProgressBar

class TwitterManager:
    tweets_list=[]
    tweet_tokens=[]
    tweet_list_dict={}
    tweet_token_dict={}
    root_path = pathlib.Path(__file__).parents[1]
    def __init__(self,data_file_dir,name=None):
        self.load_data(data_file_dir,name)

    def create_data_structure(self,this_files):
        file_names=[]
        data_structure={}
        for item in this_files:
           name=item[:item.rindex('_')]
           if name not in file_names:
               file_names.append(name)
               data_structure[name] = {}
               data_structure[name]['after'] = []
               data_structure[name]['before'] = []
        return data_structure

    def load_data(self,data_file_dir,name):
        this_file_dict={}
        this_tokens_dict={}
        files=[]
        for (dirpath, dirnames, filenames) in os.walk(data_file_dir):
            files.extend(filenames)

        if name is not None:
            this_files = [i for i in files if i.startswith(name)]
        else:
            this_files = files

        self.tweet_list_dict=self.create_data_structure(this_files)
        self.tweet_token_dict=self.create_data_structure(this_files)

        pbar=ProgressBar()
        print("Reading files....")
        for csv_file in pbar(this_files):
            this_tweets=[]
            this_tweet_tokens = []
            this_file_name=csv_file[:csv_file.rindex('_')]
            seqs=csv_file.split('_')
            seq=seqs[len(seqs)-1]
            this_file_seq=seq.split('.')[0]
            this_tweets=self.create_tweets_list(os.path.join(data_file_dir,csv_file))
            self.tweet_list_dict[this_file_name][this_file_seq]=this_tweets
            this_tweet_tokens=self.tokenize(this_tweets)
            self.tweet_token_dict[this_file_name][this_file_seq] = this_tweet_tokens

    def create_tweets_list(self,data_file):
        self.tweet_tokens=[]
        self.tweets_list=[]
        with open(data_file,encoding="utf8") as csv_file:
            csv_reader=csv.DictReader(csv_file,delimiter=',')
            line_count=0
            for row in csv_reader:
                self.tweets_list.append(row["text"])
        return self.tweets_list

    def get_tweets(self,name=None,seq=None):
        if name is None and seq is None:
            return self.tweet_list_dict
        if name is not None and seq is None:
            return self.tweet_list_dict[name]
        if name is not None and seq is not None:
            return self.tweet_list_dict[name][seq]

    def get_tweet_tokens(self,name=None,seq=None):
        if name is None and seq is None:
            return self.tweet_token_dict
        if name is not None and seq is None:
            return self.tweet_token_dict[name]
        if name is not None and seq is not None:
            return self.tweet_token_dict[name][seq]

    def tokenize(self,tweets_list):
        for tweet in tweets_list:
            self.tweet_tokens.append(tweet.split())
        return self.tweet_tokens

    def dump_to_json(self,dir,file_name,data):
        print('Writing data to json for ',file_name)
        with open(os.path.join(self.root_path,dir,file_name), 'w') as outfile:
            json.dump(data, outfile)

def main():
    cwd=os.getcwd()
    obj = TwitterManager('C:\\Users\\SIDDHARTHA\\Trinity\\TextAnalysis\\TextAnalysis\\GetOldTweets3\\testdatasets')
    #tweets_list=obj.get_tweets_list()
    #print(len(tweets_list))
    #tweets_tokens=obj.tokenize(tweets_list)

if __name__ == '__main__':
    main()


"""Utilities for analysis
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import warnings

from collections import Counter
from collections import defaultdict
import sys

columns = [
    'ORF_ID', 'ORF_type', 'status', 'phase_score', 'read_count', 'length',
    'valid_codons', 'transcript_id', 'transcript_type', 'gene_id',
    'gene_name', 'gene_type', 'chrom', 'strand', 'start_codon', 'profile\n'
]
_, orf_f, index_f = sys.argv
annotated = {}
with open(index_f, 'r') as fin:
    header = True
    annotated = defaultdict(int)
    for line in fin:
        if header:
            header = False
            continue
        fields = line.strip().split('\t')
        oid = fields[0]
        otype = fields[1]
        # status = fields[2]
        # phase_score = fields[3]
        # read_count = int(fields[4])
        # length = fields[5]
        # valid_codons = fields[6]
        # tid = fields[7]
        # ttype = fields[8]
        # gid = fields[9]
        # gname = fields[10]
        # gtype = fields[11]
        # chrom = fields[12]
        # strand = fields[13]
        # start_codon = fields[14]
        annotated[oid] = fields[4]
converted = set()
with open(orf_f, 'r') as fin:
    header = True
    for line in fin:
        if header:
            header = False
            continue
        fields = line.strip().split()
        oid = fields[0]
        converted.add(annotated[oid])
with open('uniq_genes', 'w') as fout:
    for g in converted:
        fout.write('{}\n'.format(g))


import pygame as pg
import game_page
import controller
import fighter
import tools

def main():
    pg.init()
    pg.display.set_mode((1024, 512))

    fighter0 = fighter.fighter(tools.get_fighter('zao_nan'))
    fighter1 = fighter.fighter(tools.get_fighter('za_bing'))
    controller0 = controller.controller()
    controller0.add_fighter(fighter0, 0, 0)
    controller0.add_fighter(fighter1, 1, 0)
    page0 = game_page.page()
    page0.controller = controller0

    page0.start()

if __name__ == '__main__':
    main()