import numpy as np
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation

'''
Ideje kako poboljšati kod:
1. Kreirati novu klasu satelit koja je pod gravitacijskim utjecajem ostalih planeta ali ona ne utječe na njih
2. Ta klasa ima metodu boost koja ju odjednom ubrza
3. Pogledati i probati dolje zakomentiranu metodu za optimizaciju
4. Dodati tragove orbita
'''

class Body(object):
    def __init__(self, name, colour, mass, init_position, init_velocity):
        self.name = name
        self.colour = colour
        self.mass = mass
        self.position = init_position
        self.velocity = init_velocity
        self.current_acceleration = 0  # TODO: think if better initial acceleration can be found
        self.previous_acceleration = 0

    def update_position(self, timestep):
        """Updates the position of the body"""
        new_position = self.position + self.velocity * timestep + 1 / 6 * (4 * self.current_acceleration
                                                                           - self.previous_acceleration) * timestep ** 2
        self.position = new_position

    def update_velocity(self, new_acceleration, timestep):
        """New acceleration is the acceleration in the next timestep. Updates the body velocity"""
        new_velocity = self.velocity + 1 / 6 * (2 * new_acceleration + 5 * self.current_acceleration
                                                - self.previous_acceleration) * timestep
        self.velocity = new_velocity

    def calc_KE(self):
        """Returns the kinetic energy of the body"""
        KE = 1 / 2 * self.mass * np.linalg.norm(self.velocity) ** 2
        return KE

class SmallBody(Body):
    def __init__(self, name, colour, mass, init_position, init_velocity, bodyOfInterest, bodyOfInterestPosition):
        super().__init__(name, colour, mass, init_position, init_velocity)
        self.closestDistance = np.linalg.norm(bodyOfInterestPosition - init_position)
        self.timeToBodyOfInterest = 0
        self.bodyOfInterest = bodyOfInterest

def distance_from_body(body1, body2):
    distancevec = body1.position - body2.position
    distance = np.linalg.norm(distancevec)
    return distance

class Simulation(object):
    def __init__(self, timestep, num_iterations):
        self.timestep = timestep
        self.num_iterations = num_iterations
        self.patches = []
        self.timeElapsed = 0

    def read_input_data(self, file):
        """Opens the specific file and reads the input data. File has to be in a specific format"""
        df = open(file, 'r')
        data = df.readlines()
        data.pop(0)  # pop the first two lines of the file, they describe how the file is to be formated
        data.pop(0)
        bodies = []
        smallBodies = []
        for line in data:
            line = line.strip()
            line = line.split(',')
            line[3] = line[3].split(';')
            line[4] = line[4].split(';')
            name, color, mass, init_position, init_velocity = line[0].strip(), line[1].strip(), float(line[2]), \
                                                              np.array([float(line[3][0].strip()),
                                                                        float(line[3][1].strip())]), \
                                                              np.array([float(line[4][0].strip()),
                                                                        float(line[4][1].strip())])
            if line[-1].strip() == 'Body':
                bodies.append(Body(name, color, mass, init_position, init_velocity))
            elif line[-1].strip() == 'SmallBody':
                bodyOfInterest = line[-2].strip()
                for body in bodies:
                    if body.name == bodyOfInterest:
                        bodyOfInterestPosition = body.position
                smallBodies.append(SmallBody(name, color, mass, init_position, init_velocity, bodyOfInterest,
                                             bodyOfInterestPosition))
        self.body_list = bodies
        self.smallBodies = smallBodies

        for body in self.body_list:
            # create patches for each body of the system
            xpos = body.position[0]
            ypos = body.position[1]
            if body.name == 'Sun':
                self.patches.append(plt.Circle((xpos, ypos), radius=10000000000, color=body.colour, animated=True))
            else:
                for i in range(10):
                    self.patches.append(
                        plt.Circle((xpos, ypos), radius=(5000000000 / (10 - i)), color=body.colour, animated=True))

        for smallBody in self.smallBodies:
            xpos = smallBody.position[0]
            ypos = smallBody.position[1]
            for i in range(10):
                self.patches.append(
                    plt.Circle((xpos, ypos), radius=(5000000000 / (10 - i)), color=smallBody.colour, animated=True)
                )

        xmax = 0
        for body in self.body_list:
            # find the axes range
            if body.position[0] > xmax:
                xmax = body.position[0] * 1.5
            if body.position[1] > xmax:
                xmax = body.position[1] * 1.5

        self.xmax = xmax

    def run_simulation(self):
        # running the simulation for the inputed number of iterations
        for i in range(self.num_iterations):
            self.step_forward()

    def step_forward(self):
        # Move the bodies one timestep
        # New positions of all the bodies are calculated first
        self.timeElapsed += self.timestep
        for body in self.body_list:
            body.update_position(self.timestep)

        for smallBody in self.smallBodies:
            smallBody.update_position(self.timestep)

        for body in self.body_list:
            new_acceleration = self.calc_acceleration(body)
            body.update_velocity(new_acceleration, self.timestep)
            body.previous_acceleration = body.current_acceleration
            body.current_acceleration = new_acceleration

        for smallBody in self.smallBodies:
            new_acceleration = self.calc_acceleration(smallBody)
            smallBody.update_velocity(new_acceleration, self.timestep)
            smallBody.previous_acceleration = smallBody.current_acceleration
            smallBody.current_acceleration = new_acceleration
            for body in self.body_list:
                if smallBody.bodyOfInterest == body.name:
                    distance = distance_from_body(smallBody, body)
                    if distance < smallBody.closestDistance:
                        smallBody.closestDistance = distance
                        smallBody.timeToBodyOfInterest = self.timeElapsed

    def calc_acceleration(self, body):
        # find the acceleration on a single body. Returns a np array of acceleration
        forceOnBody = np.array([0.0, 0.0])
        for secondBody in self.body_list:
            if secondBody.name != body.name:
                displacementVec = secondBody.position - body.position
                distance = np.linalg.norm(displacementVec)
                displacementVec = displacementVec / distance

                magnitude = G * body.mass * secondBody.mass / (distance ** 2)
                force = magnitude * displacementVec

                forceOnBody += force

        acceleration = forceOnBody / body.mass
        return acceleration

    def update_display(self, i):
        # single timestep change in display
        self.step_forward()

        j = 0
        for body in self.body_list:
            if body.name == 'Sun':
                self.patches[j].center = (body.position[0], body.position[1])
            else:
                for i in range(1, 10):
                    self.patches[(j - 1) * 10 + i].center = self.patches[(j - 1) * 10 + i + 1].center
                self.patches[j * 10].center = (body.position[0], body.position[1])
            j += 1

        for smallBody in self.smallBodies:
            for i in range(1, 10):
                self.patches[(j - 1) * 10 + i].center = self.patches[(j - 1) * 10 + i + 1].center
            self.patches[j * 10].center = (smallBody.position[0], smallBody.position[1])
            j += 1
        return self.patches

    def animate(self):
        # animate the bodies for the duration of the simulation
        plt.style.use('dark_background')
        fig = plt.figure(1)
        ax = plt.axes()

        for patch in self.patches:
            ax.add_patch(patch)

        ax.axis('scaled')
        ax.set_xlim(-self.xmax, self.xmax)
        ax.set_ylim(-self.xmax, self.xmax)
        ax.set_xlabel('x')
        ax.set_ylabel('y')
        plt.axis('off')

        anim = FuncAnimation(fig, self.update_display, frames=self.num_iterations, repeat=False, interval=50, blit=True)

        plt.show()

    def calc_PE(self):
        # Calculates the total potential energy. Returns a float of the energy
        PE = 0
        for body in self.body_list:
            for secondBody in self.body_list:
                if body.name != secondBody.name:
                    displacementVec = secondBody.position - body.position
                    distance = np.linalg.norm(displacementVec)

                    PE += -1 / 2 * G * body.mass * secondBody.mass / distance
        return PE

    def calc_tot_energy(self):
        # Calculates the total energy. Returns a float
        PE = self.calc_PE
        KE = 0
        for body in self.body_list:
            KE += body.calc_KE()
        return KE + PE

    def check_orbital_period(self, body):
        # Finds the orbital period of a given body using trigonometric functions. Returns a float
        orbital_period = 0
        while not 0 > np.arctan2(body.position[1], body.position[0]) > -0.01:
            self.step_forward()
            orbital_period += self.timestep
        orbital_period = orbital_period / 86400
        return orbital_period

    def launch_sattelite(self, name, colour, mass, launchBodyName, radius, initVelocity, launchOrientation,
                         interestBody):
        """This is a function that launches a satellite from a given body. Input parameters are name, colour, mass,
        name of the body from which the satellite is to be launched, distance from the center of the body where the
        satellite is launched, initial velocity, orientation of the launch - inner if launching from the side facing the
        Sun, outer otherwise, and name of the body the satellite is trying to reach"""
        for body in self.body_list:
            if interestBody == body.name:
                interestBodyName = body.name
                interestBodyPosition = body.position
            if launchBodyName == body.name:
                launchBody = body
        xBodyPos = launchBody.position[0]
        yBodyPos = launchBody.position[1]
        angle = np.arctan2(yBodyPos, xBodyPos)
        if launchOrientation == 'inner':
            xOffset = -1 * np.tan(angle) * radius
            yOffset = -1 * np.tan(angle) * radius
            launchPosition = np.array([xBodyPos + xOffset, yBodyPos + yOffset])
            self.smallBodies.append(SmallBody(name, colour, mass, launchPosition, initVelocity, interestBodyName,
                                              interestBodyPosition))
            for i in range(10):
                self.patches.append(
                    plt.Circle((launchPosition[0], launchPosition[1]), radius=(2000000000 / (10 - i)),
                               color=colour, animated=True))
        elif launchOrientation == 'outer':
            xOffset = np.tan(angle) * radius
            yOffset = np.tan(angle) * radius
            launchPosition = np.array([xBodyPos + xOffset, yBodyPos + yOffset])
            self.smallBodies.append(SmallBody(name, colour, mass, launchPosition, initVelocity, interestBodyName,
                                              interestBodyPosition))
            for i in range(10):
                self.patches.append(
                    plt.Circle((launchPosition[0], launchPosition[1]), radius=(2000000000 / (10 - i)),
                               color=colour, animated=True))

G = 6.67408e-11

Universe = Simulation(200000, 2000)
Universe.read_input_data('Parameters.txt')
Universe.animate()
# print('Orbital period of the Earth is: ' + str(Universe.check_orbital_period(Universe.body_list[3])) + ' days')
# print(str(Universe.calc_tot_energy()))
# Universe.run_simulation()


import os
from bot.misc.util import download, calculate_hash
from bot.functions import lessonsToday
from bot.database.main import filesDB
import hashlib
import datetime

files = filesDB()

def filesCheck(urls) -> list:
    done = []
    filesHash = []
    for name, url in urls.items():
        h = files.get(name)
        if h == -1:
            download(url, name)
            rez = calculate_hash(f'{name}')
            filesHash.append({'name': name, 'hash': rez, 'date': datetime.datetime.now()})
        else:
            done.append({'file_id': h['file_id'],
                         'url': url})
    return done, filesHash


import cv2
import numpy as np

video_path = '/Users/bigphess/Desktop/omnidirection/res/rabbit_250fps.mp4'
cap = cv2.VideoCapture(0)
cap2 = cv2.VideoCapture(video_path)

while True:
	ret, frame = cap2.read()
	# image = cv2.imread('/Users/bigphess/Downloads/IMG_6453.JPG')

	debug = frame
	if cv2.waitKey(100) & 0xFF == ord('q'):
		cv2.imwrite('/Users/bigphess/Desktop/xu.jpg',debug)
		print("successful write")
		break

	# h,w,ch = debug.shape

	print('the size of the frame is {} and {}'.format(debug.shape[1],debug.shape[0]))
	cv2.imshow('??', frame)
	cv2.waitKey(0)
	break

	# debug = cv2.logPolar(frame,(debug.shape[0]/2,debug.shape[1]/2),100,cv2.WARP_FILL_OUTLIERS)

	# cv2.waitKey(1)


import re
import sys

import numpy as np
import pandas as pd
from sklearn.base import TransformerMixin
from sklearn.compose import ColumnTransformer
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import OneHotEncoder

class PreprocessDataframe(TransformerMixin):
    def fit(self, df, _):
        # Drop random/singular columns
        # self.mask = [bool(pd.Series(col).dropna().size) for col, s in X.T]
        self.drop_cols = df.columns[df.isna().all(axis=0)]
        print('Drop cols', self.drop_cols, file=sys.stderr)
        return self

    def transform(self, df: pd.DataFrame):
        assert (df.iloc[:, 0].str.count(' ') == 2).all()
        df = df.drop(columns=df.columns[0])  # process pid, container id/image

        return df.drop(columns=self.drop_cols)

class PreprocessText(TransformerMixin):
    fit = lambda x, *_: x

    def _fix(self, s):
        s = re.sub(r'(?<![.:\d])[a-f\d\[\]]+(:+[a-f\d\[\]]+)+|\d+([.:-]+\d+){2,4}', ' ', s)  # IP:port
        return s

    assert not _fix(None, '''
        123.123.123.123:1234
        123:[fab:12e:123:123]:1234
    ''').strip()

    def transform(self, X):
        Xt = [self._fix(s) for s in X]
        return Xt

class Transformer(TransformerMixin):
    CATEGORICAL_WEIGHT = 30

    def fit(self, df, _):
        transformers = []
        for i, (_, s) in enumerate(df.items()):
            n_words = s.fillna('').astype(str).str.count(r'\b\w+')
            assert n_words.mean() > 0
            if n_words.mean() <= 1:
                print(i, n_words.mean(), np.unique(n_words), sep='\t')
                transformers.append((
                    str(i),
                    OneHotEncoder(handle_unknown='ignore', dtype=np.int8),
                    [i] * self.CATEGORICAL_WEIGHT,
                ))
            else:
                n_features = round(max(5, min(50, n_words.median() * 5)))
                print(i, f'{n_words.mean():.2f}', n_features, sep='\t')
                transformers.append((
                    str(i),
                    make_pipeline(
                        PreprocessText(),
                        HashingVectorizer(
                            n_features=n_features,
                            token_pattern=r'[>\w-]+',
                            ngram_range=(1, 2),
                            lowercase=False,
                            alternate_sign=False,
                            norm=None,
                            dtype=np.uint8,
                        ),
                    ),
                    i
                ))
        self.transformer = ColumnTransformer(
            transformers=transformers,
            # n_jobs=-1,
        )
        X = df.fillna('').astype(str).values
        self.transformer.fit(X)
        return self

    def transform(self, df):
        X = df.fillna('').astype(str).values
        Xt = self.transformer.transform(X)
        assert Xt.size, Xt
        return Xt


import time
from torch import optim
from torch.utils.data import DataLoader
from torchvision import datasets, transforms
from ..models import *
from ..utils import AverageMeter, calculate_accuracy, Logger, MyDataset
from visdom import Visdom

DatasetsList = ['CIFAR10', 'CIFAR100']

ModelList = {'AlexNet': AlexNet, 'alexnet': alexnet,
             'DenseNet': DenseNet, 'densenet121': densenet121, 'densenet169': densenet169,
             'densenet201': densenet201, 'densenet161': densenet161,
             'Inception3': Inception3, 'inception_v3': inception_v3,
             'ResNet': ResNet, 'resnet18': resnet18, 'resnet34': resnet34,
             'resnet50': resnet50, 'resnet101': resnet101, 'resnet152': resnet152,
             'SqueezeNet': SqueezeNet, 'squeezenet1_0': squeezenet1_0, 'squeezenet1_1': squeezenet1_1,
             'VGG': VGG, 'vgg11': vgg11, 'vgg11_bn': vgg11_bn, 'vgg13': vgg13, 'vgg13_bn': vgg13_bn,
             'vgg16': vgg16, 'vgg16_bn': vgg16_bn, 'vgg19_bn': vgg19_bn, 'vgg19': vgg19,
             'se_resnet18': se_resnet18, 'se_resnet34': se_resnet34, 'se_resnet50': se_resnet50,
             'se_resnet101': se_resnet101, 'se_resnet152': se_resnet152,
             'hr18_net': hr18_net,
             'mobilenetv3': mobilenetv3,
             'shufflenetv2': shufflenetv2}

class TrainPipline(object):
    def __init__(self, opt):
        self.root_path = opt['path']['root_path']
        self.result_path = os.path.join(self.root_path, opt['path']['result_path'])
        self.datasets_path = os.path.join(self.root_path, opt['path']['datasest_path'])

        self.n_classes = opt['model']['n_classes']
        self.momentum = opt['model']['momentum']
        self.weight_decay = opt['model']['weight_decay']
        self.nesterov = opt['model']['nesterov']

        self.n_epochs = opt['train']['n_epochs']
        self.batch_size = opt['train']['batch_size']
        self.learning_rate = opt['train']['learning_rate']
        self.n_threads = opt['train']['n_threads']
        self.checkpoint = opt['train']['checkpoint']

        self.no_cuda = opt['cuda']['no_cuda']
        self.model_name = ''
        self.model_ft = ''

        self.visdom_log_file = os.path.join(self.result_path, 'log_files', 'visdom.log')
        self.vis = Visdom(port=8097,
                          log_to_filename=self.visdom_log_file,
                          env='myTest_1')

        self.vis_loss_opts = {'xlabel': 'epoch',
                              'ylabel': 'loss',
                              'title': 'losses',
                              'legend': ['train_loss', 'val_loss']}

        self.vis_tpr_opts = {'xlabel': 'epoch',
                             'ylabel': 'tpr',
                             'title': 'val_tpr',
                             'legend': ['tpr@fpr10-2', 'tpr@fpr10-3', 'tpr@fpr10-4']}

        self.vis_epochloss_opts = {'xlabel': 'epoch',
                                   'ylabel': 'loss',
                                   'title': 'epoch_losses',
                                   'legend': ['train_loss', 'val_loss']}

    def datasets(self, data_name=None):
        assert data_name in DatasetsList

        if data_name == 'CIFAR10':
            training_data = datasets.CIFAR10(root='./modelzoo/datasets/', train=True, download=False,
                                             transform=transforms.Compose([
                                                       # transforms.RandomResizedCrop(224),
                                                       transforms.Pad(96),
                                                       transforms.ToTensor(),
                                                       transforms.Normalize((0.1307,), (0.3081,))]))

            val_data = datasets.CIFAR10(root='./modelzoo/datasets/', train=False, download=False,
                                        transform=transforms.Compose([
                                                  # transforms.RandomResizedCrop(224),
                                                  transforms.Pad(96),
                                                  transforms.ToTensor(),
                                                  transforms.Normalize((0.1307,), (0.3081,))]))

        elif data_name == 'CIFAR100':
            training_data = datasets.CIFAR100(root='./modelzoo/datasets/', train=True, download=True,
                                              transform=transforms.Compose([
                                                        # transforms.RandomResizedCrop(224),
                                                        transforms.Pad(96),
                                                        transforms.ToTensor(),
                                                        transforms.Normalize((0.1307,), (0.3081,))]))

            val_data = datasets.CIFAR100(root='./modelzoo/datasets/', train=False, download=True,
                                         transform=transforms.Compose([
                                                   # transforms.RandomResizedCrop(224),
                                                   transforms.Pad(96),
                                                   transforms.ToTensor(),
                                                   transforms.Normalize((0.1307,), (0.3081,))]))
        else:
            train_txt_path = os.path.join(self.datasets_path, 'train.txt')
            val_txt_path = os.path.join(self.datasets_path, 'val.txt')

            my_transform = transforms.Compose([transforms.Resize(224),
                                               transforms.ToTensor()])

            training_data = MyDataset(train_txt_path, transform=my_transform)
            val_data = MyDataset(val_txt_path, transform=my_transform)

        return training_data, val_data

    def model(self, model_name='resnet18', model_path=None):
        assert model_name in ModelList
        self.model_name = model_name
        # model_ft = resnet18(pretrained=True)
        # num_ftrs = model_ft.fc.in_features
        # model_ft.fc = nn.Linear(num_ftrs, 10)

        self.model_ft = ModelList[self.model_name](num_classes=self.n_classes)
        if model_path is not None:
            self.model_ft.load_state_dict(model_path)
        else:
            self.model_ft.apply(weights_init)

        return self.model_ft

    def train(self, training_data, val_data, model):
        # data init
        train_loader = DataLoader(training_data,
                                  batch_size=self.batch_size,
                                  shuffle=True,
                                  # num_workers=self.n_threads,
                                  pin_memory=True)
        # result writer
        train_logger = Logger(os.path.join(self.result_path, self.model_name + '_train.log'),
                              ['epoch', 'loss', 'acc', 'lr'])
        train_batch_logger = Logger(os.path.join(self.result_path, self.model_name + '_train_batch.log'),
                                    ['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])
        val_logger = Logger(os.path.join(self.result_path, self.model_name + '_test.log'),
                            ['time', 'loss', 'acc'])

        # optimizer init
        optimizer = optim.SGD(model.parameters(),
                              lr=self.learning_rate,
                              momentum=self.momentum,
                              weight_decay=self.weight_decay,
                              nesterov=self.nesterov)

        # loss init
        criterion = nn.CrossEntropyLoss()

        print(model)
        if not self.no_cuda:
            model = nn.DataParallel(model, device_ids=[0, 1, 2, 3]).cuda()
        # start train
        for i in range(0, self.n_epochs + 1):
            self.train_epoch(i, train_loader, model, criterion, optimizer,
                             train_logger, train_batch_logger)
            self.validation(val_data, model, criterion, val_logger)

    def train_epoch(self, epoch, data_loader, model, criterion, optimizer,
                    epoch_logger, batch_logger):
        print('train at epoch {}'.format(epoch))
        # set model to train mode
        model.train()

        batch_time = AverageMeter()
        data_time = AverageMeter()
        losses = AverageMeter()
        accuracies = AverageMeter()

        end_time = time.time()
        for i, (inputs, targets) in enumerate(data_loader):
            data_time.update(time.time() - end_time)

            if not self.no_cuda:
                model = model.cuda()
                inputs = inputs.cuda()
                targets = targets.cuda()

            # inputs = Variable(inputs)
            # targets = Variable(targets)
            outputs = model(inputs)

            loss = criterion(outputs, targets)
            acc = calculate_accuracy(outputs, targets)

            losses.update(loss.data, inputs.size(0))
            accuracies.update(acc, inputs.size(0))

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            batch_time.update(time.time() - end_time)
            end_time = time.time()

            batch_logger.log({
                'epoch': epoch,
                'batch': i + 1,
                'iter': (epoch - 1) * len(data_loader) + (i + 1),
                'loss': losses.val,
                'acc': accuracies.val,
                'lr': optimizer.param_groups[0]['lr']
            })

            self.vislog_batch(i, losses.val)

            print('Epoch: [{0}][{1}/{2}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(
                   epoch,
                   i + 1,
                   len(data_loader),
                   batch_time=batch_time,
                   data_time=data_time,
                   loss=losses,
                   acc=accuracies))

        epoch_logger.log({
            'epoch': epoch,
            'loss': losses.avg,
            'acc': accuracies.avg,
            'lr': optimizer.param_groups[0]['lr']
        })

        if epoch % self.checkpoint == 0:
            save_file_path = os.path.join(self.result_path, self.model_name+'save_{}.pth'.format(epoch))
            states = {
                'epoch': epoch + 1,
                'state_dict': model.state_dict(),
                'optimizer': optimizer.state_dict(),
            }
            torch.save(states, save_file_path)

    def validation(self, val_data, model, criterion, val_logger):
        val_loader = torch.utils.data.DataLoader(
            val_data,
            batch_size=self.batch_size,
            shuffle=False,
            # num_workers=self.n_threads,
            pin_memory=True)

        model.eval()
        losses = AverageMeter()
        accuracies = AverageMeter()

        end_time = time.time()
        for i, (inputs, targets) in enumerate(val_loader):

            if not self.no_cuda:
                inputs = inputs.cuda()
                targets = targets.cuda()

            outputs = model(inputs)

            loss = criterion(outputs, targets)
            acc = calculate_accuracy(outputs, targets)

            losses.update(loss.data, inputs.size(0))
            accuracies.update(acc, inputs.size(0))

        test_time = time.time() - end_time

        val_logger.log({'time': test_time,
                        'loss': losses.avg,
                        'acc': accuracies.avg})

        print('TestTime {test_time:.3f}\t'
              'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
              'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(
               test_time=test_time,
               loss=losses,
               acc=accuracies))

    def vislog_batch(self, batch_idx,loss):
        x_value = batch_idx
        y_value = loss
        self.vis.line([y_value], [x_value],
                      name='train_loss',
                      win='losses',
                      update='append')
        self.vis.line([2], [x_value],
                      name='test_loss',
                      win='losses',
                      update='append')
        self.vis.update_window_opts(win='losses', opts=self.vis_loss_opts)

def weights_init(m):
    # weight initialization
    if isinstance(m, nn.Conv2d):
        nn.init.kaiming_normal_(m.weight, mode='fan_out')
        # nn.init.xavier_normal_(m.weight)
        if m.bias is not None:
            nn.init.zeros_(m.bias)
    elif isinstance(m, nn.BatchNorm2d):
        nn.init.ones_(m.weight)
        nn.init.zeros_(m.bias)
    elif isinstance(m, nn.Linear):
        nn.init.normal_(m.weight, 0, 0.01)
        if m.bias is not None:
            nn.init.zeros_(m.bias)


from flask import render_template, flash, redirect, url_for, request, current_app
from flask_login import login_required, current_user

from apps.app import db
from apps.model import Task, Kind
from apps.todolist import todolist
from apps.todolist.course import AddCategory, AddToDoList, ChangeToDoList

# User View Task List
@todolist.route('/list/', methods=['GET', 'POST'])
@login_required
def list_1():
    list = AddToDoList()
    page = int(request.args.get('page', 1))
    todolistPage = Task.query.filter_by(user_id=current_user.id).paginate(per_page=current_app.config['PER_PAGE'],
                                                                          error_out=False)

    return render_template('todolist/list.html', todoObj=todolistPage, form=list)

@todolist.route('/add/', methods=['POST', 'GET'])
@login_required
def add_list():
    list = AddToDoList()
    page = int(request.args.get('page', 1))
    todolistPage = Task.query.filter_by(user_id=current_user.id).paginate(per_page=current_app.config['PER_PAGE'],
                                                                          error_out=False)

    if list.validate_on_submit():
        # get users submission
        Title = list.Title.data
        content = list.content.data
        kind_id = list.category.data
        urgent = list.urgent.data
        deadline = list.deadline.data
        # add to database
        status = False
        add = Task(task_Title=Title, task_content=content,
                   category_id=kind_id,
                   task_urgent=urgent, task_deadline=deadline, user_id=current_user.id, task_status=status)
        db.session.add(add)
        flash('Add task successfully!', category='success')
        return redirect(url_for('todolist.add_list'))
    else:
        flash('Cannot add this task!', category='error')
        return render_template('todolist/list.html', todoObj=todolistPage, form=list)

@todolist.route('/change/<int:id>', methods=['GET', 'POST'])
def change(id):
    list = ChangeToDoList()
    page = int(request.args.get('page', 1))
    todolistPage = Task.query.filter_by(user_id=current_user.id).paginate(per_page=current_app.config['PER_PAGE'],
                                                                          error_out=False)
    task = Task.query.filter_by(id=id).first()

    list.content.data = task.task_content
    # list.category.data = task.category_id
    # list.urgent.data = task.task_urgent
    # list.deadline.data = task.task_deadline
    list.Title.data = task.task_moduleTitle
    list.urgent.data = task.task_urgent
    list.category.data = task.category_id

    if list.validate_on_submit():
        content = request.form.get('content')
        Title = request.form.get('Title')
        category_id = request.form.get('category')
        urgent = request.form.get('urgent')
        task.task_content = content
        task.category_id = category_id
        task.urgent = urgent
        task.task_Title = Title

        db.session.add(task)
        flash('Task has been changed', category='success')
        return redirect(url_for('todolist.list_1'))
    else:
        flash('Changed failed', category='error')
        return render_template('todolist/change.html', form=list, todoObj=todolistPage)

@todolist.route('/delete/<int:id>/')
@login_required
def delete(id):
    task = Task.query.filter_by(id=id).first()
    # task.task_status = True
    db.session.delete(task)
    flash("Task has been deleted successfully.", category='success')
    return redirect(url_for('todolist.list_1'))

@todolist.route('/deletec/<int:id>/')
@login_required
def deletec(id):
    kind = Kind.query.filter_by(id=id).first()
    # task.task_status = True
    db.session.delete(kind)
    flash("Category has been deleted successfully.", category='success')
    category_list = AddCategory()
    KindPage = Kind.query.filter_by(user_id=current_user.id).paginate(per_page=current_app.config['PER_PAGE'],
                                                                      error_out=False)
    return render_template('todolist/category.html', kindObj=KindPage, form=category_list)

@todolist.route('/hasdone/<int:id>/')
@login_required
def hasdone(id):
    task = Task.query.filter_by(id=id).first()
    task.task_status = True
    db.session.add(task)
    flash("Task status has been changed successfully.", category='success')
    return redirect(url_for('todolist.list_1'))

@todolist.route('/hasnotdone/<int:id>/')
@login_required
def hasnotdone(id):
    task = Task.query.filter_by(id=id).first()
    task.task_status = False
    db.session.add(task)
    flash("Task status has been changed successfully.", category='success')
    return redirect(url_for('todolist.list_1'))

@todolist.route('/urgent/<int:id>/')
@login_required
def urgent(id):
    task = Task.query.filter_by(id=id).first()
    task.task_urgent = 1
    db.session.add(task)
    flash("Task priority has been changed successfully.", category='success')
    return redirect(url_for('todolist.list_1'))

@todolist.route('/noturgent/<int:id>/')
@login_required
def noturgent(id):
    task = Task.query.filter_by(id=id).first()
    task.task_urgent = 2
    db.session.add(task)
    flash("Task priority has been changed successfully.", category='success')
    return redirect(url_for('todolist.list_1'))

@todolist.route('/search/<int:id>/', methods=['POST', 'GET'])
@login_required
def search(id):
    task = Task.query.filter_by(id=id).all()
    return render_template('todolist/search.html', todoObj=task)

@todolist.route('/searchTitle/<int:id>/', methods=['POST', 'GET'])
@login_required
def search_Title(id):
    Title = request.form.get('Title')
    task = Task.query.filter_by(user_id=id, task_Title=Title).paginate(per_page=current_app.config['C_PAGE'],
                                                                          error_out=False)
    return render_template('todolist/search.html', todoObj=task)

@todolist.route('/searchContent/<int:id>/', methods=['POST', 'GET'])
@login_required
def search_Content(id):
    content = request.form.get('content')
    task = Task.query.filter_by(user_id=id, task_content=content).paginate(per_page=current_app.config['C_PAGE'],
                                                                          error_out=False)
    return render_template('todolist/search.html', todoObj=task)

@todolist.route('/searchPriority/<int:id>/', methods=['POST', 'GET'])
@login_required
def search_Urgent(id):
    priority = request.form.get('priority')
    task = Task.query.filter_by(user_id=id, task_urgent=priority).paginate(per_page=current_app.config['C_PAGE'],
                                                                          error_out=False)

    return render_template('todolist/search.html', todoObj=task)

@todolist.route('/category/', methods=['POST', 'GET'])
@login_required
def category():
    category_list = AddCategory()
    KindPage = Kind.query.filter_by(user_id=current_user.id).paginate(per_page=current_app.config['PER_PAGE'],
                                                                      error_out=False)
    return render_template('todolist/category.html', kindObj=KindPage, form=category_list)

@todolist.route('/addcategory/', methods=['POST', 'GET'])
@login_required
def add_category():
    list = AddCategory()
    KindPage = Kind.query.filter_by(user_id=current_user.id).paginate(per_page=current_app.config['PER_PAGE'],
                                                                      error_out=False)
    if list.validate_on_submit():
        # get category submission
        name = list.category.data
        user_id = current_user.id
        # add to database
        add = Kind(name=name, user_id=user_id)
        db.session.add(add)
        flash('Add category successfully!', category='success')
        return redirect(url_for('todolist.category'))
    else:
        flash('Cannot add this category', category='error')
        return render_template('todolist/category.html', kindObj=KindPage, form=list)

@todolist.route('/complete/<int:id>/', methods=['POST', 'GET'])
@login_required
def complete(id):
    todolistPage = Task.query.filter_by(user_id=current_user.id).paginate(per_page=current_app.config['C_PAGE'],
                                                                          error_out=False)

    return render_template('todolist/complete.html', todoObj=todolistPage)

@todolist.route('/uncomplete/<int:id>/', methods=['POST', 'GET'])
@login_required
def uncomplete(id):
    todolistPage = Task.query.filter_by(user_id=current_user.id).paginate(per_page=current_app.config['C_PAGE'],
                                                                          error_out=False)

    return render_template('todolist/uncomplete.html', todoObj=todolistPage)


import json
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import euclidean_distances
import numpy as np
from sklearn.linear_model import LogisticRegression

def load():
    with open('C:/Users/Administrator/Desktop/backend-interview-1/samples/generated_test_cases.txt', 'r',
              encoding='utf-8') as json_file:
        data = json.load(json_file)
        return data

def get_word_embeddings():
    embeddings = {}
    with open('C:/Users/Administrator/Desktop/numberbatch-en.txt', encoding='utf-8') as em:
        for embed in em:
            em_line = embed.split(' ')
            if len(em_line) > 2:
                word = em_line[0]
                embedding = np.array(em_line[1:])
                embeddings[word] = embedding
    print('Word embeddings:', len(embeddings))
    return embeddings

def get_min(target_list):
    min_num = target_list[0]
    for i in target_list:
        if i < min_num:
            min_num = i
    return min_num

def get_min_index(target_list):
    min_num = target_list[0]
    for i in target_list:
        if i < min_num:
            min_num = i
    return target_list.index(min_num)

def find_match_document_distance(query, candidates, W):
    if len(query) == 0:
        return "The value of query is null."
    if len(candidates) == 0:
        return "The value of candidates is null."

    vec1 = CountVectorizer(stop_words="english").fit([str(query)])
    vec2 = CountVectorizer(stop_words="english").fit([str(candidates)])

    if len(vec1.get_feature_names()) == 0 or len(vec2.get_feature_names()) == 0:
        return -1

    # print("Query Features:", ", ".join(vec1.get_feature_names()))
    # print("Candidates Features:", ", ".join(vec2.get_feature_names()))

    W1 = [W[w] for w in vec1.get_feature_names()]
    W2 = [W[w] for w in vec2.get_feature_names()]

    result = []
    for i in range(0, len(W1)):
        for j in range(0, len(W2)):
            res = {}
            res['q'] = vec1.get_feature_names()[i]
            res['c'] = vec2.get_feature_names()[j]
            res['r'] = float(euclidean_distances([W1[i]], [W2[j]])[0][0])
            result.append(res)

    t_list = []
    for i in range(0, len(result)):
        t_list.append(float(result[i]['r']))

    return get_min(t_list)

def get_model(W):
    json_data = load()
    print(len(json_data))

    list_X = []
    list_Y = []
    try:
        for i in range(0, len(json_data)):
            aaa = find_match_document_distance(str(json_data[i]['query']), str(json_data[i]['candidates']), W)
            list_X.append(aaa)
            if json_data[i]['correct_index'] >= 0:
                list_Y.append("1")
            else:
                list_Y.append("-1")
    except Exception as e:
        print(i)
        print("word_embeddings中未含有该词")

    print(len(list_X))
    print(len(list_Y))

    x = np.array(list_X).reshape(-1, 1)
    y = np.array(list_Y)

    lr = LogisticRegression(C=1000.0, random_state=0)
    lr.fit(x, y)
    # print(lr.predict(x))
    from sklearn.model_selection import cross_val_score
    acy = cross_val_score(lr, x, y)
    print(acy.mean())
    return lr

def find_best_match_with_threshold(query, candidates, lr, W):

    if len(query) == 0:
        return "The value of query is null."
    if len(candidates) == 0:
        return "The value of candidates is null."

    vec1 = CountVectorizer(stop_words="english").fit([query])
    vec2 = CountVectorizer(stop_words="english").fit([str(candidates)])

    if len(vec1.get_feature_names()) == 0 or len(vec2.get_feature_names()) == 0:
        return -1

    print("Features:", ", ".join(vec1.get_feature_names()))
    print("Features:", ", ".join(vec2.get_feature_names()))

    W1 = [W[w] for w in vec1.get_feature_names()]
    W2 = [W[w] for w in vec2.get_feature_names()]

    result = []
    for i in range(0, len(W1)):
        for j in range(0, len(W2)):
            res = {}
            res['q'] = vec1.get_feature_names()[i]
            res['c'] = vec2.get_feature_names()[j]
            res['r'] = float(euclidean_distances([W1[i]], [W2[j]])[0][0])
            result.append(res)

    t_list = []
    for i in range(0, len(result)):
        t_list.append(float(result[i]['r']))

    # print(t_list)
    # print(lr.predict(get_min(t_list))[0])
    if lr.predict(get_min(t_list))[0] == "-1":
        return -1
    else:
        return candidates.index(result[get_min_index(t_list)]['c'])

W = get_word_embeddings()
lr = get_model(W)
query = "i am really hungry"
candidates = ['story', 'song', 'wake up', 'restart']
print(find_best_match_with_threshold(query, candidates, lr, W))


import requests
from bs4 import BeautifulSoup

url = input("Entrer l'URL du site : ")
response = requests.get(url)

if response.status_code == 200:
    html_content = response.content
else:
    print("Erreur lors de la récupération de la page.")

soup = BeautifulSoup(html_content, "html.parser")

# Extraire le titre de la page
title = soup.title.text
print("Titre de la page :", title)

# Élément de la page à extraire
element = input("Entrez l'élément a extraire du site : ")
elements = soup.find_all(f"{element}")
for item in elements:
    print(item)

# Exemple : Extraire tous les liens de la page
# links = soup.find_all("a")
# for link in links:
#    print(link.get("href"))


import sqlite3
import os
import bamnostic_mod as bn
import argparse
import bisect
import time

#Downloads\Lung\47b982b3-c7ce-4ca7-8c86-c71c15979620\G28588.NCI-H1915.1.bam
#Downloads\Lung\98a0206b-29f5-42d3-957b-6480e2fde185\G20483.HCC-15.2.bam
#Downloads\Lung\18004fb1-89a2-4ba1-a321-a0aa854e98c3\G25210.NCI-H510.1.bam
#Downloads\Lung\47030e40-acbd-4905-939c-d57441d7042e\G25222.NCI-H2171.1.bam
#Downloads\Lung\1357785f-f84b-4688-9b4c-0c2b5472ef51\G27281.RERF-LC-MS.1.bam
#Downloads\Lung\e48ea2ee-1dda-4061-a199-6e22fd2df382\G25212.NCI-H661.1.bam
#Downloads\Lung\f03dbfee-a523-438f-8459-f47f2ff1880f\G25224.NCI-H2066.1.bam

#Downloads\HeadAndNeck\0e67231f-97be-447c-b3b0-a656fc30a62d\G27454.PE_CA-PJ15.2.bam
#Downloads\HeadAndNeck\1acf65a0-0268-4288-9904-33bff618a31d\G27515.PE_CA-PJ41__clone_D2_.2.bam
#Downloads\HeadAndNeck\1f290458-df28-4c78-b73d-0202fb53bb0e\G27220.SCC-4.1.bam
#Downloads\HeadAndNeck\2b507086-977b-4cb7-abd9-83ee4ce9a893\G27489.PE_CA-PJ34__clone_C12_.2.bam
#Downloads\HeadAndNeck\7ed3e895-6826-430d-a39d-338111f16083\G27512.SNU-1214.2.bam
#Downloads\HeadAndNeck\c11aa745-72ea-44ca-b70d-7811c2f244b7\G27533.SNU-1066.2.bam
#Downloads\HeadAndNeck\dc8393c0-7d9e-4040-a91a-5783544cac35\G28853.HSC-4.3.bam

parser = argparse.ArgumentParser(description='takes the given .bam file and looks through all the reads to construct a count of all exons and splices in the reference splice graphs', usage='splicerSampleProcessor database_directory bam_file sample_name novelSplicesToggle(True|False)')
parser.add_argument("Database", help='The path to where you want to store the database file.')
parser.add_argument("Bam", help='The .bam file to count the reads from')
parser.add_argument("sampleName", help='Name for current sample in the sample table')
parser.add_argument("novelSplices", choices=['True', 'False'], help='Controls whether the program tries to find new splices')
args = parser.parse_args()

os.chdir("D:\\")
#get connection to the sqlite database
conn = sqlite3.connect(args.Database + os.path.sep + 'splice.sqlite', isolation_level=None)
c = conn.cursor()

c.execute('''CREATE TABLE IF NOT EXISTS Sample
             (Sample_ID INTEGER PRIMARY KEY NOT NULL DEFAULT NULL,
              Sample_Name varchar(40) NOT NULL DEFAULT NULL,
              Total_Reads INTEGER NOT NULL DEFAULT NULL,
              Transcript_Reads INTEGER NOT NULL DEFAULT NULL);''')
c.execute("CREATE INDEX IF NOT EXISTS idx_sample_name ON Sample(sample_name);")

c.execute('''CREATE TABLE IF NOT EXISTS Sample_Attribs
             (Sample_ID INTEGER NOT NULL DEFAULT NULL,
              Attribute varchar(255) NOT NULL DEFAULT NULL,
              Value varchar(255) NOT NULL DEFAULT NULL);''')

c.execute('''CREATE TABLE IF NOT EXISTS Exon_Counts
             (Sample_ID INTEGER  NOT NULL DEFAULT NULL,
              SG_Exon_ID INTEGER NOT NULL DEFAULT NULL,
              Count INTEGER NOT NULL DEFAULT NULL);''')
c.execute("CREATE INDEX IF NOT EXISTS idx_ec_sample_id ON Exon_Counts(sample_id);")
c.execute("CREATE INDEX IF NOT EXISTS idx_ec_exon_id ON Exon_Counts(SG_Exon_ID);")

c.execute('''CREATE TABLE IF NOT EXISTS Splice_Counts
             (Sample_ID INTEGER  NOT NULL DEFAULT NULL,
              SG_Splice_ID INTEGER NOT NULL DEFAULT NULL,
              Count INTEGER NOT NULL DEFAULT NULL);''')
c.execute("CREATE INDEX IF NOT EXISTS idx_sc_sample_id ON Splice_Counts(sample_id);")
c.execute("CREATE INDEX IF NOT EXISTS idx_sc_splice_id ON Splice_Counts(SG_Splice_ID);")

#Find out the next assignable ID for this sample
c.execute("Select MAX(Sample_ID) FROM Sample")
ret = c.fetchone()
prevId = ret[0]
if prevId:
    Sample_Id = int(prevId)+1
else:
    Sample_Id = 1

novelSplices = args.novelSplices

#initialize the splice dictionary
sDict = {}
eDict = {}
epDict = {}
ecDict = {}
scDict = {}
discoverySplices = {}

start_time = time.time()

c.execute("SELECT SG_Splice_ID, Start_Position, Stop_Position, Chromosome FROM SG_Splice")
ret = c.fetchall()
#load the splice dictionary keyed on start-stop with the sg id as the value
for y in range(len(ret)):
    key = str(ret[y][1])+'-'+str(ret[y][2])
    chrom = ret[y][3]
    if not chrom.startswith("chr"):
        chrom = "chr"+chrom
    if chrom == "chrMT":
        chrom = "chrM"
    if chrom not in sDict:
        sDict[chrom] = {}
        if novelSplices == 'True':
            discoverySplices[chrom] = {}
    sDict[chrom][key] = ret[y][0]

c.execute("SELECT SG_Exon_ID, Start_Position, Stop_Position, Chromosome FROM SG_Exon")
ret = c.fetchall()
#load the exon dictionary keyed on the start and the stop with the sg id as the value
for y in range(len(ret)):
    chrom = ret[y][3]
    if not chrom.startswith("chr"):
        chrom = "chr"+chrom
    if chrom == "chrMT":
        chrom = "chrM"
    if chrom not in eDict:
        eDict[chrom] = {}
        epDict[chrom] = []

    #add start
    eDict[chrom][ret[y][1]] = ret[y][0]
    #add stop
    eDict[chrom][ret[y][2]] = ret[y][0]
    #add to tuple exon positions list (flip start and stop to correct if negative strand)
    if ret[y][1] < ret[y][2]:
        epDict[chrom].append((ret[y][1], ret[y][2]))
    else:
        epDict[chrom].append((ret[y][2], ret[y][1]))

#sorted list of all exon start stop tuples keyed on chromosome
for key in epDict:
    epDict[key] = sorted(epDict[key])

#"hg19test.bam"
samfile = bn.AlignmentFile(args.Bam, "rb")

def exonIncrement(start, stop, chro):
    exonIds = []
    try:
        pList = epDict[chro]
        #flip start in stop to correct for negative strand
        if start > stop:
            temp = start
            start = stop
            stop = temp
        #find the index that start belongs at
        idx = bisect.bisect(pList, (start,stop))

        i = idx
        if i == len(pList):
            return([])
        #move down the exons adding the ids of those included in the read
        while (i > -1  and start <= pList[i][1]):
            exonIds.append(eDict[chro][pList[i][0]])
            i-=1
        #ISSUE IF NEVER LOOP****************
        #if it goes off the end of a known exon add none and scrap the read
        if start < pList[i+1][0]:
            return([])

        i = idx
        looped = False
        #move up the exons adding ids of those included in the read
        while (i < len(pList) and i > -1 and stop >= pList[i][0]):
            exonIds.append(eDict[chro][pList[i][1]])
            i+=1
            looped = True
        #if it goes of the end of a known exon add none and scrap the read
        if looped and stop > pList[i-1][1]:
            return([])
    except Exception:
        pass

    return(exonIds)

fns = open ('novelSplices.txt', 'w')
i = 0

totalCount = 0
missingAttrCount = 0
tranCount = 0
totalDupeCount = 0
tranDupeCount = 0
totalJRcount = 0
tranJRcount = 0
totalNJRcount = 0
tranNJRcount = 0
exceptionCount = 0
prevRead = ""
prevExons = ""
prevSplices = ""
for read in samfile:
    #read does not have an alignment
    if (
        not hasattr(read, 'reference_name')  or read.reference_name == None  or
        not hasattr(read, 'reference_start') or read.reference_start == None or
        not hasattr(read, 'reference_end')   or read.reference_end == None   or
        not hasattr(read, 'cigarstring')     or read.cigarstring == None     or
        not hasattr(read, 'cigar')           or read.cigar == None
       ):
        missingAttrCount += 1
        continue

    dupeTag = False
    exonSet = set()
    spliceSet = set()
    readR_S = read.reference_start
    readR_E = read.reference_end
    i+=1
    totalCount += 1
    if totalCount % 1000000 == 0:
        print(f"{totalCount:,d}")
        break
    tranBool = False
    cigarString = read.cigarstring
    cigar = read.cigar
    chro = read.reference_name
    if not chro.startswith("chr"):
        chro = "chr"+chro
    if str(readR_S)+"-"+str(readR_E)+"-"+cigarString+"-"+chro == prevRead:
        dupeTag = True
        totalDupeCount += 1
        for exon in prevExons:
            tranBool = True
            ecDict[exon] += 1
        for splice in prevSplices:
            tranBool = True
            scDict[splice] += 1
        if tranBool == True:
            tranDupeCount += 1
    elif "N" in cigarString:
        totalJRcount += 1
        #initialize the start and stop based on the first junction
        start = readR_S+cigar[0][1]
        stop = start+cigar[1][1]+1
        #exon check from the start of the read to the start of the first splice
        exonSet.update(exonIncrement(readR_S+1, start, chro))
        for x in range(int(len(cigar)/2)):
            #if this is not the first junction adjust the start and stop
            if x != 0:
                start = stop+cigar[x*2][1]-1
                #exon check from the end of the last splice to the beginning of this splice
                exonSet.update(exonIncrement(stop, start, chro))
                stop = start+cigar[x*2+1][1]+1

            #check if the splice is known and count it if so
            try:
                if str(start)+"-"+str(stop) in sDict[chro] or str(stop)+"-"+str(start) in sDict[chro]:
                    spliceID = sDict[chro][str(start)+"-"+str(stop)]
                    spliceSet.add(spliceID)
                    tranBool = True
                    if spliceID in scDict:
                        scDict[spliceID] += 1
                    else:
                        scDict[spliceID] = 1
                elif novelSplices == 'True':
                    if start in eDict[chro] and stop in eDict[chro]:
                        if str(start)+"-"+str(stop) in discoverySplices[chro]:
                            discoverySplices[chro][str(start)+"-"+str(stop)]+=1
                        else:
                            discoverySplices[chro][str(start)+"-"+str(stop)]=1

                        experiSplicect = 1
            except Exception as e:
                exceptionCount += 1
            exonID = ""

        exonSet.update(exonIncrement(stop, readR_E, chro))
        if (tranBool or len(exonSet) != 0):
            tranJRcount += 1
    else:
        totalNJRcount += 1
        start = readR_S+1
        stop = start+cigar[0][1]
        exonSet.update(exonIncrement(start, stop, chro))
        if (len(exonSet) != 0):
            tranNJRcount += 1
        else:
            print("Missing: " + chro + ' ' + str(start) + ' ' + str(stop))

    #add in all the sets
    for exon in exonSet:
        tranBool = True
        #print("exon: "+str(exon))
        if exon in ecDict:
            ecDict[exon] += 1
        else:
            ecDict[exon] = 1

    if tranBool == True:
        tranCount += 1

    #set this line to prevRead
    if dupeTag == False:
        prevRead = str(readR_S)+"-"+str(readR_E)+"-"+cigarString+"-"+chro
        prevExons = exonSet
        prevSplices = spliceSet

    #if i == 5000000:
     #   break

c.execute('begin')
for key in scDict:
    c.execute("INSERT INTO Splice_Counts VALUES("+str(Sample_Id)+", "+str(key)+", "+str(scDict[key])+")")

for key in ecDict:
    c.execute("INSERT INTO Exon_Counts VALUES("+str(Sample_Id)+", "+str(key)+", "+str(ecDict[key])+")")

#add this sample to the sample table
c.execute("INSERT INTO Sample VALUES("+str(Sample_Id)+", "+args.sampleName+", "+str(totalCount)+", "+str(tranCount)+")")
c.execute('commit')
for chromkey in discoverySplices:
    for skey in discoverySplices[chromkey]:
        fns.write(skey + ", Count: " + str(discoverySplices[chromkey][skey])+'\n')
fns.close()
print("missing attribute reads: " + str(missingAttrCount))
print("transcript junction reads: "+str(tranJRcount))
print("total junction reads: "+str(totalJRcount))
print("transcript duplicate reads: "+str(tranDupeCount))
print("total duplicate reads: "+str(totalDupeCount))
print("transcript Non junction reads: "+str(tranNJRcount))
print("total Non junction reads: "+str(totalNJRcount))

print("number of exceptions caught: "+ str(exceptionCount))

print("--- %.2f seconds---" % (time.time() - start_time))
print('Done')


import mtcnn
from mtcnn.mtcnn import MTCNN
import cv2

detector = MTCNN()  # MTCNN is CNN based algorithm

video = cv2.VideoCapture(0)
video.set(3,2000)
video.set(4,3000) # Same as previous technique

while (True):
    ret, frame = video.read()
    if ret == True:
        location = detector.detect_faces(frame)  # dectect faces frame by frame
        if len(location) > 0:
            for face in location:
                x, y, width, height = face['box']
                x2, y2 = x + width, y + height
                cv2.rectangle(frame, (x, y), (x2, y2), (0, 0, 255), 4) # Bounding box width and height
        cv2.imshow("Output",frame) # its will show frame's and update it frame by frame to same output file named as "Output"
        if cv2.waitKey(1) & 0xFF == ord(' '):   # same as previous
            break
    else:
        break

video.release()  # releasing camera port
cv2.destroyAllWindows() # destroying all windows

# Lets run and see , Results
# as you can see, it is giving precise result

# so we will go with MTCNN Algorithm


# Based on https://github.com/NATSpeech/NATSpeech
import utils.commons.single_thread_env  # NOQA
import json
import numpy as np
import os
import random
import traceback

from functools import partial
from resemblyzer import VoiceEncoder
from tqdm import tqdm

from utils.audio.align import get_mel2note
from utils.audio.mel_processing import torch_wav2spec
from utils.audio.pitch.utils import f0_to_coarse
from utils.audio.pitch_extractors import extract_pitch_simple
from utils.commons.hparams import hparams
from utils.commons.indexed_datasets import IndexedDatasetBuilder
from utils.commons.multiprocess_utils import multiprocess_run_tqdm
from utils.os_utils import remove_file, copy_file

np.seterr(divide="ignore", invalid="ignore")

class BinarizationError(Exception):
    pass

class BaseBinarizer:
    def __init__(self, processed_data_dir=None):
        if processed_data_dir is None:
            processed_data_dir = hparams["processed_data_dir"]
        self.processed_data_dir = processed_data_dir
        self.hparams = hparams
        self.binary_data_dir = hparams["binary_data_dir"]
        self.preprocess_args = hparams["preprocess_args"]
        self.binarization_args = hparams["binarization_args"]
        self.items = {}
        self.item_names = []
        if self.binarization_args["with_spk_f0_norm"]:
            self.spk_pitch_map = {}

    def load_meta_data(self):
        processed_data_dir = self.processed_data_dir
        item_list = json.load(open(f"{processed_data_dir}/metadata.json"))
        for r in tqdm(item_list, desc="Loading meta data."):
            item_name = r["item_name"]
            self.items[item_name] = r
            self.item_names.append(item_name)
        if self.binarization_args["shuffle"]:
            random.seed(1234)
            random.shuffle(self.item_names)

    @property
    def train_index_item_names(self):
        range_ = self._convert_range(self.binarization_args["train_range"])
        return self.item_names[range_[0]:range_[1]]

    @property
    def valid_index_item_names(self):
        range_ = self._convert_range(self.binarization_args["valid_range"])
        return self.item_names[range_[0]:range_[1]]

    @property
    def test_index_item_names(self) -> list:
        range_ = self._convert_range(self.binarization_args["test_range"])
        return self.item_names[range_[0]:range_[1]]

    def _convert_range(self, range_: list):
        if range_[1] == -1:
            range_[1] = len(self.item_names)
        return range_

    @property
    def train_title_item_names(self):
        return [item_name for item_name in self.item_names \
                if any(title in item_name for title in self.binarization_args["train_range"])]

    @property
    def valid_title_item_names(self):
        return [item_name for item_name in self.item_names \
                if any(title in item_name for title in self.binarization_args["valid_range"])]

    @property
    def test_title_item_names(self):
        return [item_name for item_name in self.item_names \
                if any(title in item_name for title in self.binarization_args["test_range"])]

    def meta_data(self, prefix: str, dataset_range):
        """
        Parameter
        ---------
        prefix: str
            Choose one of ["train", "valid", "test"]
        """
        if prefix == "valid":
            if dataset_range == "index":
                item_names = self.valid_index_item_names
            elif dataset_range == "title":
                item_names = self.valid_title_item_names
        elif prefix == "test":
            if dataset_range == "index":
                item_names = self.test_index_item_names
            elif dataset_range == "title":
                    item_names = self.test_title_item_names
        else:
            if dataset_range == "index":
                item_names = self.train_index_item_names
            elif dataset_range == "title":
                item_names = self.train_title_item_names
        for item_name in item_names:
            yield self.items[item_name]

    def process(self):
        self.load_meta_data()
        os.makedirs(hparams["binary_data_dir"], exist_ok=True)
        for fn in ["phone_set.json", "spk_map.json"]:
            remove_file(f"{hparams['binary_data_dir']}/{fn}")
            copy_file(f"{hparams['processed_data_dir']}/{fn}", f"{hparams['binary_data_dir']}/{fn}")
        self.note_pitch_map = self.build_pitch_map()
        self.note_dur_map = self.build_dur_map()
        self.note_tempo_map = self.build_tempo_map()
        self.process_data("valid")
        self.process_data("test")
        self.process_data("train")

    def process_data(self, prefix: str):
        """
        Parameter
        ---------
        prefix: str
            Choose one of ["train", "valid", "test"]
        """
        data_dir = hparams["binary_data_dir"]
        meta_data = list(self.meta_data(prefix, self.binarization_args["dataset_range"]))
        process_item = partial(self.process_item, preprocess_args=self.preprocess_args,
                               binarization_args=self.binarization_args)
        builder = IndexedDatasetBuilder(f"{data_dir}/{prefix}")
        ph_lengths = []
        mel_lengths = []
        total_sec = 0
        max_sec = 0
        total_file = 0
        items = []
        args = [{"item": item, "note_pitch_map": self.note_pitch_map, "note_dur_map": self.note_dur_map,
                 "note_tempo_map": self.note_tempo_map} for item in meta_data[:len(meta_data)]]
        # Get information from audio and transcript
        for item_id, item in multiprocess_run_tqdm(process_item, args, desc="Processing data"):
            if item is not None:
                items.append(item)
                if self.binarization_args["with_spk_f0_norm"] and prefix == "train":
                    self.calculate_spk_stats(item["f0"], item["spk_id"])
        # Use pre-trained speaker embeddings
        if self.binarization_args["with_spk_embed"]:
            args = [{"wav": item["wav"]} for item in items]
            for item_id, spk_embed in multiprocess_run_tqdm(
                    self.get_spk_embed, args,
                    init_ctx_func=lambda wid: {"voice_encoder": VoiceEncoder().cuda()}, num_workers=4,
                    desc="Extracting spk embed"):
                items[item_id]["spk_embed"] = spk_embed

        for item in items:
            if not self.binarization_args["with_wav"] and "wav" in item:
                del item["wav"]
            mel_lengths.append(item["len"])
            assert item["len"] > 0, (item["item_name"], item["text"], item["mel2ph"])
            if "ph_len" in item:
                ph_lengths.append(item["ph_len"])
            if max_sec < item["sec"]:
                max_sec = item["sec"]
            total_sec += item["sec"]
            if "midi_info" in item:
                del item["midi_info"]
                del item["sec"]
                del item["others"]
            if not self.binarization_args["with_mel"] and "mel" in item:
                del item["mel"]
            builder.add_item(item)
        total_file += len(items)
        builder.finalize()

        if os.path.exists(f"{data_dir}/{prefix}_lengths.npy"):
            mel_lengths_ = np.load(f"{data_dir}/{prefix}_lengths.npy").tolist()
            mel_lengths_.extend(mel_lengths)
            mel_lengths = mel_lengths_
        np.save(f"{data_dir}/{prefix}_lengths.npy", mel_lengths)
        if len(ph_lengths) > 0:
            if os.path.exists(f"{data_dir}/{prefix}_ph_lenghts.npy"):
                ph_lengths_ = np.load(f"{data_dir}/{prefix}_ph_lenghts.npy").tolist()
                ph_lengths.extend(ph_lengths_)
            np.save(f"{data_dir}/{prefix}_ph_lenghts.npy", ph_lengths)
        if self.binarization_args["with_spk_f0_norm"] and prefix == "train":
            self.build_spk_pitch_map()
        print(f"| {prefix} total files: {total_file}, total duration: {total_sec:.3f}s, max duration: {max_sec:.3f}s")

    @classmethod
    def process_item(cls, item: dict, note_pitch_map, note_dur_map, note_tempo_map, preprocess_args, binarization_args: dict):
        item["ph_len"] = len(item["ph_token"])
        item_name = item["item_name"]
        wav_fn = item["wav_fn"]
        # Get Waveform and Mel-spectrogram information
        wav, mel = cls.process_audio(wav_fn, item, binarization_args)
        if mel.shape[0] > 2:
            try:
                n_bos_frames, n_eos_frames = 0, 0
                if preprocess_args["use_text"] and preprocess_args["use_midi"]:
                    if binarization_args["with_midi_align"]:
                        # Align text information
                        cls.process_midi_align(item)
                # Mapping pitch and dur map of note
                cls.process_note(item, note_pitch_map, note_dur_map, note_tempo_map, preprocess_args, binarization_args)
                if binarization_args["trim_eos_bos"]:
                    n_bos_frames = item["duration"][0] if preprocess_args["use_midi"] else 0
                    n_eos_frames = item["duration"][-1] if preprocess_args["use_midi"] else mel.shape[0]
                    T = len(mel)
                    item["mel"] = mel[n_bos_frames:T - n_eos_frames]
                    item["mel2ph"] = item["mel2ph"][n_bos_frames:T - n_eos_frames]
                    item["duration"] = item["duration"][1:-1]
                    item["duration_midi"] = item["duration_midi"][1:-1]
                    item["len"] = item["mel"].shape[0]
                    item["wav"] = wav[n_bos_frames * hparams["hop_size"]:len(wav) - n_eos_frames * hparams["hop_size"]]
                if binarization_args["with_f0"]:
                    # Get pitch information
                    cls.process_pitch(item, n_bos_frames, n_eos_frames)
            except BinarizationError as e:
                print(f"| Skip item ({e}). item_name: {item_name}, wav_fm: {wav_fn}")
                return None
            except Exception as e:
                traceback.print_exc()
                print(f"| Skip item. item_name: {item_name}, wav_fm: {wav_fn}")
                return None
        return item

    @classmethod
    def process_audio(cls, wav_fn: str, res: dict, binarization_args: dict):
        # Get Mel-spectrogram
        wav2spec_dict = torch_wav2spec(wav_fn,
                                       fft_size=hparams["fft_size"],
                                       hop_size=hparams["hop_size"],
                                       win_length=hparams["win_size"],
                                       num_mels=hparams["num_mel_bins"],
                                       fmin=hparams["fmin"],
                                       fmax=hparams["fmax"],
                                       sample_rate=hparams["sample_rate"])
        mel = wav2spec_dict["mel"]
        wav = wav2spec_dict["wav"].astype(np.float16)

        #  Check Linear-spectrogram
        if binarization_args["with_linear"]:
            res["linear"] = wav2spec_dict["linear"]
        if "wav_norm" in wav2spec_dict:
            res["wav_norm"] = wav2spec_dict["wav_norm"]
        res.update({"mel": mel, "wav": wav, "sec": len(wav) / hparams["sample_rate"], "len": mel.shape[0]})
        return wav, mel

    @staticmethod
    def process_midi_align(item: dict):
        mel = item["mel"]
        midi_info = item["midi_info"]
        # Get align information and duration
        mel2phone, mel2note, duration, ph_token, ph_list, _, item["midi_info"] = get_mel2note(midi_info, mel, hparams["hop_size"],
                                                                                hparams["sample_rate"], item["silence"])
        item["ph_token"] = ph_token
        item["text"] = ph_list
        if len(ph_list) < hparams["binarization_args"]["min_text"] or ph_list is None:
            raise BinarizationError(
                f"| Less than min text sequence: {len(item['ph_token'])}")
        if np.array(mel2phone).max() - 1 >= len(ph_token):
            raise BinarizationError(
                f"Align does not match: mel2ph.max - 1 != {np.array(mel2phone).max() - 1}, len(ph_token): {len(ph_token)}")
        item["mel2ph"] = mel2phone
        item["mel2note"] = mel2note
        item["duration"] = duration
        # Get phoneme to word information
        assert len(ph_token) == len(duration), "| phoneme : {len(ph_token)}, ph_duration : {len(duration)}"

    @staticmethod
    def process_note(item, note_pitch_map, note_dur_map, note_tempo_map, preprocess_args, binarization_args):
        dur_enc = list()
        dur_dec = list()
        for i in range(binarization_args["max_durations"]):
            for _ in range(binarization_args["pos_resolution"]):
                dur_dec.append(len(dur_enc))
                for _ in range(2 ** i):
                    dur_enc.append(len(dur_dec) - 1)
        def d2e(x):
            return dur_enc[x] if x < len(dur_enc) else dur_enc[-1]
        if preprocess_args["use_midi"]:
            item["note_duration"] = [note_dur_map[str(d2e(note[3]))] for note in item["midi_info"]]
            item["note_pitch"] = [note_pitch_map[str(note[2])] for note in item["midi_info"]]
            item["note_tempo"] = [note_tempo_map[str(note[6])] for note in item["midi_info"]]
        else:
            item["note_duration"] = [0]
            item["note_pitch"] = [0]
            item["note_tempo"] = [0]

    @staticmethod
    def process_pitch(item: dict, n_bos_frames: int, n_eos_frames: int):
        wav, mel = item["wav"], item["mel"]
        # Get f0 from waveform
        f0 = extract_pitch_simple(wav)
        if sum(f0) == 0:
            raise BinarizationError("Empty f0")
        assert len(mel) == len(f0) // hparams["f0_resolution"], (len(mel), len(f0))
        # Quantize f0 values
        pitch_coarse = f0_to_coarse(f0)
        item["f0"] = f0
        item["pitch"] = pitch_coarse
        if hparams["binarization_args"]["with_f0cwt"]:
            _, cont_logf0 = get_cont_logf0(f0)
            logf0s_mean, logf0s_std = np.mean(cont_logf0), np.std(cont_logf0)
            cont_logf0_norm = (cont_logf0 - logf0s_mean) / logf0s_std
            cwt_spec, _ = get_logf0_cwt(cont_logf0_norm)
            item["cwt_spec"] = cwt_spec
            item["cwt_mean"] = logf0s_mean
            item["cwt_std"] = logf0s_std

    def build_pitch_map(self):
        """ Using 0 to 128 notes for MIDI. """
        pitch_map = {"0": 0}
        for i, x in enumerate(range(self.hparams["note_range"][0], self.hparams["note_range"][1])):
            pitch_map[str(x)] = i + 1
        json.dump(pitch_map, open(f"{self.binary_data_dir}/pitch_map.json", "w"), ensure_ascii=False)

        return pitch_map

    def build_dur_map(self):
        """ Using max duration for MIDI. """
        dur_map = {"[PAD]": 0, "[BOS]": 1, "[EOS]": 2}
        for i, x in enumerate(range(0, 128)):
            dur_map[str(x)] = i + 4
        json.dump(dur_map, open(f"{self.binary_data_dir}/dur_map.json", "w"), ensure_ascii=False)
        return dur_map

    def build_tempo_map(self):
        tempo_map = {"[PAD]": 0, "[BOS]": 1, "[EOS]": 2}
        tempo_range = self.binarization_args["tempo_range"]
        for i, x in enumerate(range(tempo_range[0], tempo_range[1] + 1)):
            tempo_map[str(x)] = i + 4
        json.dump(tempo_map, open(f"{self.binary_data_dir}/tempo_map.json", "w"), ensure_ascii=False)
        return tempo_map

    def calculate_spk_stats(self, f0, spk_id):
        f0_min = f0[np.nonzero(f0)].min()
        f0_max = f0.max()
        if str(spk_id) in self.spk_pitch_map:
            spk_pitch_stat = self.spk_pitch_map[str(spk_id)]
            if spk_pitch_stat["min"] > f0_min:
                self.spk_pitch_map[str(spk_id)]["min"] = f0_min
            if spk_pitch_stat["max"] < f0_max:
                self.spk_pitch_map[str(spk_id)]["max"] = f0_max
        else:
            spk_pitch_stat = {}
            spk_pitch_stat["max"] = f0_max
            spk_pitch_stat["min"] = f0_min
            self.spk_pitch_map[str(spk_id)] = spk_pitch_stat

    def build_spk_pitch_map(self):
        spk_pitch_map = {}
        stat_map_dir = f"{self.binary_data_dir}/spk_pitch_map.json"
        if os.path.exists(stat_map_dir):
            spk_pitch_map = json.load(open(stat_map_dir, "r"))
        spk_pitch_map.update(self.spk_pitch_map)
        spk_pitch_map = {key: value for key, value in sorted(spk_pitch_map.items(), key=lambda x: int(x[0]))}
        print("| Statistics of speaker's pitch is saved.")
        json.dump(spk_pitch_map, open(stat_map_dir, "w"), ensure_ascii=False)

    @staticmethod
    def get_spk_embed(wav, ctx):
        return ctx["voice_encoder"].embed_utterance(wav.astype(float))

    @property
    def num_workers(self):
        return int(os.getenv("N_PROC", hparams.get("N_PROC", os.cpu_count())))


import yaml

from populate.populator.common.errors import ConfigurationError

from .projects_manager import ProjectsManager

def project_constructor(loader, node):
    if isinstance(node, yaml.ScalarNode):
        item = loader.construct_scalar(node)
        if not isinstance(item, str) or not item:
            raise ConfigurationError(
                'value {} cannot be interpreted as project'.format(item))
    elif isinstance(node, yaml.MappingNode):
        item = loader.construct_mapping(node)
        if not isinstance(item, dict) or not item:
            raise ConfigurationError(
                'value {} cannot be interpreted as project'.format(item))
    return ProjectsManager().get_object(item)


from __future__ import annotations
from dataclasses import dataclass
import bson.json_util as json

__all__ = ['Node', 'build_execution_tree']

@dataclass
class Node:
    """Represent SBE tree node."""

    stage: str
    plan_node_id: int
    total_execution_time: int
    n_returned: int
    n_processed: int
    children: list[Node]

    def get_execution_time(self):
        """Execution time of the SBE node without execuion time of its children."""
        return self.total_execution_time - sum(n.total_execution_time for n in self.children)

    def print(self, level=0):
        """Pretty print of the SBE tree."""
        print(
            f'{"| "*level}{self.stage}, plaNodeId: {self.plan_node_id}, totalExecutionTime: {self.total_execution_time:,}, nReturned: {self.n_returned}, nProcessed: {self.n_processed}'
        )
        for child in self.children:
            child.print(level + 1)

def build_execution_tree(execution_stats: dict[str, any]) -> Node:
    """Build SBE executioon tree from 'executionStats' field of query explain."""
    assert execution_stats['executionSuccess']
    return process_stage(execution_stats['executionStages'])

def process_stage(stage: dict[str, any]) -> Node:
    """Parse the given SBE stage."""
    processors = {
        'filter': process_filter,
        'cfilter': process_filter,
        'traverse': process_traverse,
        'project': process_inner_node,
        'limit': process_inner_node,
        'scan': process_seek,
        'coscan': process_leaf_node,
        'nlj': process_nlj,
        'hj': process_hash_join_node,
        'mj': process_hash_join_node,
        'seek': process_seek,
        'ixseek': process_seek,
        'limitskip': process_inner_node,
        'group': process_inner_node,
        'union': process_union_node,
        'unique': process_unique_node,
        'unwind': process_unwind_node,
    }

    processor = processors.get(stage['stage'])
    if processor is None:
        print(json.dumps(stage, indent=4))
        raise ValueError(f'Unknown stage: {stage}')

    return processor(stage)

def process_filter(stage: dict[str, any]) -> Node:
    """Process filter stage."""
    input_stage = process_stage(stage['inputStage'])
    return Node(**get_common_fields(stage), n_processed=stage['numTested'], children=[input_stage])

def process_traverse(stage: dict[str, any]) -> Node:
    """Process traverse, not used by Bonsai."""
    outer_stage = process_stage(stage['outerStage'])
    inner_stage = process_stage(stage['innerStage'])
    return Node(**get_common_fields(stage), n_processed=stage['nReturned'],
                children=[outer_stage, inner_stage])

def process_hash_join_node(stage: dict[str, any]) -> Node:
    """Process hj node."""
    outer_stage = process_stage(stage['outerStage'])
    inner_stage = process_stage(stage['innerStage'])
    n_processed = outer_stage.n_returned + inner_stage.n_returned
    return Node(**get_common_fields(stage), n_processed=n_processed,
                children=[outer_stage, inner_stage])

def process_nlj(stage: dict[str, any]) -> Node:
    """Process nlj stage."""
    outer_stage = process_stage(stage['outerStage'])
    inner_stage = process_stage(stage['innerStage'])
    n_processed = stage['totalDocsExamined']
    return Node(**get_common_fields(stage), n_processed=n_processed,
                children=[outer_stage, inner_stage])

def process_inner_node(stage: dict[str, any]) -> Node:
    """Process SBE stage with one input stage."""
    input_stage = process_stage(stage['inputStage'])
    return Node(**get_common_fields(stage), n_processed=input_stage.n_returned,
                children=[input_stage])

def process_leaf_node(stage: dict[str, any]) -> Node:
    """Process SBE stage without input stages."""
    return Node(**get_common_fields(stage), n_processed=stage['nReturned'], children=[])

def process_seek(stage: dict[str, any]) -> Node:
    """Process seek stage."""
    return Node(**get_common_fields(stage), n_processed=stage['numReads'], children=[])

def process_union_node(stage: dict[str, any]) -> Node:
    """Process union stage."""
    children = [process_stage(child) for child in stage['inputStages']]
    return Node(**get_common_fields(stage), n_processed=stage['nReturned'], children=children)

def process_unwind_node(stage: dict[str, any]) -> Node:
    """Process unwind stage."""
    input_stage = process_stage(stage['inputStage'])
    return Node(**get_common_fields(stage), n_processed=input_stage.n_returned,
                children=[input_stage])

def process_unique_node(stage: dict[str, any]) -> Node:
    """Process unique stage."""
    input_stage = process_stage(stage['inputStage'])
    n_processed = stage['dupsTested']
    return Node(**get_common_fields(stage), n_processed=n_processed, children=[input_stage])

def get_common_fields(json_stage: dict[str, any]) -> dict[str, any]:
    """Exctract common field from json representation of SBE stage."""
    return {
        'stage': json_stage['stage'], 'plan_node_id': json_stage['planNodeId'],
        'total_execution_time': json_stage['executionTimeNanos'],
        'n_returned': json_stage['nReturned']
    }


#!/usr/bin/env python

import gzip
import sys
import tarfile
import threading
import urllib.request
import zipfile

import lib.download.task as task
import lib.independence.fs as fs
import lib.ui.color as printer

# The greater purpose of (functions in) this file is
# to download a list of DownloadTasks

class Downloader(object):
    '''Object to facilitate downloading'''

    # Constructor
    # tasklist is the list of DownloadTasks which should be downloaded
    def __init__(self, tasklist):
        self.tasklist = tasklist

    # Returns True if the name suggests it is a tar archive, otherwise False
    def is_tar(self, name):
        return name.endswith('.tar') or name.endswith('tar.xz')\
        or name.endswith('tar.gz') or name.endswith('.tgz')

    # Returns True if the name suggests it is a zip archive, otherwise False
    def is_zip(self, name):
        return name.endswith('.zip')

    # Returns True if the name suggests it is a gzip archive, otherwise False
    def is_gzip(self, name):
        return name.endswith('.gz')

    # Extract output of downloaded file, if it has a compatible format
    # task is the task for which the file is downloaded
    def extract(self, task):
        target = fs.join(task.directory,task.name)
        basicstring = printer.format('extracter', printer.Color.CAN)
        extractstring = printer.format('extracting', printer.Color.YEL)
        print('[{0}] {1} {2}'.format(basicstring, extractstring, task.name))

        if self.is_tar(task.url):
            ttar = tarfile.open(target, 'r')
            ttar.extractall(path=task.directory)
        elif self.is_zip(task.url):
            tzip = zipfile.ZipFile(target, 'r')
            tzip.extractall(task.directory)
            tzip.close()
        elif self.is_gzip(task.url):
            with gzip.open(target, 'rb') as f_in:
                with open(task.directory, 'wb') as f_out:
                    fs.cp(f_in, f_out)
        else:
            return

        finishedstring = printer.format('extracted', printer.Color.GRN)
        print('[{0}] {1} {2}'.format(basicstring, finishedstring, task.name))

        fs.rm(fs.join(task.directory, task.name))

        dircontents = fs.ls(task.directory)
        if len(dircontents) == 1 and fs.isdir(task.directory,dircontents[0]):
            subdircontents = fs.ls(task.directory,dircontents[0])
            for file in subdircontents:
                path = fs.join(task.directory,dircontents[0])
                fs.mv(fs.join(path,file), task.directory)
            fs.rm(task.directory, dircontents[0], ignore_errors=True)

    # Downloads a DownloadTask and prints some user information
    # task is the downloadtask which contains the download information
    def download(self, task):
        basicstring = printer.format('downloader', printer.Color.CAN)
        downloadstring = printer.format('downloading', printer.Color.YEL)
        print('[{0}] {1} {2}'.format(basicstring, downloadstring, task.name))

        u = urllib.request.urlopen(task.url)
        with open(fs.join(task.directory,task.name), 'wb') as out_file:
            out_file.write(u.read())

        finishedstring = printer.format('downloaded', printer.Color.GRN)
        print('[{0}] {1} {2}'.format(basicstring, finishedstring, task.name))

    # Parallel function, which is called to download all tasks
    # task is the downloadtask which should be operated on
    def parallel_exec(self, task):
        self.download(task)

        if self.is_tar(task.url)\
        or self.is_zip(task.url)\
        or self.is_gzip(task.url):
            self.extract(task)

    # Main function to call. Each DownloadTask will be performed
    def download_all(self):
        threads = []
        for task in self.tasklist:
            threads.append(threading.Thread(target=self.parallel_exec, args=(task,)))

        for thread in threads:
            thread.start()
        for thread in threads:
            thread.join()


import tensorflow as tf
from model.model_builder import ModelBuilder
from utils.model_post_processing import merge_post_process
from tensorflow.keras.models import Model
from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2
from utils.priors import *
import argparse

parser = argparse.ArgumentParser()
parser.add_argument("--backbone_name",       type=str,    help="Pretrained backbone name\
                                                            |   model_name    : description | \
                                                            [ 1. mobilenetv2       : MobileNetV2 ]\
                                                            [ 2. mobilenetv3s      : MobileNetV3-Small ] \
                                                            [ 3. mobilenetv3l      : MobileNetV3-Large ] \
                                                            [ 4. efficient_lite_v0 : EfficientNet-Lite-B0 ]\
                                                            [ 5. efficientnetv2b0  : EfficientNet-V2-B0 ]\
                                                            [ 6. efficientnetv2b3  : EfficientNet-V2-B3 ]",
                    default='efficient_lite_v0')
parser.add_argument("--checkpoint_dir",      type=str,    help="Set the model storage directory",
                    default='./checkpoints/')
parser.add_argument("--model_weights",       type=str,    help="Saved model weights directory",
                    default='0906/_0906_efficient_lite_v0_display-detection_e200_lr0.001_b32_without-norm-small_prior-adam_best_loss.h5')
parser.add_argument("--num_classes",         type=int,    help="Set num classes for model and post-processing",
                    default=4)
parser.add_argument("--image_size",          type=tuple,  help="Set image size for priors and post-processing",
                    default=(300, 300))
parser.add_argument("--gpu_num",             type=int,    help="Set GPU number to use(When without distribute training)",
                    default=0)
parser.add_argument("--frozen_dir",          type=str,    help="Path to save frozen graph transformation result",
                    default='./checkpoints/converted_frozen_graph/')
parser.add_argument("--frozen_name",         type=str,    help="Frozen graph file name to save",
                    default='frozen_graph')
parser.add_argument("--include_postprocess",   help="Frozen graph file name to save",
                    action='store_true')
parser.add_argument("--load_keras_model",   help="Load model from Saved format(.pb) or Keras(.h5)",
                    action='store_true')

args = parser.parse_args()

if __name__ == '__main__':
    tf.config.set_soft_device_placement(True)
    tf.config.run_functions_eagerly(True)

    gpu_number = '/device:GPU:' + str(args.gpu_num)
    with tf.device(gpu_number):

        spec_list = convert_spec_list()
        priors = create_priors_boxes(specs=spec_list, image_size=args.image_size[0], clamp=True)
        target_transform = MatchingPriors(priors, center_variance, size_variance, iou_threshold)

        if args.load_keras_model:
            model = tf.keras.models.load_model('./checkpoints/pruning', compile=False)

        else:
            model = ModelBuilder(image_size=args.image_size,
                                 num_classes=args.num_classes,
                                 include_preprocessing=args.include_postprocess).build_model(args.backbone_name)
            model.load_weights(args.checkpoint_dir + args.model_weights, by_name=True)

        model.summary()

        if args.include_postprocess:
            detection_output = merge_post_process(detections=model.output,
                                                  target_transform=target_transform,
                                                  confidence_threshold=0.5,
                                                  classes=args.num_classes)
            model = Model(inputs=model.input, outputs=detection_output)

        # model.summary()

        #path of the directory where you want to save your model
        frozen_out_path = args.frozen_dir
        # name of the .pb file
        frozen_graph_filename = args.frozen_name
        # Convert Keras model to ConcreteFunction
        full_model = tf.function(lambda x: model(x))
        full_model = full_model.get_concrete_function(
            tf.TensorSpec(model.inputs[0].shape, model.inputs[0].dtype))
        # Get frozen ConcreteFunction
        frozen_func = convert_variables_to_constants_v2(full_model)
        frozen_func.graph.as_graph_def()
        layers = [op.name for op in frozen_func.graph.get_operations()]

        print("Frozen model layers: ")
        # for layer in layers:
        #     print(layer)

        print("Frozen model inputs: {0}".format(frozen_func.inputs))
        print("Frozen model outputs: {0}".format(frozen_func.outputs))

        # Save frozen graph to disk
        tf.io.write_graph(graph_or_graph_def=frozen_func.graph,
                        logdir=frozen_out_path,
                        name=f"{frozen_graph_filename}.pb",
                        as_text=False)
        tf.io.write_graph(graph_or_graph_def=frozen_func.graph,
                        logdir=frozen_out_path,
                        name=f"{frozen_graph_filename}.pbtxt",
                        as_text=True)


import json
from PIL import Image
import os

def main(json_file_path):

    json_file = open (json_file_path)
    json_string = json_file.read()
    json_data = json.loads(json_string)

    image = json_data[0]
    #for image in json_data:
    image_file_path = image['image_path']
    image_to_crop = Image.open(image_file_path)

    cropped_images = []

    image_rects = image['rects']

    for r in image_rects:
        score = r['score']

        if score > 0.0:

            diff_x = r['x2']-r['x1']
            diff_y = r['y2']-r['y1']

            if diff_x < diff_y:
                wider_x = diff_y-diff_x + 10
                wider_y = 10
            else:
                wider_y = diff_x - diff_y + 10
                wider_x = 10

            new_image = image_to_crop.crop((r['x1']-wider_x/2, r['y1']-wider_y/2, r['x2']+wider_x/2, r['y2']+wider_y/2))

            #new_image = transform_image_to_square(new_image) #transform the image to square
            cropped_images.append(new_image)

    return cropped_images

def transform_image_to_square(img):
    longer_side = max(img.size)
    horizontal_padding = (longer_side - img.size[0]) / 2
    vertical_padding = (longer_side - img.size[1]) / 2

    square_img = img.crop(
        (
            -horizontal_padding,
            -vertical_padding,
            img.size[0] + horizontal_padding,
            img.size[1] + vertical_padding
        )

    )

    return square_img


import os
from logging import (
    CRITICAL,
    DEBUG,
    ERROR,
    getLogger,
    INFO,
    Logger,
    WARNING,
)
from pathlib import Path

from rich.logging import RichHandler
from rich.highlighter import NullHighlighter

from .config import BodyworkConfig
from .constants import (
    DEFAULT_LOG_LEVEL,
    DEFAULT_LOG_LEVEL_ENV_VAR,
    DEFAULT_PROJECT_DIR,
    LOG_TIME_FORMAT,
    PROJECT_CONFIG_FILENAME,
)
from .cli.terminal import console
from .exceptions import BodyworkConfigError

def bodywork_log_factory(
    log_level: str = None,
    config_file_path: Path = DEFAULT_PROJECT_DIR / PROJECT_CONFIG_FILENAME,
) -> Logger:
    """Create a standardised Bodywork logger.

    If a log level is specified as an argument, then it will take
    precedence overall all other methods of setting the log-level. Next
    in the waterfall of priority is the log-level set in the project
    config file, and then after that the level set by the
    BODYWORK_LOG_LEVEL environment variable. Failing that, the default
    log level (INFO) will be used.

    :param log_level: The minimum severity level of messages to log,
        defaults to None.
    :param config_file_path: Path to project config file, defaults
        DEFAULT_PROJECT_DIR/PROJECT_CONFIG_FILENAME.
    """
    log_level_mapping = {
        "DEBUG": DEBUG,
        "INFO": INFO,
        "WARNING": WARNING,
        "ERROR": ERROR,
        "CRITICAL": CRITICAL,
    }
    log = getLogger("bodywork")
    if log_level is not None:
        log.setLevel(log_level_mapping[log_level])
    else:
        try:
            bodywork_config = BodyworkConfig(config_file_path)
            log.setLevel(bodywork_config.logging.log_level)
        except BodyworkConfigError:
            try:
                log_level_from_env_var = os.environ[DEFAULT_LOG_LEVEL_ENV_VAR]
                log.setLevel(log_level_mapping[log_level_from_env_var])
            except KeyError:
                log.setLevel(log_level_mapping[DEFAULT_LOG_LEVEL])
    if not log.hasHandlers():
        log_handler = RichHandler(
            console=console,
            show_path=False,
            highlighter=NullHighlighter(),
            omit_repeated_times=False,
            rich_tracebacks=True,
            log_time_format=LOG_TIME_FORMAT,
        )
        log.addHandler(log_handler)
    return log


#! /usr/bin/env python
# -*- coding:utf-8 -*-
"""
@author  : MG
@Time    : 19-4-3 下午5:28
@File    : __init__.py.py
@contact : mmmaaaggg@163.com
@desc    :
"""
import logging
from logging.config import dictConfig

# log settings
logging_config = dict(
    version=1,
    formatters={
        'simple': {
            'format': '%(asctime)s %(levelname)s %(name)s %(filename)s.%(funcName)s:%(lineno)d|%(message)s'}
    },
    handlers={
        'file_handler':
            {
                'class': 'logging.handlers.RotatingFileHandler',
                'filename': 'logger.log',
                'maxBytes': 1024 * 1024 * 10,
                'backupCount': 5,
                'level': 'DEBUG',
                'formatter': 'simple',
                'encoding': 'utf8'
            },
        'console_handler':
            {
                'class': 'logging.StreamHandler',
                'level': 'DEBUG',
                'formatter': 'simple'
            }
    },

    root={
        'handlers': ['console_handler', 'file_handler'],
        'level': logging.DEBUG,
    }
)
# logging.getLogger('sqlalchemy.engine').setLevel(logging.WARN)
# logging.getLogger('urllib3.connectionpool').setLevel(logging.INFO)
dictConfig(logging_config)

if __name__ == "__main__":
    pass