import itertools
def permutation(s):
    if(not s):
        return None
    if(len(s) == 1):
        return list(s)
    l = list(s)
    l.sort()
    result = []
    for i in range(len(l)):
        if(i > 0 and l[i] == l[i-1]):
            continue
        part2 = permutation(''.join(l[:i]) +''.join(l[i+1:]))
        for j in part2:
            result.append(l[i]+j)
    return result

def permutation2(s):
    result = [''.join(x) for x in itertools.permutations(s, len(s))]
    return list(result)
if __name__ == '__main__':
    s = 'abc'
    result = permutation(s)
    for i in result:
        print(i)
    result = permutation2(s)
    for i in result:
        print(i)


import os
import yara
import uuid

from loguru import logger

from vigil.schema import YaraMatch
from vigil.schema import ScanModel
from vigil.schema import BaseScanner

from vigil.registry import Registration

@Registration.scanner(name='yara', requires_config=True)
class YaraScanner(BaseScanner):
    def __init__(self, rules_dir: str):
        self.name = 'scanner:yara'
        self.rules_dir = rules_dir
        self.compiled_rules = None

        if not os.path.exists(self.rules_dir):
            logger.error(f'Directory not found: {self.rules_dir}')
            raise Exception

        if not os.path.isdir(self.rules_dir):
            logger.error(f'Path is not a valid directory: {self.rules_dir}')
            raise Exception

        self.load_rules()
        logger.success('Loaded scanner')

    def load_rules(self) -> bool:
        """Compile all YARA rules in a directory and store in memory"""
        logger.info(f'Loading rules from directory: {self.rules_dir}')
        rules = os.listdir(self.rules_dir)

        if len(rules) == 0:
            return

        yara_paths = {}
        for _file in rules:
            if self.is_yara_file(_file):
                yara_paths[_file] = os.path.join(self.rules_dir, _file)

        try:
            self.compiled_rules = yara.compile(filepaths=yara_paths)
        except Exception as err:
            logger.error(f'YARA compilation error: {err}')
            raise err

    def is_yara_file(self, file_path: str) -> bool:
        """Check if file is rule by extension"""
        if file_path.lower().endswith('.yara') or file_path.lower().endswith('.yar'):
            return True
        return False

    def analyze(self, scan_obj: ScanModel, scan_id: uuid.uuid4) -> ScanModel:
        """Run scan against input data and return list of YaraMatchs"""
        logger.info(f'Performing scan; id="{scan_id}"')

        if scan_obj.prompt.strip() == '':
            logger.error(f'No input data; id="{scan_id}"')
            return scan_obj

        try:
            matches = self.compiled_rules.match(data=scan_obj.prompt)
        except Exception as err:
            logger.error(f'Failed to perform yara scan; id="{scan_id}" error="{err}"')
            return scan_obj

        for match in matches:
            m = YaraMatch(rule_name=match.rule, tags=match.tags, category=match.meta.get('category', None))
            logger.warning(f'Matched rule rule="{m.rule_name} tags="{m.tags}" category="{m.category}"')
            scan_obj.results.append(m)

        if len(scan_obj.results) == 0:
            logger.info(f'No matches found; id="{scan_id}"')

        return scan_obj


import os

from molotov.ui.app import MolotovApp
from molotov.util import printable_error

class Console:
    def __init__(
        self,
        interval=0.3,
        max_lines_displayed=25,
        simple_console=False,
        single_process=True,
    ):
        self._interval = interval
        self._stop = True
        self._creator = os.getpid()
        self._stop = False
        self._max_lines_displayed = max_lines_displayed
        self._simple_console = simple_console
        self.ui = MolotovApp(
            refresh_interval=interval,
            max_lines=max_lines_displayed,
            simple_console=simple_console,
            single_process=single_process,
        )
        self.terminal = self.ui.terminal
        self.errors = self.ui.errors
        self.status = self.ui.status
        self.started = False

    async def start(self):
        await self.ui.start()
        self.started = True

    async def stop(self):
        await self.ui.stop()
        self.started = False

    def print_results(self, results):
        self.status.update(results)

    def print(self, data):
        if self.terminal is None:
            return
        for line in data.split("\n"):
            line = line.strip()
            self.terminal.write_line(line)

    def print_error(self, error, tb=None):
        if self.errors is None:
            return
        if isinstance(error, str):
            for line in error.split("\n"):
                line = line.strip()
                self.errors.write_line(line, fg="gray")
            return

        for line in printable_error(error, tb):
            self.errors.write_line(line, fg="gray")

        self.errors.write_line("", fg="gray")

    def print_block(self, start, callable, end="OK"):
        if self.terminal is None:
            return callable()

        self.terminal.write_line(f"{start}...")
        try:
            return callable()
        finally:
            self.terminal.write_line("OK")


import torch

# make vectors unit length
def normalize(vec, dim, keepLength=False, eps=0):
  length = vec.norm(dim=dim, keepdim=True)
  unit = vec / (length + eps)
  if keepLength:
    return unit, length
  return unit

# compute a gradient, possibly with AMP
def grad(inputs, outputs, gradScaler=None):
  if gradScaler is not None:
    outputs = gradScaler.scale(outputs)
  out = torch.autograd.grad(outputs=outputs,
                            inputs=inputs,
                            grad_outputs=torch.ones_like(outputs),
                            create_graph=True)[0]
  if gradScaler is not None:
    return out / gradScaler.get_scale()
  return out

# uniform random value in a range
def rand_range(rnge, shape, device):
  a, b = rnge
  return torch.rand(shape, device=device) * (b - a) + a

# iterate over count objects in batches, where count possibly not divisible by batch
def iterBatch(count, batch):
  steps = count // batch
  for i in range(steps + 1):
    if i == steps:
      if count % batch == 0:
        break
      batch = count % batch

    yield i * batch, batch


from keras import backend as keras_back
from keras.layers import Input
import tensorflow as tf
import keras

import efficientnet.keras as efn
from keras.applications.resnet50 import ResNet50
# Todo - Testar a resnet 50 V2
from tensorflow.keras.applications.resnet_v2 import ResNet50V2
from tensorflow.keras.applications.resnet_v2 import ResNet101V2
from tensorflow.keras.applications.resnet_v2 import ResNet152V2
from keras.applications import InceptionV3, InceptionResNetV2, Xception
from keras.layers import (Layer,
                          Input, Dense, Dropout, Lambda, BatchNormalization,
                          Flatten, Add, Concatenate,
                          Activation, LeakyReLU,
                          Conv1D, Conv2D, Conv3D,
                          Conv2DTranspose, Conv3DTranspose,
                          SeparableConv1D, SeparableConv2D,
                          MaxPooling1D, MaxPooling2D, MaxPooling3D,
                          GlobalAveragePooling1D, GlobalAveragePooling2D,
                          GlobalAveragePooling3D)

from utils import utils
from icecream import ic

def get_model_effnet(x_data, weights, effnet_version):
    keras_back.set_image_data_format('channels_last')
    img_shape = (x_data.shape[1], x_data.shape[2], x_data.shape[3])
    ic("Input Shape Matrix: ", img_shape)
    # ic("X_data Shape: 1- {}, 2- {}, 3- {}".format(x_data.shape[1], x_data.shape[2], x_data.shape[3]))
    img_input = Input(shape=img_shape)

    ic('\n ** Utilizando a Rede EfficientNet ', effnet_version)
    ic('\n ** Pesos carregados: ', weights)

    if effnet_version == 'B0':
        effnet = efn.EfficientNetB0(include_top=False, input_tensor=img_input, weights=weights, pooling=None,
                                    input_shape=img_shape)

    elif effnet_version == 'B1':
        effnet = efn.EfficientNetB1(include_top=False, input_tensor=img_input, weights=weights, pooling=None,
                                    input_shape=img_shape)

    elif effnet_version == 'B2':
        effnet = efn.EfficientNetB2(include_top=False, input_tensor=img_input, weights=weights, pooling=None,
                                    input_shape=img_shape)

    elif effnet_version == 'B3':
        effnet = efn.EfficientNetB3(include_top=False, input_tensor=img_input, weights=weights, pooling=None,
                                    input_shape=img_shape)

    elif effnet_version == 'B4':
        effnet = efn.EfficientNetB4(include_top=False, input_tensor=img_input, weights=weights, pooling=None,
                                    input_shape=img_shape)

    elif effnet_version == 'B5':
        effnet = efn.EfficientNetB5(include_top=False, input_tensor=img_input, weights=weights, pooling=None,
                                    input_shape=img_shape)

    elif effnet_version == 'B6':
        effnet = efn.EfficientNetB6(include_top=False, input_tensor=img_input, weights=weights, pooling=None,
                                    input_shape=img_shape)

    else:
        effnet = efn.EfficientNetB7(include_top=False, input_tensor=img_input, weights=weights, pooling=None,
                                    input_shape=img_shape)

    # flat = tf.keras.layers.Flatten()(res_net.output)
    avg_pool = GlobalAveragePooling2D()(effnet.output)
    # dropout = tf.keras.layers.Dropout(0.5)(avg_pool)
    y_hat = Dense(2, activation="sigmoid")(avg_pool)
    model = keras.models.Model(effnet.input, y_hat)

    return model

def compile_model(name_file_rede, model, opt, fold, version):
    ic('\n Compilando rede: ', name_file_rede)
    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])

    ic('\n ** Plotting model and callbacks...')
    name_weights = 'Train_model_weights_%s_{epoch:02d}_%s.h5' % (name_file_rede, version)
    csv_name = 'training_{}_fold_{}_ver_{}.csv'.format(name_file_rede, fold, version)
    callbacks = utils.get_callbacks(name_weights=name_weights, patience_lr=10, name_csv=csv_name)

    return callbacks

def fit_model(model, generator, x_data_cv, batch_size, num_epochs, x_val_cv, y_val_cv, callbacks):
    history = model.fit_generator(
        generator,
        steps_per_epoch=len(x_data_cv) / batch_size,
        epochs=num_epochs,
        verbose=1,
        validation_data=(x_val_cv, y_val_cv),
        validation_steps=len(x_val_cv) / batch_size,
        callbacks=callbacks)

    return history

def get_model_resnet(x_data, weights, resnet_depth):
    keras_back.set_image_data_format('channels_last')
    img_shape = (x_data.shape[1], x_data.shape[2], x_data.shape[3])
    ic('Input Shape Matrix: ', img_shape)
    # ic('X_data Shape: 1- {}, 2- {}, 3- {}'.format(x_data.shape[1], x_data.shape[2], x_data.shape[3]))
    img_input = Input(shape=img_shape)

    ic('\n ** Utilizando a Rede Resnet', resnet_depth)
    ic('\n ** Pesos carregados: ', weights)

    if resnet_depth == 50:

        res_net = ResNet50(include_top=False, weights=weights, input_tensor=img_input, input_shape=img_shape,
                           pooling=None)
        avg_pool = GlobalAveragePooling2D()(res_net.output)
        # dropout = tf.keras.layers.Dropout(0.5)(flat)
        # y_hat = tf.keras.layers.Dense(2, activation='sigmoid')(avg_pool)
        y_hat = Dense(2, activation='softmax')(avg_pool)
        model = keras.models.Model(res_net.input, y_hat)

    elif resnet_depth == 101:
        res_net = ResNet101V2(include_top=False, weights=weights, input_tensor=img_input, input_shape=img_shape,
                              pooling=None)
        avg_pool = GlobalAveragePooling2D()(res_net.output)
        # dropout = tf.keras.layers.Dropout(0.5)(flat)
        # y_hat = tf.keras.layers.Dense(2, activation='sigmoid')(avg_pool)
        y_hat = Dense(2, activation='softmax')(avg_pool)
        model = keras.models.Model(res_net.input, y_hat)

    else:
        res_net = ResNet152V2(include_top=False, weights=weights, input_tensor=img_input, input_shape=img_shape,
                              pooling=None)
        avg_pool = GlobalAveragePooling2D()(res_net.output)
        # dropout = tf.keras.layers.Dropout(0.5)(flat)
        # y_hat = tf.keras.layers.Dense(2, activation='sigmoid')(avg_pool)
        y_hat = Dense(2, activation='softmax')(avg_pool)
        model = keras.models.Model(res_net.input, y_hat)

    return model

def get_model_inception(x_data, weights, version):
    keras_back.set_image_data_format('channels_last')
    img_shape = (x_data.shape[1], x_data.shape[2], x_data.shape[3])
    ic('Input Shape Matrix: ', img_shape)
    # ic('X_data Shape: 1- {}, 2- {}, 3- {}'.format(x_data.shape[1], x_data.shape[2], x_data.shape[3]))
    img_input = Input(shape=img_shape)

    ic('\n ** Utilizando a Inception', version)
    ic('\n ** Pesos carregados: ', weights)

    if version == 'V2':
        res_net = InceptionResNetV2(include_top=False, weights=weights, input_tensor=img_input, input_shape=img_shape, pooling=None)
    elif version == 'V3':
        res_net = InceptionV3(include_top=False, weights=weights, input_tensor=img_input, input_shape=img_shape, pooling=None)

    avg_pool = GlobalAveragePooling2D()(res_net.output)
    y_hat = Dense(2, activation='softmax')(avg_pool)
    model = keras.models.Model(res_net.input, y_hat)

    return model

def get_model_xception(x_data, weights):
    keras_back.set_image_data_format('channels_last')
    img_shape = (x_data.shape[1], x_data.shape[2], x_data.shape[3])
    ic('Input Shape Matrix: ', img_shape)
    # ic('X_data Shape: 1- {}, 2- {}, 3- {}'.format(x_data.shape[1], x_data.shape[2], x_data.shape[3]))
    img_input = Input(shape=img_shape)

    ic('\n ** Utilizando a Xception')
    ic('\n ** Pesos carregados: ', weights)

    res_net = Xception(include_top=False, weights=weights, input_tensor=img_input, input_shape=img_shape, pooling=None)

    avg_pool = GlobalAveragePooling2D()(res_net.output)
    y_hat = Dense(2, activation='softmax')(avg_pool)
    model = keras.models.Model(res_net.input, y_hat)

    return model


import json
import codecs
import rpy2.robjects as robjects
from rpy2.robjects.packages import importr

#IMPORT LIBRARIES
wnom = importr('wnominate')
imjson = importr('RJSONIO')
r = robjects.r

#READ ARCHIVE
votaciones = r['read.csv']('./Data/nSalida.csv', encoding='utf-8')
votaciones2 =r['as.matrix'](votaciones).rx(True,-1).rx(True,-1).rx(True,-1) #Quitar 3 Columnas

#SAVE DATA
iD = r['matrix'](votaciones.rx(True,1))  #Rescatar Columna de iD
nombres = r['matrix'](votaciones.rx(True,2)) #Rescatar Columna de nombres
partidos = r['matrix'](votaciones.rx(True,3)) # #Rescatar Columna de partidos

#FORMAT WNOMINATE
rc = r.rollcall(votaciones2, yea = [1,2,3], nay = [4,5,6],
                missing = [7,8,9], notInLegis = 0,)
#Inicio de Calculos
result = wnom.wnominate(rc,polarity=robjects.IntVector((1,90)),minvotes = 10)
#Muestra Resultados
sum =r.summary(result)

#rescatar Coordenadas
cx = result[0][6]
cy = result[0][7]

#CREATE AND EXPORTATION ARCHIVE JSON
objects = {}
objects['Legislatura'] = []
for object in range(len(nombres)):
    objects['Legislatura'].append(
    {
        'Id_P':''+str(iD[object])+'',
        'Nombre': ''+ nombres[object]+'',
        'Partido': ''+partidos[object]+'',
        'X': ''+str(cx[object])+'',
        'Y': ''+str(cy[object])+''
        })
with open('./Data/Datos.json', 'w') as file:
    json.dump(objects, file, encoding='latin1')

new/Pruebas R/Prueba 5 -Stress/new-Rnominate.py


import os
import trackhub
import glob
from config import genome_version, data_load_sources_supported, data_load_sources_supported_short_form, \
    ucsc_track_visibility
from binding_analysis_binding_sites import overlap_conflict
from data_load_functions import column_data

def populate_local_track_hub(overarching_path, rbp, rna_info, local_stage, rbp_no_dict, rbp_peaks):
    [RNA, RNA_chr_no, _, _] = rna_info

    f = open(overarching_path + "threshold_config.txt")
    _str = f.read()
    competitive_threshold_bp = _str.split("\n")[0].split()[-1]
    cooperative_threshold_bp = _str.split("\n")[1].split()[-1]

    print(overlap_conflict)

    rnas = RNA
    proteins = rbp
    hub_name = "RBPs on " + rnas + " " + overlap_conflict
    hub, genomes_file, genome, trackdb = trackhub.default_hub(
        hub_name=hub_name,
        short_label=("RBPs on " + rnas + " w.r.t. " + proteins + ": " + overlap_conflict),
        long_label=("RNA binding proteins on long non-coding RNAs " +
                    rnas + " with respect to the binding sites of " + proteins +
                    ". The red sites are the places where proteins have " +
                    "competitive binding with " + proteins + ", whereas green sites are " +
                    "places where cooperative binding occurs. The thresholds " +
                    "used for competitive binding was 0bp to " +
                    str(competitive_threshold_bp) + "bp and " +
                    str(competitive_threshold_bp) + "bp to " +
                    str(cooperative_threshold_bp) + "bp for cooperative binding"),
        genome=genome_version,
        email="mnk1@andrew.cmu.edu")

    print("Hub set up")
    for filename in glob.iglob(overarching_path + "**/*.bb", recursive=True):
        dots, directory1, directory2, category, name = filename.split("/")
        rbp = name.split("_")[0]
        rbp = rbp.replace(",", "_")
        rbp = rbp.replace("*", "_mut_")
        rbp = rbp.replace("(", "")
        rbp = rbp.replace(")", "")

        visibility = ucsc_track_visibility
        # TODO: consider options for this for the user (add to Config at least)

        track = trackhub.Track(
            name=rbp + "_" + category + "binding_sites",
            short_label=rbp + "_" + category,
            long_label=("Binding sites of " + rbp + " derived from " +
                        data_load_sources_supported[data_load_sources_supported_short_form.index(category)]),
            source=filename,
            tracktype='bigBed 9 +',
            itemRgb="on",
            spectrum="on",
            visibility=visibility,
            chromosomes="chr" + str(RNA_chr_no),
            # labelFields="",
            # defaultLabelFields="",
            # mouseOverField=""
            # labelFields=",".join([column_data[category]["names"][i] for i in column_data[category]["interest"]]),
            # defaultLabelFields=",".join(
            #     [column_data[category]["names"][i] for i in column_data[category]["default_label"]]),
            # mouseOverField=column_data[category]["names"][column_data[category]["default_mouse_over"]],
            # maxItems=25

        )
        # TODO: Add options for mouse hover views of information

        trackdb.add_tracks(track)
    for filename in glob.iglob(overarching_path + "**/*.bw", recursive=True):
        dots, directory1, directory2, data_load_source, name = filename.split("/")
        rbp_no = rbp_no_dict[data_load_source]
        rbp_peak = max(rbp_peaks.values())
        rbp_peak = (rbp_peak // 10 + 1) * 10
        visibility = "full"
        # TODO: consider options for this for the user (add to Config at least)

        track = trackhub.Track(
            name=RNA + "_" + data_load_source + "_density_plot",
            short_label="00 " + RNA + "_density",
            long_label=("Density plot of " + str(rbp_no) + " RBPs on " + RNA + " using data from " +
                        data_load_source.upper()),
            source=filename,
            tracktype='bigWig',
            color="128,0,0",  # TODO: what color ought bigWig density plots be?
            visibility=visibility,
            chromosomes="chr" + str(RNA_chr_no),
            viewLimits="0:" + str(rbp_peak),
            maxHeightPixels="128:50:8",
            autoScale="on"
        )

        trackdb.add_tracks(track)
    trackhub.upload.stage_hub(hub, staging=local_stage)
    return hub_name

def prepare_auto_sql(data_load_source):
    source_columns_of_interest = range(len(column_data[data_load_source]["names"]))
    no_of_extra_fields = len(source_columns_of_interest)
    name_of_file = data_load_source + "".join([str(c) for c in source_columns_of_interest]) + ".as"
    file_path = "../autosql_files/" + name_of_file
    template_file_path = "../autosql_files/general_template.as"
    try:
        open(file_path, 'r').close()
    except FileNotFoundError:
        with open(file_path, 'w') as handle:
            # TODO: make an auto generator of auto_sql template files here
            with open(template_file_path, "r") as template_handle:
                template_string = template_handle.read()
                template_string = template_string.replace("insert_source_name_here", data_load_source)

            handle.write(template_string)
            column_names = [column_data[data_load_source]["names"][i] for i in source_columns_of_interest]
            descriptions = [column_data[data_load_source]["descriptions"][i] for i in source_columns_of_interest]
            additional_str = ""
            for column, description in zip(column_names, descriptions):
                additional_str += "\t".join(["lstring", column + ";", '"' + description + '"']) + "\n"
            additional_str += ")"
            handle.write(additional_str)
    return no_of_extra_fields, name_of_file

def convert_bed_to_bb(overarching_path, data_load_sources):
    debug = False
    if genome_version != "hg38":
        raise ValueError("Update this function for this genome version!")

    CUR = os.getcwd()
    for data_load_source in data_load_sources:
        no_of_extra_fields, as_file_name = prepare_auto_sql(data_load_source)
        os.chdir(overarching_path + data_load_source + "/")

        os.system(
            "for file in * .bed; do ../../../ucsc-tools/bedToBigBed -as=../../../autosql_files/" + as_file_name +
            " type=bed9+" + str(no_of_extra_fields) + ' "$file" ' +
            '../../../ucsc-tools/hg38.chrom.sizes "$file.bb"; done' + (' >/dev/null 2>&1' if not debug else ''))
        os.chdir(CUR)
    return

def upload_online(local_dir, github_dir):
    os.system("rsync -avzL " + local_dir + " " + github_dir + " >/dev/null 2>&1")
    os.chdir(github_dir)
    os.system("git add . >/dev/null 2>&1; git commit -m 'update hub' >/dev/null 2>&1; git push origin >/dev/null 2>&1")
    return

def density_plot(big_storage, rna_info, data_load_sources, overarching_path, return_rbp_no=False):
    [RNA, RNA_chr_no, RNA_start_chr_coord, _] = rna_info
    rbp_no_dict = {}
    for data_load_source in data_load_sources:
        print("")
        print("Starting with", data_load_source, "data...")

        storage = big_storage[data_load_source]
        # TODO: check if displacement needs to be shifted by one for all data sources or just RBPDB
        wig_string = storage.print_wig(chr_no=RNA_chr_no, displacement=RNA_start_chr_coord - 1, include_name=True,
                                       include_description=True, name=RNA,
                                       description="Density plot of " + str(len(storage)) + " RBPs on " + RNA,
                                       include_header=True)

        rbp_no_dict[data_load_source] = len(storage)

        folder_path = overarching_path + data_load_source + "/"
        filepath = RNA + "_" + data_load_source + "_" + genome_version + "_density_plot.wig"
        filepath = folder_path + filepath

        try:
            f = open(filepath, "w")
        except FileNotFoundError:
            os.makedirs(folder_path)
            f = open(filepath, "w")

        f.write(wig_string)
        f.close()

    if return_rbp_no:
        return rbp_no_dict

def convert_wig_to_bw(overarching_path, data_load_sources):
    if genome_version != "hg38":
        raise ValueError("Update this function for this genome version!")

    CUR = os.getcwd()
    for data_load_source in data_load_sources:
        os.chdir(overarching_path + data_load_source + "/")
        os.system(
            '''for file in *.wig; do ../../../ucsc-tools/wigToBigWig "$file" ../../../ucsc-tools/hg38.chrom.sizes ''' +
            '''"$file.bw"; done >/dev/null 2>&1'''
        )
        os.chdir(CUR)
    return


# -*- coding: utf-8 -*-
"""
Created on Wed Jun 23 19:47:03 2021

@author: DELL
"""

import numpy as np
# import matplotlib.pyplot as plt
# from mpl_toolkits.mplot3d import Axes3D
from my_spectrum_analyzer import img_spectrum_analyzer
import torch
from torchvision import transforms #datasets, models,
from tqdm import tqdm
import os
import sys
import torch.nn as nn
import time
from models.cifar.allconv import AllConvNet
from models.resnet import resnet50
from models.vgg import vgg16_bn
from models.cifar.allconv import AllConvNet
import torchvision.models as models
from third_party.ResNeXt_DenseNet.models.densenet import densenet
from third_party.ResNeXt_DenseNet.models.resnext import resnext29
from third_party.WideResNet_pytorch.wideresnet import WideResNet
from art.attacks.evasion import FastGradientMethod,DeepFool
from art.attacks.evasion import CarliniL2Method,CarliniLInfMethod
from art.attacks.evasion import ProjectedGradientDescent
# from art.attacks.evasion import UniversalPerturbation
from art.estimators.classification import PyTorchClassifier
import json
sys.path.append("..")
# from train_code.my_img_transformer import img_transformer
# from mpl_toolkits.mplot3d import Axes3D
# from sklearn.manifold import TSNE
sys.path.append('../common_code')
from load_cifar_data import load_CIFAR_batch,load_CIFAR_train

class img_spectrum_labeler:

    # 解释器初始化
    def __init__(self,dataset):
        if 'imagenet'==dataset:
            self.img_size=224
            mean_now=[0.485, 0.456, 0.406]
            std_now=[0.229, 0.224, 0.225]
            self.num_classes=1000

        elif 'cifar-10'==dataset:
            self.img_size=32
            mean_now=[0.5] * 3
            std_now=[0.5] * 3
            self.num_classes=10
        else:
            print('ERROR DATASET')

        self.trans=transforms.Compose([transforms.Normalize(mean=mean_now, std=std_now)])
        self.s_analyzer=img_spectrum_analyzer(self.img_size).batch_get_spectrum_feature#batch_get_spectrum_energy

    def select_attack(self, fmodel, attack_idx, eps_idx):
        attack_names=['FGSM_L2_IDP','PGD_L2_IDP','CW_L2_IDP','Deepfool_L2_IDP','FGSM_Linf_IDP','PGD_Linf_IDP','CW_Linf_IDP']
        eps_L2=[0.1,1.0,10.0,100.0]
        eps_Linf=[0.01,0.1,1.0,10.0]

        att_method=attack_names[attack_idx]
        if 'L2' in att_method:
            eps=float(eps_L2[eps_idx%len(eps_L2)])
        else:
            eps=float(eps_Linf[eps_idx%len(eps_Linf)])

        if att_method   == 'FGSM_L2_IDP':
            attack = FastGradientMethod(estimator=fmodel,eps=eps,norm=2)
        elif att_method == 'PGD_L2_IDP':
            attack = ProjectedGradientDescent(estimator=fmodel,eps=eps,norm=2,batch_size=512,verbose=False)
        elif att_method == 'CW_L2_IDP':
            attack = CarliniL2Method(classifier=fmodel,batch_size=512,verbose=False)
        elif att_method == 'Deepfool_L2_IDP':
            attack = DeepFool(classifier=fmodel,batch_size=512,verbose=False)

        elif att_method == 'FGSM_Linf_IDP':
            attack = FastGradientMethod(estimator=fmodel,eps=eps,norm=np.inf)
        elif att_method == 'PGD_Linf_IDP':
            attack = ProjectedGradientDescent(estimator=fmodel,eps=eps,norm=np.inf,batch_size=512,verbose=False)
        elif att_method == 'CW_Linf_IDP':
            attack = CarliniLInfMethod(classifier=fmodel,eps=eps,batch_size=512,verbose=False)

        else:
            raise Exception('Wrong Attack Mode: {} !!!'.format(att_method))
        return attack, eps

    def get_energy_label(self, model, imgs_in, labels_in, is_adv):
        assert imgs_in.shape[-2]==imgs_in.shape[-1]
        spectrum  = self.s_analyzer(imgs_in)
        labels_out = is_adv * np.ones(labels_in.shape)
        return spectrum,labels_out.reshape((-1,1))

if __name__=='__main__':
    '''
    settings
    '''
    # 配置解释器参数
    if len(sys.argv)!=4:
        print('Manual Mode !!!')
        model_type    = 'allconv'
        data          = 'train'
        device        = 2
    else:
        print('Terminal Mode !!!')
        model_type  = sys.argv[1]
        data        = sys.argv[2]
        device      = int(sys.argv[3])

    '''
    加载cifar-10图像
    '''
    os.environ['CUDA_VISIBLE_DEVICES']=str(device)
    dir_cifar     = '../../../../../media/ubuntu204/F/Dataset/Dataset_tar/cifar-10-batches-py'
    if 'test'==data:
        images,labels = load_CIFAR_batch(os.path.join(dir_cifar,'test_batch'))
    elif 'train'==data:
        images,labels = load_CIFAR_train(dir_cifar)
    else:
        print('Wrong data mode !!!')

    '''
    加载模型
    '''
    dataset='cifar-10'
    dir_model = '../models/cifar_vanilla_'+model_type+'.pth.tar'
    if model_type == 'resnet50_imagenet':
        model = models.resnet50(pretrained=True).eval()
        model = torch.nn.DataParallel(model).cuda()
        dataset = 'imagenet'
    elif model_type == 'vgg16_imagenet':
        model = models.vgg16(pretrained=True).eval()
        model = torch.nn.DataParallel(model).cuda()
        dataset = 'imagenet'
    elif model_type == 'alexnet_imagenet':
        model = models.alexnet(pretrained=True).eval()
        model = torch.nn.DataParallel(model).cuda()
        dataset = 'imagenet'
    elif model_type == 'resnet50':
        model = resnet50().eval()
        model = torch.nn.DataParallel(model).cuda()
        checkpoint = torch.load(dir_model)
        model.load_state_dict(checkpoint["state_dict"],True)
    elif model_type == 'vgg16':
        model = vgg16_bn().eval()
        model = torch.nn.DataParallel(model).cuda()
        checkpoint = torch.load(dir_model)
        model.load_state_dict(checkpoint["state_dict"],True)
    elif model_type == 'allconv':
        model = AllConvNet(10).eval()
        model = torch.nn.DataParallel(model).cuda()
        checkpoint = torch.load(dir_model)
        model.load_state_dict(checkpoint["state_dict"],True)
    else:
        raise Exception('Wrong model name: {} !!!'.format(model_type))
    model.eval()

    if 'cifar-10'==dataset:
        mean   = np.array((0.5,0.5,0.5),dtype=np.float32)
        std    = np.array((0.5,0.5,0.5),dtype=np.float32)
        nb_classes = 10
        input_shape=(3,32,32)
        rober_np= np.array([[0,1],[0,2],[0,3],
                        [1,1],[1,2],[1,3],
                        [8,1],[8,2],[8,3],
                        [9,1],[9,2],[9,3],
                        [4,1],[4,2],[4,3],
                        [5,1],[5,2],[5,3],
                        [12,1],[12,2],[12,3],
                        [13,1],[13,2],[13,3],
                       ]) # 被标为rober的设置
    elif 'imagenet'==dataset:
        mean   = np.array((0.485, 0.456, 0.406),dtype=np.float32)
        std    = np.array((0.229, 0.224, 0.225),dtype=np.float32)
        nb_classes = 1000
        input_shape=(3,224,224)
        rober_np= np.array([[0,1],[0,2],[0,3],
                        [1,1],[1,2],[1,3],
                        [8,1],[8,2],[8,3],
                        [9,1],[9,2],[9,3],
                        [4,1],[4,2],[4,3],
                        [5,1],[5,2],[5,3],
                        [12,1],[12,2],[12,3],
                        [13,1],[13,2],[13,3],
                       ]) # 被标为rober的设置
    else:
        raise Exception('Wrong dataset type: {} !!!'.format(dataset))

    fmodel = PyTorchClassifier(model = model,nb_classes=nb_classes,clip_values=(0,1),
                               input_shape=input_shape,loss = nn.CrossEntropyLoss(),
                               preprocessing=(mean, std))

    '''
    读取数据
    '''
    labeler = img_spectrum_labeler(dataset)
    # fft_transformer = img_transformer(8,0,6)

    batch           = 10
    batch_num       = int(len(labels)/batch)
    spectrums_list  = []
    labels_list     = []
    start_time = time.time()
    for i in tqdm(range(batch_num)):
        '''
        攻击与防护
        '''
        images_batch = images[batch*i:batch*(i+1),...].transpose(0,3,1,2)
        labels_batch = labels[batch*i:batch*(i+1),...]

        # 标为对抗样本
        attack_name=np.random.randint(1)
        attack_eps=np.random.randint(4)

        attack,eps=labeler.select_attack(fmodel,attack_name, attack_eps)
        images_adv = attack.generate(x=images_batch,y=labels_batch)

        # 根据列表标为有害或无害样本
        flag_rober=0
        for m in range(rober_np.shape[0]):
            if (attack_name==rober_np[m,0]) and (attack_eps==rober_np[m,1]):
                flag_rober=1
                break

        spectrums_save,labels_save=labeler.get_energy_label(model, images_adv, labels_batch, flag_rober)
        spectrums_list.append(spectrums_save)
        labels_list.append(labels_save)

    sub_dir='spectrum_label/'+model_type
    saved_dir_path  = '../saved_tests/img_attack/'+sub_dir
    if not os.path.exists(saved_dir_path):
        os.makedirs(saved_dir_path)
    spectrums_np=np.vstack(spectrums_list)
    labels_np=np.vstack(labels_list)

    np.save(os.path.join(saved_dir_path,'spectrums_'+data+'.npy'), spectrums_np)
    np.save(os.path.join(saved_dir_path,'labels_'+data+'.npy'), labels_np)
    end_time=time.time()
    print(("Time %f s")%(end_time-start_time))


from aiogram.types import InlineKeyboardButton, InlineKeyboardMarkup, ReplyKeyboardMarkup, KeyboardButton, ReplyKeyboardRemove

async def kb_return() -> ReplyKeyboardMarkup:

    kb = InlineKeyboardMarkup()
    button = InlineKeyboardButton("Вернуться", callback_data="return")
    kb.add(button)

    return kb

async def kb_remove():
    return ReplyKeyboardRemove()


from typing import Union
from fastapi import Depends, HTTPException, Response, status
from fastapi.security import HTTPBearer

from src.database import AsyncSession, get_db_session
from src import models
from src.utils import VerifyToken
from src.crud.crud_user import UserCRUD

token_auth_scheme = HTTPBearer()

async def get_current_user(
        response: Response,
        user_crud: UserCRUD = Depends(),
        token: str = Depends(token_auth_scheme)
) -> Union[models.User, Response]:
    pyload_from_auth = await VerifyToken(token.credentials).verify_token_from_auth0()

    if pyload_from_auth.get("status"):
        pyload_from_me = await VerifyToken(token.credentials).verify_token_from_me()
        if pyload_from_me.get("status"):
            response.status_code = status.HTTP_400_BAD_REQUEST
            return response

        user = await user_crud.get_user_by_email(email=pyload_from_me.get("email"))
        if not user:
            raise HTTPException(status_code=404, detail="User not found")
        return user

    user = await user_crud.get_user_by_email(email=pyload_from_auth.get("email"))
    if not user:
        user = await user_crud.create_user_by_email(email=pyload_from_auth.get("email"))
    return user


"""
Driver for intel realsense devices. This file contains simple wrappers for functions provided by Intel SDK.
https://github.com/IntelRealSense/librealsense/tree/master/wrappers/python/examples
https://dev.intelrealsense.com/docs/python2

some names of variables and functions come Intel SDK

Currently supports:
Intel LiDAR L515
Intel Depth Camera D435i
"""

from matplotlib import image
import numpy as np
import time
import pyrealsense2 as rs

DEPTH = "depth"
COLOR = "color"
INFRARED = "infrared"
GYRO = "gyro"
ACCEL = "accel"
IMAGE = "images"
FRAMEN = "frameN"

DEPTHCHANNEL = 0
COLORCHANNEL = 1
INFARAREDCHANNEL = 2
ACCELCHANNEL = 3
GYROCHANNEL = 4
FPS = "fps"

class Driver():
    """
    """

    def __init__(self):
        """
        Create a context object. This object owns the handles to all connected realsense devices
        """
        self.pipeline = {}
        self.profile = {}
        self.conf = {}
        self.config_dict = {}

    def init(self, config_dict):
        """
        Method for Initalizing camera
        """
        from logging import warn, error
        if "serial_number" not in config_dict:
            error(" No serial_number paremter in configuration dictionary")
            return

        serial_number = config_dict["serial_number"]

        # check if SN matches
        if not self.SN_match(serial_number):
            warn('camera with given serial number is not found')
            return

        self.config_dict = config_dict

        # creates the piplines
        self.pipeline[ACCEL] = rs.pipeline()
        self.pipeline[GYRO] = rs.pipeline()
        self.pipeline[IMAGE] = rs.pipeline()

        # sets up the device
        self.pipeline_wrapper = rs.pipeline_wrapper(self.pipeline[IMAGE])
        self.pipeline_profile = self.conf[IMAGE].resolve(self.pipeline_wrapper)
        self.device = self.pipeline_profile.get_device()

        self.print_device_info()
        self.configure() # calls method to configure the device
        self.start()

    def SN_match(self, serial_number):
        """
        returns flag if serial number matches, configures and enables the device
        Parameter : serial number
        Returns: bool
        """
        from logging import warn, info
        ctx = rs.context()
        devices = ctx.query_devices()
        for device in devices:
            serial = device.get_info(rs.camera_info.serial_number)
            if serial == serial_number:
                info('Device Found')

                # setup the configuration for each frame type
                self.conf[ACCEL] = rs.config()
                self.conf[GYRO] = rs.config()
                self.conf[IMAGE] = rs.config()

                #enable device
                self.conf[IMAGE].enable_device(serial_number)
                self.conf[GYRO].enable_device(serial_number)

                return True
        warn('Device not found')
        return False

    def print_device_info(self):
        """
        Prints the device information
        Parameter : Nothing
        Returns: Nothings
        """
        import pyrealsense2 as rs
        from logging import info
        info(' ----- Device ----- ')
        info(f'  Device PID: {self.device.get_info(rs.camera_info.product_id)}')
        info(f'  Device name:  {self.device.get_info(rs.camera_info.name)}')
        info(f'  Serial number:  {self.device.get_info(rs.camera_info.serial_number)}')
        info(f'  Firmware version:   {self.device.get_info(rs.camera_info.firmware_version)}')
        info(f'  USB:  {self.device.get_info(rs.camera_info.usb_type_descriptor)}')

    def start(self):
        """
        starts the pipelines
        """
        self.profile[ACCEL] = self.pipeline[ACCEL].start(self.conf[ACCEL])
        self.profile[GYRO] = self.pipeline[GYRO].start(self.conf[GYRO])
        self.profile[IMAGE] = self.pipeline[IMAGE].start(self.conf[IMAGE])

    def stop(self):
        """
        stops the pipelines
        """
        self.profile[ACCEL] = self.pipeline[ACCEL].stop()
        self.profile[GYRO] = self.pipeline[GYRO].stop()
        self.profile[IMAGE] = self.pipeline[IMAGE].stop()

    def find_devices(self):
        """
        returns a list of all connected intel realsense devices
        """
        context = rs.context()
        connect_device = []

        for d in context.devices:
            if d.get_info(rs.camera_info.name).lower() != 'platform camera':
                serial = d.get_info(rs.camera_info.serial_number)
                product_line = d.get_info(rs.camera_info.product_line)
                device_info = (serial, product_line) # (serial_number, product_line)
                connect_device.append( device_info )
        return connect_device

    def configure(self):
        """
        Enables the stream for all configurations of the camera
        """
        from logging import error, warn, info, debug
        device_serial_number = str(self.device.get_info(rs.camera_info.serial_number))
        device_product_line = str(self.device.get_info(rs.camera_info.product_line))
        self.conf[IMAGE].enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30)

        if device_product_line == 'L500':
            self.conf[IMAGE].enable_stream(rs.stream.color, 960, 540, rs.format.bgr8, 30)
        else:
            self.conf[IMAGE].enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 30)

        try:
            self.conf[IMAGE].enable_stream(rs.stream.infrared)
            self.conf[IMAGE].enable_stream(rs.stream.depth)
            info("Enabled color, infared and depth streams ")

        except Exception as e:
            error('during image configuratuon the following error occured',e)

        if self.config_dict["channels"] != None:

            try:
                gyro_fps = self.config_dict["channels"][GYROCHANNEL][FPS]
                accel_fps = self.config_dict["channels"][ACCELCHANNEL][FPS]

                self.conf[ACCEL].enable_stream(rs.stream.accel,stream_index = 0,format = rs.format.motion_xyz32f, framerate = accel_fps)
                self.conf[GYRO].enable_stream(rs.stream.gyro,stream_index = 0,format = rs.format.motion_xyz32f, framerate = gyro_fps)

            except Exception as e:
                error('during IMU configuratuon the following error occured',e)

        else:
            try:
                self.conf[ACCEL].enable_stream(rs.stream.accel)
                self.conf[GYRO].enable_stream(rs.stream.gyro)

            except Exception as e:
                error('during DEFAULT IMU configuratuon the following error occured',e)

    def hardware_reset(self):
        """
        resets hardware
        """
        for frame in self.profile.keys():
            dev = self.profile[frame].get_device()
            dev.hardware_reset()

    def get_data(self):
        import numpy as np
        frames = self.pipeline[IMAGE].wait_for_frames()
        depth_frame = frames.get_depth_frame()
        color_frame = frames.get_color_frame()
        try:
            imu = frames[2].as_motion_frame().get_motion_data()
            imu1 = (imu.x,imu.y,imu.z)
            imu = frames[3].as_motion_frame().get_motion_data()
            imu2 = (imu.x,imu.y,imu.z)
        except:
            imu1 = (np.nan,np.nan,np.nan)
            imu2 = (np.nan,np.nan,np.nan)
        frameN = frames.get_frame_number()
        # Convert images to numpy arrays
        # it is important to copy the array.
        #Otherwise it stops returning new frames after 15 images.
        depth_image = np.copy(np.asanyarray(depth_frame.get_data()))
        color_image = np.copy(np.asanyarray(color_frame.get_data()))
        return {'depth':depth_image,'color': color_image,'imu1':imu1,'imu2':imu2,'frame#':frameN}

    def get_depth_resolution(self):
        """
        returns depth resolution of the camera in meters per count
        """
        result = self.device.first_depth_sensor().get_depth_scale()
        return result

    depth_resolution = property(get_depth_resolution)

    def get_all_available_options(self):
        """
        an auxiliary function to retrieve all available options for a given model
        """
        pipeline_profile = self.pipeline_profile
        dev = pipeline_profile.get_device()
        depth_sensor = None
        for sensor in dev.query_sensors():
            if sensor.is_depth_sensor():
                depth_sensor = dev.query_sensors()[0]
        for option in depth_sensor.get_supported_options():
            print('---',option,'---')
            print('description:',depth_sensor.get_option_description(option))
            print('currennt value:',depth_sensor.get_option(option))

    def set_laser_intensity(self, laser_power):
        """
        Sets laser intensity for Intel realsense camera
        Paramter: takes in laser power intensity
        Returns: nothing
        """
        from logging import error,warn,info,debug
        if laser_power < 0 or laser_power > 100 :
            warn("Laser power must be between 0-100")
            return

        for frame_type in self.profile.keys():
            dev = self.profile[frame_type].get_device()
            depth_sensor = dev.query_sensors()[0]
            depth_sensor.set_option(rs.option.laser_power, laser_power)

    def get_images(self):
        """
        Setting up a dict containing images that are recived from the pipline
        Parameters: Nothing
        Returns: Dict containing images
        """
        from logging import error,warn,info,debug
        f = self.pipeline[IMAGE].wait_for_frames()

        #collect the frame for each frame type
        frameN = f.get_frame_number()
        color = f.get_color_frame()
        infrared = f.get_infrared_frame()
        depth = f.get_depth_frame()
        # print("frameN", frameN)
        color_img = np.asanyarray(color.get_data())
        ir_img = np.asanyarray(infrared.get_data())
        depth_img = np.asanyarray(depth.get_data())
        FrameN_nparray = np.asanyarray(frameN)

        return {COLOR: color_img, DEPTH : depth_img, INFRARED : ir_img , FRAMEN : FrameN_nparray}

    def get_image_dtype(self, frame_type):
        """
        Parameter: Nothing
        Returns: image data type
        """
        return self.get_images()[frame_type].dtype

    def get_image_shape(self, frame_type):
        """
        Parameter: Nothing
        Returns: image Shape size
        """
        return self.get_images()[frame_type].shape

if __name__ == "__main__":
    from tempfile import gettempdir
    import logging
    import os
    import yaml

    logging.getLogger("blib2to3").setLevel(logging.ERROR)
    logging.getLogger("parso").setLevel(logging.ERROR)
    logging.getLogger("matplotlib").setLevel(logging.ERROR)
    logging.getLogger("PIL").setLevel(logging.ERROR)
    logging.getLogger("asyncio").setLevel(logging.ERROR)

    log_filename = os.path.join(gettempdir(),'intel_realsense_driver.log')
    logging.basicConfig(filename=log_filename,
                level=logging.DEBUG,
                format="%(asctime)-15s|PID:%(process)-6s|%(levelname)-8s|%(name)s| module:%(module)s-%(funcName)s|message:%(message)s")

    from matplotlib import pyplot as plt
    #plt.ion()
    driver = Driver()
    # SN = "139522074713"
    # SN = "f1231322"
    # config_filename = r"C:\Users\Abdel Nasser\Documents\L151 Camera\intel-realsense-devices\intel_realsense_devices\test_files\config_d435i__139522074713.yaml"
    config_filename = r"test_files\config_L515_f1320305.yaml"
    config_filename = r"test_files\config_L515_f1231322.yaml"

    with open(config_filename) as f:
        config_dict = yaml.safe_load(f)
        driver.init(config_dict)

{'warn': 'logging.warn', 'error': 'logging.error', 'info': 'logging.info', 'rs': 'pyrealsense2', 'debug': 'logging.debug', 'np': 'numpy'}


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
@author: chenhao
@time: 2023/7/13 15:17
"""
import math
import random
import re

from icetk import icetk
from snippets import *
from tqdm import tqdm

def try_load(p):
    try:
        ds = jload_lines(p)
        return ds
    except Exception:
        ds = jload(p)
        return ds

def is_valid(item, show=False, no_gpt=True):
    input = item.get("input")
    target = item.get("target")
    if not input or not target:
        return False

    input_list = icetk.encode(item["input"])
    target_list = icetk.encode(item["target"])
    rs = True
    if not 4 <= len(input_list) <= 512:
        rs = False
    if not 8 <= len(target_list) <= 2048:
        rs = False
    input_unk_num = len([x for x in input_list if x == 20000])
    target_unk_num = len([x for x in target_list if x == 20000])
    if input_unk_num >= 2 or target_unk_num >= 2:
        rs = False
    if no_gpt:
        pt = "gpt"
        if re.search(pt, item["input"], re.I) or re.search(pt, item["target"], re.I):
            rs = False

    if not rs and show:
        print(f"item: {item} is not valid")
    return rs

def parse_alpaca_zh(item):
    rs = dict(input=item["instruction"].strip(), target=item["output"].strip(), source="alpaca_gpt4_data_zh")
    return rs

def parse_tigerbot(item):
    rs = dict(input=(item["instruction"] + "\n" + item["input"]).strip(), target=item["output"].strip(),
              source="tigerbot")
    return rs

def parse_belle(item):
    rs = dict(input=(item["instruction"] + "\n" + item["input"]).strip(), target=item["output"].strip(),
              source="belle")
    return rs

def is_literature_generation(item):
    pt = "故事|诗|小说|文案|菜谱|报告|总结|剧本|社论|文章|道歉信|感激信|消化|发言稿|作文|菜谱|对联|教程|评论|短文"
    return re.search(pt, item["input"]) is not None

def extract_word_num(text, pattern="至少(\d+)字|不少于(\d+)字"):
    for e in re.findall(pattern, text):
        for m in e:
            try:
                # print(m)
                rs = int(m.strip())
                return rs
            except Exception as e:
                pass
    return None

def match_word_limit(item):

    tgt_len = len(item["target"])
    ipt = item["input"]
    min_num = extract_word_num(ipt, "[至最]少(\d+)字|不少于(\d+)字")
    if min_num and tgt_len >= min_num:
        return True
    max_num = extract_word_num(ipt, "[至最]多(\d+)字|不多于(\d+)字")
    if max_num and tgt_len <= max_num:
        return True
    around_num = extract_word_num(ipt, "大约(\d+)字|(\d+)字左右")
    if around_num and tgt_len * 0.85 <= around_num <= tgt_len * 1.15:
        return True
    return False

# def parse_share_gpt(item):
#     rs = []
#     pre = ""
#     rd = 1
#     for idx, ele in enumerate(item["conversations"]):
#         if ele["from"] == "human":
#             pre = pre + f"##第 {rd} 轮##\n\n问:{ele['value']}\n\n答:"
#         else:
#             rs.append(dict(input=pre.strip(), target=ele["value"], source="share-gpt"))
#             pre = pre + ele["value"] + "\n\n"
#             rd += 1
#     return rs

def is_chinese(strs, pct=0.4):
    ch_num = 0

    for _char in strs:
        if '\u4e00' <= _char <= '\u9fa5':
            ch_num += 1
    return ch_num / len(strs) >= pct

def parse_guanaco_non_chat(item):
    source = "guanaco_non_chat"
    if not item["output"].strip():
        return None

    if is_chinese(item["output"]):
        source += "_cn"

    rs = dict(input=(item["instruction"] + "\n" + item["input"]).strip(), target=item["output"].strip(),
              source=source)
    return rs

def parse_guanaco_chat(item):
    ipt = (item["instruction"] + item["input"]).strip()
    sentences = re.split(r"User:|Assistant:|System:", ipt)
    sentences = [e.strip() for e in sentences if e.strip()]
    # print(sentences)
    # sentences
    sentences += [item["output"].strip()]
    if not len(sentences) % 2 == 0:
        print(f"error in {len(sentences)} sentences")
        for s in sentences:
            print(s)
        return []

    pre = ""
    rs = []
    for idx in range(0, len(sentences), 2):
        rd = idx // 2 + 1
        pmt = pre + f"##第 {rd} 轮##\n\n问:{sentences[idx]}\n\n答:"
        if idx + 1 >= len(sentences):
            break
        ans = sentences[idx + 1].strip()
        if not ans:
            break
        source = "guanaco_chat"
        if is_chinese(ans):
            source += "_cn"
        rs.append(dict(input=pmt.strip(), target=ans, source=source))
        pre = pmt + ans + "\n\n"
    return rs

def _get_sentences_multiturn_chat(instruction):
    cur_sentence = ""
    rs = []
    tag = None
    sentences = re.split("(Assistant:)|(Human:)",instruction, flags=re.I)

    for s in sentences:
        if not s:
            continue
        if s in ["Assistant:", "Human:"]:
            if not tag or tag != s:
                tag = s
                if cur_sentence:
                    rs.append(cur_sentence.strip())
                    cur_sentence = ""
        else:
            cur_sentence += s

    if cur_sentence:
        rs.append(cur_sentence.strip())
    return rs

def parse_belle_multiturn_chat(item):
    ipt = (item["instruction"]).strip()
    sentences = _get_sentences_multiturn_chat(ipt)
    sentences += [item["output"].strip()]
    if not len(sentences) % 2 == 0:
        print(f"error in {len(sentences)} sentences")
        for idx, s in enumerate(sentences):
            print((idx,s))
        return []

    pre = ""
    rd = 1
    rs = []

    for idx, ele in enumerate(sentences):
        if idx %2 == 0:
            pre = pre + f"##第 {rd} 轮##\n\n问:{ele}\n\n答:"
        else:
            rs.append(dict(input=pre.strip(), target=ele, source="multiturn_chat"))
            rd += 1
            pre = pre + ele+ "\n\n"
    return rs

def parse_belle_chat(item):
    ipt = (item["instruction"] + item["input"]).strip()
    sentences = re.split(r"Human:|Assistant:", ipt)
    sentences = [e.strip() for e in sentences if e.strip()]
    # print(sentences)
    # sentences
    sentences += [item["output"].strip()]
    if not len(sentences) % 2 == 0:
        # print(f"error in {len(sentences)} sentences")
        # for idx, s in enumerate(sentences):
        #     print((idx,s))
        return []

    pre = ""
    rs = []
    for idx in range(0, len(sentences), 2):
        rd = idx // 2 + 1
        pmt = pre + f"##第 {rd} 轮##\n\n问:{sentences[idx]}\n\n答:"
        if idx + 1 >= len(sentences):
            break
        ans = sentences[idx + 1].strip()
        if not ans:
            break
        source = "belle_chat"
        rs.append(dict(input=pmt.strip(), target=ans, source=source))
        pre = pmt + ans + "\n\n"
    return rs

def contain_gpt(t):
    t = t.upper()
    return "GPT" in t or "OPENAI" in t

def parse_tsinghua_human_label(item):

    rs = []
    source = "tsinghua_human_label"

    history = item["history"]+[dict(prompt=item["prompt"], response=item["response"])]
    # print(history)
    # print(len(history))
    if len(history) > 1:
        source += "_multichat"
        pre = ""
        for idx, ele in enumerate(item["history"]):
            pre += f"##第 {idx+1} 轮##\n\n问:{ele['prompt']}\n\n答:"
            ans = ele["response"]
            rs.append(dict(input=pre, target=ans, source= source))
            pre += ans + "\n\n"
    else:
        source += "_qa"
        rs = [dict(input=history[0]["prompt"], target=history[0]["response"], source=source)]
    return rs

def parse_share_gpt(item):
    rs = []
    cov = item["conversations"]
    if len(cov) == 2:
        rs = [dict(input=cov[0]["value"], target=cov[1]["value"], source="share-gpt")]
    else:
        pre = ""
        sentences = [e["value"] for e in cov]
        for idx in range(0, len(sentences), 2):
            rd = idx // 2 + 1
            pmt = pre + f"##第 {rd} 轮##\n\n问:{sentences[idx]}\n\n答:"
            if idx + 1 >= len(sentences):
                break
            ans = sentences[idx + 1].strip()
            if not ans:
                break
            source = "share-gpt_chat"
            rs.append(dict(input=pmt.strip(), target=ans, source=source))
            pre = pmt + ans + "\n\n"
    return rs

def process_moss_qa(item):
    # item
    sentences = re.split(r"<eoh>|<eoa>", item["plain_text"])
    sentences = [e[e.index(":") + 1:].strip() for e in sentences if ":" in e]
    # sentences
    assert len(sentences) % 2 == 1
    pre = ""
    rs = []
    for idx in range(0, len(sentences), 2):
        rd = idx // 2 + 1
        pmt = pre + f"##第 {rd} 轮##\n\n问:{sentences[idx]}\n\n答:"
        ans = sentences[idx + 1]
        rs.append(dict(input=pmt, target=ans, source="moss_qa"))
        pre = pmt + ans + "\n\n"

    return rs

def process_moss_multi_chat(item, do_multi_chat=True):
    # item
    chats = item["chat"]
    # chats
    rs = []
    pre =""

    for idx, (t,ele) in enumerate(chats.items()):
        # ele
        human = ele["Human"].replace("<|Human|>: ","").replace("<eoh>","").strip()
        moss = ele["MOSS"].replace("<|MOSS|>: ","").replace("<eom>","").strip()
        human, moss
        if not do_multi_chat:
            rs.append(dict(input=human, target=moss, source="moss_qa"))
        else:
            q = \
    f'''##第 {idx+1} 轮##
    问:{human}

    答:
    '''
            a = moss
            pre = pre + "\n\n" + q
            rs.append(dict(input=pre.strip(), target=a.strip(), source="moss_multi_chat"))
            pre += a
    return rs

def parse_data(data, func):
    rs = []
    for idx, item in enumerate(tqdm(data[:])):
        try:
            tmp = func(item)
            if tmp:
                rs.append(tmp)
            else:
                pass
        except:
            pass
    print(f"origin rs len:{len(rs)}")
    if isinstance(rs[0], list):
        rs = flat(rs)
    print(f"parsed rs len:{len(rs)}")
    print(f"sample rs item :{random.choice(rs)}")
    return rs

def view_data(data, sample_num=10):
    sample = random.sample(data, min(len(data), sample_num))
    for item in sample:
        print(item["input"])
        print("*" * 30)
        print(item["target"])
        print("*" * 30)
        print(item["source"])
        print("*" * 30)
        print("\n\n")

def equal_sample(sources, tgt_num):
    rs = []
    t_num = int(math.ceil(tgt_num / len(sources)))
    for s in sources:
        # print(len(s))
        # print(t_num)
        print(f"sample {t_num} from {len(s)}")
        assert len(s) > t_num
        rs.extend(random.sample(s, t_num))
    return rs[:tgt_num]

if __name__ == "__main__":
    print(match_word_limit(dict(input="写一篇609999字左右的作文")))


'''
Created on Feb 8, 2018

@author: stelios
'''
from base.base_table import BaseTable
import adql
import logging
from astropy.table import Table as astropy_Table
import config as config

class AdqlTable(BaseTable):
    """
    classdocs
    """

    def __init__(self, adql_schema, json_object=None, url=None):
        """
        Constructor
        """
        super().__init__(adql_schema, json_object, url)

    def select_columns(self):
        column_list = []
        json_list = self.get_json(self.url + "/columns/select")

        for column in json_list:
            column_list.append(adql.AdqlColumn(json_object=column, adql_table=self))

        return column_list

    def select_column_by_ident(self, ident):
        return adql.AdqlColumn(adql_table=self, url=ident)

    def select_column_by_name(self, column_name):
        """Get table by name

        Parameters
        ----------
        table_name: string, required
            The name of the Table being searched

        Returns
        -------
        table_list: list
            List of table names
        """
        response_json = {}
        try :
            response_json = self.get_json(  self.url + "/columns/select", { "adql.column.name": column_name })
        except Exception as e:
            logging.exception(e)

        return adql.AdqlColumn(json_object = response_json, adql_table=self)

    def create_adql_column(self, column_name):
        return

    def count(self):
        """Get Row count

        Returns
        -------
        rowcount: integer
            Count of rows
        """
        rowcount=-1

        try:
            if (self.json_object!=None):
                rowcount = self.json_object.get("metadata",[]).get("adql",[]).get("count",None)
        except Exception as e:
            logging.exception(e)

        return rowcount

    def as_astropy (self, limit=True):
        """Get Astropy table

        Returns
        -------
        astropy_table: Astropy.Table
            Table as Astropy table
        """
        if (limit):
            if (self.count()>config.maxrows):
                raise Exception ("Max row limit exceeded")
            else :
                return astropy_Table.read(self.url + "/votable", format="votable",use_names_over_ids=True)
        else:
            return astropy_Table.read(self.url + "/votable", format="votable", use_names_over_ids=True)


from geopy.distance import geodesic
from itertools import permutations
import math

def calculate_distance(coords1, coords2):
    return geodesic(coords1, coords2).kilometers

def calculate_total_distance(route, points):
    total_distance = 0
    for i in range(len(route) - 1):
        total_distance += calculate_distance(points[route[i]], points[route[i+1]])
    return total_distance

def nearest_neighbor_algorithm(points):
    n = len(points)
    unvisited = list(range(n))
    route = [0]  # Start at the first point (index 0)
    unvisited.remove(0)

    while unvisited:
        nearest_point = min(unvisited, key=lambda x: calculate_distance(points[route[-1]], points[x]))
        route.append(nearest_point)
        unvisited.remove(nearest_point)

    route.append(0)  # Return to the starting point to complete the cycle
    return route

def find_optimal_route(points):
    n = len(points)
    optimal_route = None
    min_distance = float('inf')

    for perm in permutations(range(1, n)):
        route = [0] + list(perm) + [0]
        total_distance = calculate_total_distance(route, points)
        if total_distance < min_distance:
            min_distance = total_distance
            optimal_route = route

    return optimal_route, min_distance

def distance(point1, point2):
    lat1, lng1 = point1
    lat2, lng2 = point2
    return math.sqrt((lat2 - lat1)**2 + (lng2 - lng1)**2)

def nearest_neighbor_tsp(points, start_point):
    tour = [start_point]
    remaining_points = set(points)
    remaining_points.remove(start_point)

    while remaining_points:
        nearest_point = min(remaining_points, key=lambda x: distance(tour[-1], x))
        tour.append(nearest_point)
        remaining_points.remove(nearest_point)

    #tour.append(start_point) # enable for return to starting point

    return tour

def approximatePathOrdering(datasetFeatures):
    # Using Nearest Neighbor algorithm to get an approximate optimal route
    gps_points = []
    for feature in datasetFeatures:
        gps_points.append((feature['geometry']['coordinates'][1], feature['geometry']['coordinates'][0]))

    #nn_route = nearest_neighbor_algorithm(gps_points)
    starting_point = (51.50448,-0.07662)

    # Find the optimal tour starting from the chosen data point
    ordered_gps_points = nearest_neighbor_tsp(points=gps_points, start_point=starting_point)
    print("Nearest Neighbor route calculated")
    return ordered_gps_points


import io
import re

from setuptools import find_packages
from setuptools import setup

with io.open("src/upparat/__init__.py", "rt", encoding="utf8") as f:
    version = re.search(r'__version__ = "(.*?)"', f.read()).group(1)

setup(
    name="upparat",
    version=version,
    description="Update Service for AWS IoT Core",
    url="https://github.com/caruhome/upparat",
    author="Livio Bieri / Reto Aebersold",
    long_description_content_type="text/markdown",
    long_description="The Upparat is a secure and robust service that runs on your IoT device to download and install files such as firmware updates. [Learn more](https://github.com/caruhome/upparat).",  # noqa
    license="MIT",
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: System Administrators",
        "Intended Audience :: Developers",
        "License :: OSI Approved :: MIT License",
        "Operating System :: POSIX",
        "Programming Language :: Python :: 3",
        "Topic :: System :: Installation/Setup",
        "Topic :: System :: Operating System",
    ],
    python_requires=">=3.6",
    packages=find_packages("src"),
    package_dir={"": "src"},
    include_package_data=True,
    entry_points={"console_scripts": ["upparat=upparat.cli:main"]},
    install_requires=["paho-mqtt>=1.4.0", "backoff>=1.8.1", "pysm>=0.3.7"],
    extras_require={
        "dev": ["pytest", "pytest-mock", "boto3", "ipdb", "coverage"],
        "sentry": ["sentry-sdk"],
    },
)


__author__    = "Individual contributors (see AUTHORS file)"
__date__      = "$DATE$"
__rev__       = "$REV$"
__license__   = "AGPL v.3"
__copyright__ = """
This file is part of the ESP Web Site
Copyright (c) 2007 by the individual contributors
  (see AUTHORS file)

The ESP Web Site is free software; you can redistribute it and/or
modify it under the terms of the GNU Affero General Public License
as published by the Free Software Foundation; either version 3
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public
License along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

Contact information:
MIT Educational Studies Program
  84 Massachusetts Ave W20-467, Cambridge, MA 02139
  Phone: 617-253-4882
  Email: esp-webmasters@mit.edu
Learning Unlimited, Inc.
  527 Franklin St, Cambridge, MA 02139
  Phone: 617-379-0178
  Email: web-team@learningu.org
"""
from esp.users.models import ContactInfo, ESPUser
from esp.program.models import ArchiveClass, ClassSubject, ClassCategories
from esp.utils.web import render_to_response
from django.db.models.query import QuerySet
from django.http import HttpResponse, Http404, HttpResponseNotAllowed, HttpResponseRedirect
from django.contrib.auth.decorators import login_required
from datetime import datetime

#    Two inputs to each function:
#    -    category: what you sort or view by
#        (such as: year, program, subject category, title)
#    -    options: the particular filter for that data
#        (eg. 2004, Splash, ComputerScience)

class ArchiveFilter(object):
    category = ""
    options = ""
    def __init__(self, category = "", options = ""):
        self.category = unicode(category)
        self.options  = unicode(options)

    def __unicode__(self):
        return '%s, %s' % (self.category, self.options)

def compute_range(postvars, num_records):
    default_num_records = 10
    results_start = 0
    results_end = None
    if 'results_start' in postvars:
        results_start = int(postvars['results_start'])
    if 'results_end' in postvars:
        results_end = int(postvars['results_end'])
    if (num_records > default_num_records) and results_end == None:
        if postvars.get('max_num_results') == "Show all":
            results_end = num_records
        elif postvars.get('max_num_results'):
            results_end = results_start + int(postvars['max_num_results'])
        else:
            results_end = results_start + default_num_records

    return {'start': results_start, 'end': results_end}

def extract_criteria(postvars):
    #    Use filters
    criteria = []
    for key in postvars.keys():
        if len(postvars[key]) > 0 and key.find("filter_") != -1 and len(key) > 7: criteria.append(ArchiveFilter(category = key[7:], options = postvars[key]))

    return criteria

def filter_archive(records, criteria):
    result = records
    for c in criteria:
        if c.category == 'year':
            result = result.filter(year = c.options)
        elif c.category == 'program':
            result = result.filter(program__iexact = c.options)
        elif c.category == 'title':
            result = result.filter(title__istartswith = c.options)
        elif c.category == 'category':
            result = result.filter(category__istartswith = c.options)
        elif c.category == 'teacher':
            result = result.filter(teacher__icontains = c.options)
        elif c.category == 'description':
            result = result.filter(description__icontains = c.options)

    return result

def title_heading(title_content):
    if len(title_content) > 0:
        return title_content[0]
    else:
        return '[No Title]'

def archive_classes(request, category, options, sortorder = None):
    context = {'selection': 'Classes'}
    context['category'] = category
    context['options'] = options

    #    Take filtering criteria both from form and from URL
    if category != None and options != None:
        url_criterion = ArchiveFilter(category = category, options = options)
        criteria_list = [url_criterion]
    else:
        criteria_list = []

    criteria_list += extract_criteria(request.POST)
    criteria_list += extract_criteria(request.GET)

    category_list = list(ArchiveClass.objects.all().values_list('category', flat=True).distinct())
    program_list = list(ArchiveClass.objects.all().values_list('program', flat=True).distinct())

    category_dict = {}
    classcatList = ClassCategories.objects.all()
    for letter in map(chr, range(65, 91)):
        category_dict[letter] = 'Unknown Category'

    for category in classcatList:
        category_dict[category.category[0].upper()] = category.category

    filter_keys = {'category': [{'name': c, 'value': c, 'selected': False} for c in category_list],
            'year': [{'name': str(y), 'value': str(y), 'selected': False} for y in range(1998, datetime.now().year + 1)],
            'title': [{'name': 'Starts with ' + letter, 'value': letter, 'selected': False} for letter in map(chr, range(65,91))],
             'program': [{'name': p, 'value': p, 'selected': False} for p in program_list],
            'teacher': [{}],
            'description': [{}]
            }
    if 'filter_teacher' in request.POST: filter_keys['teacher'][0]['default_value'] = request.POST['filter_teacher']
    if 'filter_description' in request.POST: filter_keys['description'][0]['default_value'] = request.POST['filter_description']

    results = filter_archive(ArchiveClass.objects.all(), criteria_list)

    #    Sort the results by the specified order
    if (not isinstance(sortorder, list)) or len(sortorder) < 1:
        sortorder = ['year', 'category', 'program', 'title', 'teacher', 'description']

    sortorder.reverse()
    for parameter in sortorder:
        results = results.order_by(parameter)
    sortorder.reverse()

    context['sortorder'] = sortorder

    for c in criteria_list:
        for k in filter_keys[c.category]:
            if 'name' in k and 'value' in k:
                if c.options == k['value']:
                    k['selected'] = True

    context['sortparams'] = [{'name': k, 'options': filter_keys[k]} for k in filter_keys.keys()]

    #    Display the appropriate range of results
    postvars = request.POST.copy()
    relevant_keys = ['max_num_results', 'results_start', 'results_end']
    for k in relevant_keys:
        if k in request.GET:
            postvars[k] = request.GET[k]

    res_range = compute_range(postvars, results.count())
    context['results_range'] = res_range

    #    Deal with the Django bug preventing you from using no "end"
    if res_range['end'] is None:
        res_range['end'] = results.count()

    #    Rename all of the class categories and uppercase the programs
    for entry in results[res_range['start']:res_range['end']]:
        entry.category = category_dict[entry.category[:1].upper()]
        entry.program = entry.program.upper()
        #    entry.title = entry.title.capitalize()

    #    Compute the headings for the 'jump to category' part
    if sortorder[0] == 'title':
        headings = [title_heading(item.__dict__['title']) for item in results[res_range['start']:res_range['end']]]
    else:
        headings = [item.__dict__[sortorder[0]] for item in results[res_range['start']:res_range['end']]]

    context['headings'] = list(set([unicode(h) for h in headings]))
    context['headings'].sort()

    #    Fill in context some more
    context['num_results'] = results.count()
    context['results'] = list(results[res_range['start']:res_range['end']])
    context['results_range']['start'] = context['results_range']['start'] + 1
    if res_range['end'] > context['num_results']:
        context['results_range']['end'] = context['num_results']
    if 'max_num_results' in request.POST:
        context['max_num_results'] = request.POST['max_num_results']
    else:
        context['max_num_results'] = '25'
    context['num_results_list'] = ['10', '25', '50', '100', '250', 'Show all']
    context['num_results_shown'] = len(context['results'])

    return render_to_response('program/archives.html', request, context)

def archive_teachers(request, category, options):
    context = {'selection': 'Teachers'}
    context['category'] = category
    context['options'] = options

    return render_to_response('program/archives.html', request, context)

def archive_programs(request, category, options):
    context = {'selection': 'Programs'}
    context['category'] = category
    context['options'] = options

    return render_to_response('program/archives.html', request, context)

archive_handlers = {    'classes': archive_classes,
            'teachers': archive_teachers,
            'programs': archive_programs,
        }


from qiskit import QuantumCircuit, assemble, Aer
from math import pi, sqrt
import matplotlib
from matplotlib import pyplot as plt
from qiskit.visualization import plot_bloch_multivector, plot_histogram, circuit_drawer
sim = Aer.get_backend('aer_simulator')

# Let's do an X-gate on a |0> qubit
qc = QuantumCircuit(1)
qc.x(0)
qc.x(0)
qc.draw(output = "mpl" )
print(qc)

plt.show()

def had_test(gate_type,num_of_qubits, parameters):

    ancilla = QuantumRegister(1,'ancilla')
    qr = QuantumRegister(num_of_qubits,'q')
    cr = ClassicalRegister(1)
    circ =QuantumCircuit(ancilla,qr,cr)
    circ.h(ancilla[0])

    apply_fixed_ansatz(circ,qr[0:3], parameters)

    for i in range (0, len(gate_type[0])):
        if (gate_type[0][i] == 1):
            circ.cz(ancilla[0], qr[i])

    for i in range (0, len(gate_type[0])):
        if (gate_type[1][i] == 1):
            circ.cz(ancilla[0], qr[i])

    circ.h(ancilla[0])
    circ.measure(ancilla,cr)
    circ.draw(output="mpl")

had_test([[0, 0, 0], [0, 0, 1]], 3, [[1, 1, 1], [1, 1, 1], [1, 1, 1]])


import sys, random, re, logging, collections, os, io
from numpy import random

class PSM(object):
    """
    This class represents the data pertained on a particular (peptideID, charge state) tuple
    """

    __slots__ = ['id', 'seq', 'weights', 'ions', 'scores', 'y_fractions']

    def __init__(self, key, seq):
        self.id = key
        self.seq = seq
        self.weights = {'b': list(), 'y': list()}
        self.ions = {'b': list(), 'y': list()}
        self.scores = list()
        self.y_fractions = list()

    def addSpectra(self, ions_n_ints, score, y_frac):
        self.scores.append(score)
        self.y_fractions.append(y_frac)
        for ion_type, val_tuple in ions_n_ints.items():
            self.ions[ion_type].append(val_tuple[1])
            self.weights[ion_type].append(val_tuple[2])

    def addSpectra2(self, ions_n_ints, score, y_frac):
        """
        Stores the ion series as a list of ints rather than a list of strings
        this is useful for decreasing memory consumption
        """
        self.scores.append(score)
        self.y_fractions.append(y_frac)
        for ion_type, val_tuple in ions_n_ints.items():
            key, ion_series, ion_ints = val_tuple
            ion_series = [int(x[1:]) for x in ion_series]
            self.ions[ion_type].append(ion_series)
            self.weights[ion_type].append(ion_ints)

    def getSize(self):
        from common_utils import get_size
        total = sys.getsizeof(self.id)
        total += sys.getsizeof(self.seq)
        total += get_size(self.weights)
        total += get_size(self.ions)
        total += get_size(self.scores)
        total += get_size(self.y_fractions)
        return total

    def listSize(self):
        from common_utils import get_size
        print(f"Size listing of PSMs for {(self.seq, self.id[1])}, # of spectra={len(self.scores)}")
        print("-"*60)
        print(f"{'Id/key':8s}:{sys.getsizeof(self.id):>6d}")
        print(f"{'Sequence':8s}:{sys.getsizeof(self.seq):>6d}")
        print(f"{'Weights':8s}:{get_size(self.weights):>6d}")
        print(f"{'Ions':8s}:{get_size(self.ions):>6d}")
        print(f"{'Scores':8s}:{get_size(self.scores):>6d}")
        print(f"{'y_fracs':8s}:{get_size(self.y_fractions):>6d}")
        print("=" * 60)

    def listIons(self):
        from itertools import islice
        for i in islice(range(len(self.ions['y'])), 7):
            y = self.ions['y'][i]
            b = self.ions['b'][i]
            print(', '.join(y))
            print(', '.join(b))
            print()

class MQParser(object):
    __slots__ = ['psms', 'pos', 'neg', 'fdr', 'fdr_threshold', 'ion_threshold', 'mc', 'debug']

    def __init__(self, fdr_threshold=0.01, ion_threshold=0.001, mc=1, dodebug=False):
        self.psms = {}
        self.pos = 0.0
        self.neg = 0.0
        self.fdr = 0.0
        self.fdr_threshold = fdr_threshold
        self.ion_threshold = ion_threshold
        self.mc = mc
        self.debug = dodebug

    def newPSM(self, key, seq):
        psm = PSM(key, seq)
        self.psms[key] = psm

    def updateInfo(self, key, ions_ints, s, yfrac):
        self.psms[key].addSpectra2(ions_ints, s, yfrac)

    def getDataAsTuple(self, key):
        if key not in self.psms:
            raise KeyError('Specified key not found')

        hit = self.psms[key]

        # Assertions:
        if len(hit.scores) != len(hit.y_fractions):
            raise AssertionError("Values of uneven length for {}".format(key))
        if len(hit.ions['b']) != len(hit.weights['b']):
            raise AssertionError("Values of uneven length for {}".format(key))
        if len(hit.ions['y']) != len(hit.weights['y']):
            raise AssertionError("Values of uneven length for {}".format(key))

        _getionseries = lambda itype, iseries: [[itype+str(ion) for ion in s] for s in iseries]

        return zip(hit.scores,
                   _getionseries('y', hit.ions['y']), hit.weights['y'],
                   _getionseries('b', hit.ions['b']), hit.weights['b'],
                   hit.y_fractions)

    def getKeys(self):
        return list(self.psms.keys())

    def get_subset(self, keyset):
        import copy
        parser = copy.copy(self)
        parser.psms = {k: v for k, v in self.psms.items() if k in keyset}
        return parser

    def incrementNeg(self):
        self.neg += 1
        self._updateFDR()

    def incrementPos(self):
        self.pos += 1

    def _updateFDR(self):
        self.fdr = self.neg / (self.neg + self.pos)
        return self.fdr

    def __contains__(self, key):
        return self.psms.__contains__(key)

    def processChunk(self, df, skip_sort):
        if self.debug:
            print("{0:10}: {1:>12,}".format("Data Frame", sys.getsizeof(df)))
            print("{0:10}: {1:>12,}".format("Parser", sys.getsizeof(self)))
            input("Enter to continue...")

        if not skip_sort:
            df = df.sort_values(by='PEP')

        '''
        Initiate variables, TODO: Consider placing these in a container
        '''
        for row in df.itertuples():
            '''
            Filtering rows based on reverse hits, modified peptides or missing intensities
            '''
            if row.Reverse == '+' or row.Proteins == '':
                self.incrementNeg()
                continue
            if row.Modifications != 'Unmodified':
                continue
            if row.Intensities == '':
                continue
            if row.Number_of_Matches == 0:
                continue
            if not self._checkMisscleavage(row.Sequence):
                continue

            '''
            Use (Peptide ID, Charge) tuple as a key. OBS: The consistency of peptide IDs is delegated to MQ
            '''
            key = (int(row.Peptide_ID), int(row.Charge))
            intensities = [float(x) for x in row.Intensities.split(';')]
            ion_series = re.findall('[abcxyz]\d+', row.Matches)
            ii = list(zip(intensities, ion_series))
            total_int = {}
            temp = {}

            '''
            Normalize each ion in b/y series to the sum of respective series
            '''
            if key not in self:
                self.newPSM(key, row.Sequence)

            for ion_type in 'by':
                total_int[ion_type] = sum([i for (i, s) in ii if s[0] == ion_type])
                ion_int = collections.defaultdict(float)
                for _int, _ion in ii:
                    if _ion[0] == ion_type:
                        ion_int[_ion] += _int / total_int[ion_type]

                if len(ion_int) != 0:
                    _ions, _weights = zip(*(x for x in ion_int.items() if x[1] > self.ion_threshold))
                    temp[ion_type] = (key, _ions, _weights)
                else:
                    # If a spectra has only y- or b- ions an empty list is appended
                    # TODO: Is there a better way to handle this? Are the empty lists used ever?
                    temp[ion_type] = (key, [], [])
            try:
                yfrac = total_int['y'] / (total_int['y'] + total_int['b'])
                self.updateInfo(key, temp, float(row.Score), yfrac)

            except ZeroDivisionError:
                '''
                This happens when there are no y- or b- ions, for ex. when one spectra
                only has an a2 ion, and no other ions! Such spectra are useless and thus deleted
                '''

                logging.debug("Empty/Useless spectra found:")
                logging.debug(row)

            '''
            Update FDR counter and check limit criterion
            '''
            self.incrementPos()
            if self.fdr > self.fdr_threshold:
                print("Breaking loop on FDR limit: pos={}, neg={}".format(self.pos, self.neg))
                return False

        return True

    def outputResults(self, outf, spectra_t, nslices):
        all_keys = self.getKeys()
        l = len(all_keys)
        random.shuffle(all_keys)
        nfiles = nslices
        key_slices = [all_keys[i * l // nfiles:(i + 1) * l // nfiles] for i in range(nfiles)]

        filepath = os.path.join(outf, "training_set_{}_{}.tsv")
        _openf = lambda i: open(filepath.format(i, spectra), 'w')

        for i, keys in enumerate(key_slices):
            with open(filepath.format(i, spectra_t), 'w') as outfile:
                for key in keys:
                    data = list(self.getDataAsTuple(key))
                    # data = sorted(data, key=itemgetter(0), reverse=True) # TODO: Unnecessary, data is already sorted
                    if not data:    # check for empty spectra
                        continue

                    charge = key[1]
                    seq = self.psms[key].seq
                    spectra = self._getSpectra(data, spectra_t)
                    for s in spectra:
                        self.write_results(outfile, charge, seq, *s)

    def _checkMisscleavage(self, seq):
        matches = re.findall('[KR]', seq)
        if seq.endswith('K') or seq.endswith('R'):
            return len(matches) <= self.mc + 1
        else:
            return len(matches) <= self.mc

    def _getSpectra(self, data, spectra):
        # Pick the spectra to use, depending on runtime param
        if spectra == 'best':
            return [data[0]]

        elif spectra == 'rand':
            r = random.randint(len(data))
            return [data[r]]

        elif spectra == 'med':
            return [data[len(data) // 2]]

        elif spectra == 'all':
            return data

        elif spectra == 'ave':
            _ysum_weight = collections.defaultdict(float)
            _bsum_weight = collections.defaultdict(float)
            yf = 0
            score = 0
            nhits = len(data)
            for _s, _yi, _yw, _bi, _bw, _yf in data:
                for i, w in zip(_yi, _yw):
                    _ysum_weight[i] += w / nhits

                for i, w in zip(_bi, _bw):
                    _bsum_weight[i] += w / nhits

                yf += _yf
                score += _s

            yi, yw, bi, bw = [], [], [], []
            if len(_ysum_weight) != 0:
                yi, yw = zip(*_ysum_weight.items())
            if len(_bsum_weight) != 0:
                bi, bw = zip(*_bsum_weight.items())

            yf /= nhits
            score /= nhits

            return [(score, yi, yw, bi, bw, yf)]

    def _write(self, f, charge, seq, score, y_ions, y_weights, b_ions, b_weights, y_frag):
        y_ions = ' '.join(map(str, y_ions))
        b_ions = ' '.join(map(str, b_ions))
        y_weights = ' '.join(map(str, y_weights))
        b_weights = ' '.join(map(str, b_weights))

        f.write("\t".join(map(str, (charge, seq, score, y_ions, y_weights, b_ions, b_weights, y_frag))))
        f.write('\n')

    def write_results(self, f, charge, seq, score, y_ions, y_weights, b_ions, b_weights, y_frag, i=-1):
        if isinstance(f, io.IOBase):
            self._write(f, charge, seq, score, y_ions, y_weights, b_ions, b_weights, y_frag)
        else:
            # TODO: is this necessary, when is it ever run?
            if i % 5 == 0:
                # write to benchmark file every 1/5 time
                self._write(f[-1], charge, seq, score, y_ions, y_weights, b_ions, b_weights, y_frag)
            elif i % 5 == 1:
                # write to test or training file every 1/5 time
                self._write(f[-2], charge, seq, score, y_ions, y_weights, b_ions, b_weights, y_frag)
            else:
                # write to training the rest of the time
                self._write(f[0], charge, seq, score, y_ions, y_weights, b_ions, b_weights, y_frag)

{'get_size': 'common_utils.get_size', 'islice': 'itertools.islice'}


import sys, re, subprocess, json
from math import *
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from scipy.spatial.distance import pdist,squareform
from scipy.cluster.hierarchy import linkage, dendrogram
from scipy.cluster import hierarchy

jsonFile = open(sys.argv[1], 'r')

print("Loading json file ..."),
probDict = json.load(jsonFile)
print("Done")

sortedRowNames = sorted(probDict.keys())
sortedColNames = sorted(probDict[sortedRowNames[0]].keys())

probArray = np.array([[probDict[rowName][colName] for colName in sortedRowNames] for rowName in sortedRowNames])

probDf = pd.DataFrame(probArray, columns = sortedRowNames, index = sortedRowNames)

'''
fig = plt.figure()

ax = fig.add_subplot(111)

cax = ax.matshow(probDf, interpolation='nearest', cmap='hot_r')
fig.colorbar(cax)

ax.set_xticklabels([''] + list(probDf.columns))
ax.set_yticklabels([''] + list(probDf.index))

plt.show()
'''

rowDist = pd.DataFrame(squareform(pdist(probDf, metric='euclidean')), columns=sortedRowNames, index=sortedRowNames)

rowClusters = linkage(pdist(probDf, metric='euclidean'), method='complete')

hierarchy.set_link_color_palette(['black'])

fig = plt.figure(figsize = (8,8))
axd = fig.add_axes([0.09,0.1,0.2,0.6])

rowDendr = dendrogram(rowClusters, orientation = 'right', color_threshold = np.inf,)
dfRowClust = probDf.ix[rowDendr['leaves'][::-1]]
print(rowDendr['leaves'])

axd.set_xticks([])
axd.set_yticks([])

for i in axd.spines.values():
        i.set_visible(False)

axm = fig.add_axes([0.26,0.1,0.6,0.6]) # x-pos, y-pos, width, height
cax = axm.matshow(probDf, interpolation='nearest')
fig.colorbar(cax)
plt.xticks(range(0, len(list(dfRowClust.columns))))
axm.set_xticklabels(list(dfRowClust.columns), rotation = 90, fontsize = 5, visible = True)
axm.yaxis.tick_right()
axm.set_yticklabels([''] + list(dfRowClust.index), fontsize = 5, visible = True)

plt.show()

#http://nbviewer.ipython.org/github/rasbt/pattern_classification/blob/master/clustering/hierarchical/clust_complete_linkage.ipynb

#Heatmaps in R:
#http://sebastianraschka.com/Articles/heatmaps_in_r.html


import os,sys
from imutils.video import VideoStream
from pyzbar import pyzbar
import argparse
import datetime
import imutils
import time
import cv2
#from barcode import EAN13
#from barcode.writer import ImageWriter
from qtpy.QtCore import Signal, QThread, QObject, Slot

class Camera(QObject):
    global camPtr
    global vid
    signal = Signal(str)
    def __init__(self):
        self.vid = cv2.VideoCapture(0)
        self.thread = QThread()
        super().__init__()
        self.thread.started.connect(self.loop)
        self.img = None

    @Slot()
    def loop(self):
        try:
            while True:
                _, frm = self.vid.read()
                frame = imutils.resize(frm, width=1024)
                image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
                barcodes = pyzbar.decode(image)
                self.img = image
                if len(barcodes) > 0:
                    self.signal.emit(str(len(barcodes)))
                else:
                    self.signal.emit("")
                time.sleep(0.5)
        except Exception as e:
            print(str(e))

    def startLoop(self):
        self.moveToThread(self.thread)
        self.thread.start()

    def start(self):
        try:
            #camPtr = VideoStream(usePiCamera=True).start()
            camPtr = VideoStream(src=0).start()
            print("Got CamPtr as: ", camPtr)
        except Exception as err:
            print("Got Error while opening Camera interface :", err)
            sys.exit()
        return camPtr

    def capture(self):
        vid = cv2.VideoCapture(0)
        vid.read()
        vid.read()
        vid.read()
        vid.read()
        vid.read()
        vid.read()
        ret, frame = vid.read()
        frame = imutils.resize(frame, width=640)
        image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        vid.release()
        return image

    def Barcode(self):
        frame = self.capture()
        barcodes = pyzbar.decode(frame)
        if len(barcodes) > 0:
            print("barcode found - " + str(len(barcodes)))
        else:
            print("barcode not found")
        for barcode in barcodes:
            pass

    def getBarcode(self, camPtr):
        frame = camPtr.read()
        if frame is not None:
            frame = imutils.resize(frame, width=640)
        return frame


from fastapi import Depends

from sqlalchemy.orm import Session
from typing import Union

from src.car import models, schemas

def get_cars(db: Session, skip: int = 0, limit: int = 10, user_id: Union[int, None]=None):
    queryset = db.query(models.Car)
    if user_id:
        queryset = queryset.filter(models.Car.owner_id == user_id)
    return queryset.offset(skip).limit(limit).all()

def get_car_by_plate_number(db: Session, plate_number: str):
    return db.query(models.Car).filter(plate_number == models.Car.plate_number).first()

def get_user_cars(db: Session, user_id: int, skip: int = 0, limit: int = 10):
    return (
        db.query(models.Car)
        .filter(models.Car.owner_id == user_id)
        .offset(skip)
        .limit(limit)
        .all()
    )

def create_user_car(db: Session, car: schemas.CarCreate, user_id: int):
    db_car = models.Car(**car.dict(), owner_id=user_id)
    db.add(db_car)
    db.commit()
    db.refresh(db_car)
    return db_car

def get_or_create_user_car(
    db: Session, car: Depends(get_car_by_plate_number), plate_number: str, user_id=int
):
    create_flag = False
    if car:
        return car, create_flag
    car = create_user_car(car={"plate_number": plate_number}, user_id=user_id)