from __future__ import division
import numpy as np
import inspect
# import scipy.special as special_fun
from scipy.special import gamma

# Other packages import
from ...devtools.BLonD_Rc import rcBLonDparams
from ...fitting import profile

def check_greater_zero(value, func_name):
    if value <= 0.0:
        raise ValueError("%s needs to be positive" % (func_name))

class _DistributionObject(object):
    r'''Base class for all analytic distributions

    This base class is not to be used directly. Instead, all analytic
    distributions should derive from it.

    Attributes
    ----------
    amplitude : float
        amplitude of the profile
    position : float
        position of the profile (usually the position of the maximum)
    RMS : float
        RMS of the distribution
    FWHM : float
        FWHM of the distribution
    fourSigma_RMS : float
        $4\times$ RMS
    fourSigma_FWHM : float
        Gaussian equivalent $4\sigma$ from FWHM of profile; i.e.
        $4\sigma_{FWHM} = 2/\sqrt(\ln 4) FWHM$
    full_bunch_length : float
        the length from the first zero of the profile function to the other
        zero
    integral : float
        the integrated profile/distribution
    store_data : bool
        if False, calls to functions like 'profile' return the value, otherwise
        an attribute is created

    Methods
    ----------
    profile(t, *args)
        Computes the profile at 't'. If no extra args are passed, it computes
        the profile using the object attributes (amplitdue, etc.). The
        store_data keyword if the profile is returned or stored as a class
        attribute.
    distribution(J)
        Computes the distribution function for action J or Hamiltonian H
    phase_space()
        Computes the 2d phase_space in (dt, dE) coordinates
    spectrum(f)
        Computes the spectrum at frequency 'f'
    '''

    def __init__(self, store_data=None, **kwargs):
        self.amplitude = None
        self.position = None

        if store_data is None:
            self.store_data = rcBLonDparams['distribution.store_data']
        else:
            self.store_data = store_data

    @property
    def store_data(self):
        return self._store_data

    @store_data.setter
    def store_data(self, value):
        self._store_data = value
        if not value:
            self._return_function = self._return_value
        else:
            self._return_function = self._store_value

    def _return_value(self, arg):
        return arg

    def _store_value(self, value):
        setattr(self, 'computed_'+inspect.currentframe().f_back.f_code.co_name,
                value)

    @property
    def RMS(self):
        raise RuntimeError(
            '%s not implemented' % (inspect.currentframe().f_code.co_name))

    @property
    def FWHM(self):
        raise RuntimeError(
            '%s not implemented' % (inspect.currentframe().f_code.co_name))

    @property
    def fourSigma_RMS(self):
        raise RuntimeError(
            '%s not implemented' % (inspect.currentframe().f_code.co_name))

    @property
    def fourSigma_FWHM(self):
        raise RuntimeError(
            '%s not implemented' % (inspect.currentframe().f_code.co_name))

    @property
    def full_bunch_length(self):
        raise RuntimeError(
            '%s not implemented' % (inspect.currentframe().f_code.co_name))

    @property
    def integral(self):
        raise RuntimeError(
            '%s not implemented' % (inspect.currentframe().f_code.co_name))

    def profile(self):
        r""" Computes the profile (e.g. in time)
        """
        raise RuntimeError(
            '%s not implemented' % (inspect.currentframe().f_code.co_name))

    def distribution(self):
        r""" Computes the distribution (e.g. in action)
        """
        raise RuntimeError(
            '%s not implemented' % (inspect.currentframe().f_code.co_name))

    def phase_space(self):
        r""" Computes the longitudinal phase space (in dt,dE)
        """
        raise RuntimeError(
            '%s not implemented' % (inspect.currentframe().f_code.co_name))

    def spectrum(self):
        r""" Returns the spectrum (Fourier transform of the profile)
        """
        raise RuntimeError(
            '%s not implemented' % (inspect.currentframe().f_code.co_name))

class Gaussian(_DistributionObject):
    r""" Gaussian profile function
    .. math::
        profile(t) = A\,\exp(-(t-t_0)^2/2\sigma^2) \,

    Parameters
    ----------
    args :
        If args has the three parameters amplitude, position, scale a
        distribution object is created. If passed as
        amplitude, position, scale, time the profile is computed at time
        with these parameters. If passed as time, y_data a Gaussian fit
        to these data is performed.
    scale_means: str
        controlls how 'scale' is interpreted. Valid options are 'RMS',
        'FWHM', 'fourSigma_RMS', 'fourSigma_FWHM' (and 'full_bunch_length'
        for other distributions)

    Attributes
    ----------
    amplitude : float
        maximum $A$ of the Gaussian profile
    position : float
        position $t_0$ of the the maximum
    RMS : float
        RMS of the distribution
    FWHM : float
        FWHM of the distribution
    fourSigma_RMS : float
        $4\times$ RMS
    fourSigma_FWHM : float
        Gaussian equivalent $4\sigma$ from FWHM of profile; i.e.
        $4\sigma_{FWHM} = 2/\sqrt(\ln 4) FWHM$
    full_bunch_length : np.inf
        infinity for Gaussian bunch
    integral : np.inf
        infinity for Gaussian bunch
    """

    def __init__(self, *args, scale_means=None,
                 store_data=None, **kwargs):

        _DistributionObject.__init__(self, store_data=store_data)

#        self.amplitude = amplitude  # other call signature
#        self.position = position  # other call signature

        if len(args) == 3:  # amplitude, position, scale
            self.amplitude = args[0]
            self.position = args[1]
            scale = args[2]
        elif len(args) == 4:  # amplitude, position, scale; profile computed below
            self.amplitude = args[0]
            self.position = args[1]
            scale = args[2]
        elif len(args) == 2:  # fit Gaussian to time, y_data
            fitPars = profile.gaussianFit(args[0], args[1])
            self.amplitude, self.position, scale = fitPars
        else:
            raise ValueError("invalid number of arguments")

        if scale_means is None:
            scale_means = rcBLonDparams['distribution.scale_means']

        if scale_means == 'RMS':
            self.RMS = scale
        elif scale_means == 'FWHM':
            self.FWHM = scale
        elif scale_means == 'fourSigma_RMS':
            self.fourSigma_RMS = scale
        elif scale_means == 'fourSigma_FWHM':
            self.fourSigma_FWHM = scale
        elif scale_means == 'full_bunch_length':
            raise ValueError("'full_bunch_length' argument no possible for"
                             + " Gaussian")

        if len(args) == 4:
            # store_data needs to be True, since __init__ needs to return None
            # TODO: maybe raise a warning, since this overwrites 'store_data'
            self.profile(args[3], store_data=True)

    def _computeBunchlenghtsFromRMS(self, RMS):
        FWHM = 2*np.sqrt(np.log(4)) * RMS
        # return order is RMS, FWHM, fourSigma_RMS, fourSigma_FWHM, full_bunch_length
        return RMS, FWHM, 4*RMS, 2/np.sqrt(np.log(4)) * FWHM, np.inf

    def _computeBunchlengths(self, value, scale_means):
        if scale_means == 'RMS':
            bls = self._computeBunchlenghtsFromRMS(value)
        elif scale_means == 'FWHM':
            bls = self._computeBunchlenghtsFromRMS(value / (2*np.sqrt(np.log(4))))
        elif scale_means == 'fourSigma_RMS':
            bls = self._computeBunchlenghtsFromRMS(value/4)
        elif scale_means == 'fourSigma_FWHM':
            bls = self._computeBunchlenghtsFromRMS(value/4)
        return bls

    @property
    def RMS(self):
        return self._RMS

    @RMS.setter
    def RMS(self, value):
        check_greater_zero(value, inspect.currentframe().f_code.co_name)
        self._RMS, self._FWHM, self._fourSigma_RMS, self._fourSigma_FWHM,\
            self._full_bunch_length = self._computeBunchlenghtsFromRMS(value)

    @property
    def FWHM(self):
        return self._FWHM

    @FWHM.setter
    def FWHM(self, value):
        check_greater_zero(value, inspect.currentframe().f_code.co_name)
        self.RMS = value / (2*np.sqrt(np.log(4)))  # updates all other parameters

    @property
    def fourSigma_RMS(self):
        return self._fourSigma_RMS

    @fourSigma_RMS.setter
    def fourSigma_RMS(self, value):
        check_greater_zero(value, inspect.currentframe().f_code.co_name)
        self.RMS = value / 4  # updates all other parameters

    @property
    def fourSigma_FWHM(self):
        return self._fourSigma_FWHM

    @fourSigma_FWHM.setter
    def fourSigma_FWHM(self, value):
        check_greater_zero(value, inspect.currentframe().f_code.co_name)
        self.RMS = value / 4  # updates all other parameters

    @property
    def full_bunch_length(self):
        return self._full_bunch_length

    @full_bunch_length.setter
    def full_bunch_length(self, value):
        self._full_bunch_length = np.inf

    @property
    def integral(self):
        return self.amplitude*self._RMS*np.sqrt(2*np.pi)

    def profile(self, x, *args, **kwargs):
        """ Computes the Gaussian profile at x
        """

        if 'store_data' in kwargs:
            self.store_data = kwargs['store_data']

        if len(args) == 0:
            return self._return_function(
                    self._profile(x, self.amplitude, self.position, self.RMS))
        else:
            amplitude = args[0]
            position = args[1]

            if 'scale_means' not in kwargs:
                scale_means = rcBLonDparams['distribution.scale_means']
            else:
                scale_means = kwargs['scale_means']

            RMS = self._computeBunchlengths(args[2], scale_means)[0]  # first return value is RMS

            return self._return_function(
                    self._profile(x, amplitude, position, RMS))

    def _profile(self, x, amplitude, position, RMS):
        return amplitude * np.exp(-0.5*(x-position)**2/RMS**2)

    def spectrum(self, f, **kwargs):
        """ Computes the Gaussian spectrum at frequency f
        """
        #TODO implement different factors of Fouriertransform

        if 'store_data' in kwargs:
            self.store_data = kwargs['store_data']

        return_value = np.sqrt(2*np.pi)*self.amplitude * self.RMS\
            * np.exp(-0.5*(2*np.pi*f*self.RMS)**2)

        if self.position != 0.0:  # assures real spectrum for symmetric profile
            return_value = return_value * np.exp(-2j*np.pi*self.position * f)

        return self._return_function(return_value)

# class BinomialAmplitudeN(_DistributionObject):
#
#     def __init__(self, amplitude, position, scale, mu, scale_means=None, **kwargs):
#         r"""
#         amplitude, center, scale, others
#         scale_means='RMS', 'FWHM', ...
#         """
#         _DistributionObject.__init__(self)
#
#         self.amplitude = amplitude
#         self.position = position
#         self.mu = mu
#
#         if scale_means is None:
#             scale_means = rcBLonDparams['distribution.scale_means']
#
#         if scale_means == 'RMS':
#             self.RMS = scale
#         elif scale_means == 'FWHM':
#             self.FWHM = scale
#         elif scale_means == 'fourSigma_RMS':
#             self.fourSigma_RMS = scale
#         elif scale_means == 'fourSigma_FWHM':
#             self.fourSigma_FWHM = scale
#         elif scale_means == 'full_bunch_length':
#             self.full_bunch_length = scale
#
#     @property
#     def full_bunch_length(self):
#         return self._full_bunch_length
#     @full_bunch_length.setter
#     def full_bunch_length(self, value):
#         check_greater_zero(value, inspect.currentframe().f_code.co_name)
#         self._full_bunch_length = value
#         self._RMS = value / np.sqrt(16+8*self.mu)
#         self._FWHM = value * np.sqrt(1-1/2**(1/(self.mu+0.5)))
#         self._fourSigma_RMS = value / np.sqrt(1+0.5*self.mu)
#         self._fourSigma_FWHM = 2/np.sqrt(np.log(4)) * self._FWHM
#
#     @property
#     def RMS(self):
#         return self._RMS
#     @RMS.setter
#     def RMS(self, value):
#         check_greater_zero(value, inspect.currentframe().f_code.co_name)
#         self.full_bunch_length = np.sqrt(16+8*self.mu) * value   # updates all other parameters
#
#     @property
#     def FWHM(self):
#         return self._FWHM
#     @FWHM.setter
#     def FWHM(self, value):
#         check_greater_zero(value, inspect.currentframe().f_code.co_name)
#         self.full_bunch_length = value / np.sqrt(1-1/2**(1/(self.mu+0.5)))  # updates all other parameters
#
#     @property
#     def fourSigma_RMS(self):
#         return self._fourSigma_RMS
#     @fourSigma_RMS.setter
#     def fourSigma_RMS(self, value):
#         check_greater_zero(value, inspect.currentframe().f_code.co_name)
#         self.full_bunch_length = value * np.sqrt(1+0.5*self.mu)  # updates all other parameters
#
#     @property
#     def fourSigma_FWHM(self):
#         return self._fourSigma_FWHM
#     @fourSigma_FWHM.setter
#     def fourSigma_FWHM(self, value):
#         check_greater_zero(value, inspect.currentframe().f_code.co_name)
#         self.FWHM = value * 0.5*np.sqrt(np.log(4))  # updates all other parameters
#
#     def profile(self, x):
#         """ Returns the Binomial amplitude profile at x
#         """
#
#         try:
#             return_value = np.zeros(len(x))
#
#             indexes = np.abs(x-self.position) <= self.full_bunch_length/2
#             return_value[indexes] = self.amplitude\
#                 * (1- (2*(x[indexes]-self.position)/self.full_bunch_length)**2)**(self.mu+0.5)
#         except:
#             if np.abs(x-self.position) <= self.full_bunch_length/2:
#                 return_value = self.amplitude\
#                     * (1- (2*(x-self.position)/self.full_bunch_length)**2)**(self.mu+0.5)
#             else:
#                 return_value = 0.0
#         return return_value
#
#     def spectrum(self, f):
#         """ Returns the Binomial amplitude spectrum at frequency f
#         """
#
#         return_value = self.amplitude * self.full_bunch_length * np.sqrt(np.pi)\
#             * special_fun.gamma(self.mu+1.5) / (2*special_fun.gamma(self.mu+2))\
#             * special_fun.hyp0f1(self.mu+2, -(0.5*np.pi*self.full_bunch_length*f)**2)
#
#         if self.position != 0.0:
#             return_value = return_value * np.exp(-2j*np.pi*self.position * f)
#
#         return return_value

def gaussian(time, *fitParameters):
    '''
    Gaussian line density
    '''

    amplitude = fitParameters[0]
    bunchPosition = fitParameters[1]
    sigma = abs(fitParameters[2])

    lineDensityFunction = amplitude * np.exp(
        -(time-bunchPosition)**2/(2*sigma**2))

    return lineDensityFunction

def generalizedGaussian(time, *fitParameters):
    '''
    Generalized gaussian line density
    '''

    amplitude = fitParameters[0]
    bunchPosition = fitParameters[1]
    alpha = abs(fitParameters[2])
    exponent = abs(fitParameters[3])

    lineDensityFunction = amplitude * np.exp(
        -(np.abs(time - bunchPosition)/(2*alpha))**exponent)

    return lineDensityFunction

def waterbag(time, *fitParameters):
    '''
    Waterbag distribution line density
    '''

    amplitude = fitParameters[0]
    bunchPosition = fitParameters[1]
    bunchLength = abs(fitParameters[2])

    lineDensityFunction = np.zeros(len(time))
    lineDensityFunction[np.abs(time-bunchPosition) < bunchLength/2] = \
        amplitude * (1-(
            (time[np.abs(time-bunchPosition) < bunchLength/2]-bunchPosition) /
            (bunchLength/2))**2)**0.5

    return lineDensityFunction

def parabolicLine(time, *fitParameters):
    '''
    Parabolic line density
    '''

    amplitude = fitParameters[0]
    bunchPosition = fitParameters[1]
    bunchLength = abs(fitParameters[2])

    lineDensityFunction = np.zeros(len(time))
    lineDensityFunction[np.abs(time-bunchPosition) < bunchLength/2] = \
        amplitude * (1-(
            (time[np.abs(time-bunchPosition) < bunchLength/2]-bunchPosition) /
            (bunchLength/2))**2)

    return lineDensityFunction

def parabolicAmplitude(time, *fitParameters):
    '''
    Parabolic in action line density
    '''

    amplitude = fitParameters[0]
    bunchPosition = fitParameters[1]
    bunchLength = abs(fitParameters[2])

    lineDensityFunction = np.zeros(len(time))
    lineDensityFunction[np.abs(time-bunchPosition) < bunchLength/2] = \
        amplitude * (1-(
            (time[np.abs(time-bunchPosition) < bunchLength/2]-bunchPosition) /
            (bunchLength/2))**2)**1.5

    return lineDensityFunction

def binomialAmplitude2(time, *fitParameters):
    '''
    Binomial exponent 2 in action line density
    '''

    amplitude = fitParameters[0]
    bunchPosition = fitParameters[1]
    bunchLength = abs(fitParameters[2])

    lineDensityFunction = np.zeros(len(time))
    lineDensityFunction[np.abs(time-bunchPosition) < bunchLength/2] = \
        amplitude * (1-(
            (time[np.abs(time-bunchPosition) < bunchLength/2]-bunchPosition) /
            (bunchLength/2))**2)**2.

    return lineDensityFunction

def binomialAmplitudeN(time, *fitParameters):
    '''
    Binomial exponent n in action line density
    '''

    amplitude = fitParameters[0]
    bunchPosition = fitParameters[1]
    bunchLength = abs(fitParameters[2])
    exponent = abs(fitParameters[3])

    lineDensityFunction = np.zeros(len(time))
    lineDensityFunction[np.abs(time-bunchPosition) < bunchLength/2] = \
        amplitude * (1-(
            (time[np.abs(time-bunchPosition) < bunchLength/2]-bunchPosition) /
            (bunchLength/2))**2)**exponent

    return lineDensityFunction

def _binomial_full_to_rms(full_bunch_length, exponent):
    '''
    Returns the RMS bunch length from the full bunch length and exponent

    - TODO: To be included as @property in the Binomial/Parabolic distributions

    '''

    return full_bunch_length/(2*np.sqrt(3+2*exponent))

def _binomial_full_to_fwhm(full_bunch_length, exponent, level=0.5):
    '''
    Returns the FWHM from the full bunch length and exponent

    - TODO: To be included as @property in the Binomial/Parabolic distributions

    '''

    return full_bunch_length*np.sqrt(1-level**(1/exponent))

def _binomial_integral(amplitude, full_bunch_length, exponent):
    '''
    Returns the integrated profile

    - TODO: To be included as @property in the Binomial/Parabolic distributions

    '''

    return amplitude*full_bunch_length*np.sqrt(np.pi)*gamma(1.+exponent)/(
        2.*gamma(1.5+exponent))

def cosine(time, *fitParameters):
    '''
    * Cosine line density *
    '''

    amplitude = fitParameters[0]
    bunchPosition = fitParameters[1]
    bunchLength = abs(fitParameters[2])

    lineDensityFunction = np.zeros(len(time))
    lineDensityFunction[np.abs(time-bunchPosition) < bunchLength/2] = \
        amplitude * np.cos(
            np.pi*(time[np.abs(time-bunchPosition) < bunchLength/2] -
                   bunchPosition) / bunchLength)

    return lineDensityFunction

def cosineSquared(time, *fitParameters):
    '''
    * Cosine squared line density *
    '''

    amplitude = fitParameters[0]
    bunchPosition = fitParameters[1]
    bunchLength = abs(fitParameters[2])

    lineDensityFunction = np.zeros(len(time))
    lineDensityFunction[np.abs(time-bunchPosition) < bunchLength/2] = \
        amplitude * np.cos(
            np.pi*(time[np.abs(time-bunchPosition) < bunchLength/2] -
                   bunchPosition) / bunchLength)**2.

    return lineDensityFunction


import glob
import io
import os
import pandas as pd
import tensorflow.compat.v1 as tf
import xml.etree.ElementTree as ET

from collections import namedtuple
from object_detection.utils import dataset_util, label_map_util
from PIL import Image
from training_server.utils import download_file

class DatasetBuilder:
    def __init__(
        self,
        dataset_dir: str,
        labels_path: str,
        output_path: str,
    ):
        self.dataset_dir = dataset_dir
        self.labels_path = labels_path
        self.output_path = output_path

    def build(self, dataset):
        self.download_dataset(dataset, self.dataset_dir)
        self.process_dataset(self.dataset_dir, self.labels_path, self.output_path)

    def download_dataset(self, dataset, output_dir):
        for data in dataset:
            filename = data["filename"]
            image_url = data["imageURL"]
            annotation_url = data["annotationURL"]
            image_path = os.path.join(output_dir, f"{filename}{file_extension(image_url)}")
            annotation_path = os.path.join(output_dir, f"{filename}{file_extension(annotation_url)}")
            download_file(image_url, image_path)
            download_file(annotation_url, annotation_path)

    def process_dataset(
        self,
        xml_dir: str,
        labels_path: str,
        output_path: str,
    ):
        generate_label_map(xml_dir, labels_path)
        generate_tfrecord(xml_dir, labels_path, output_path)

def generate_label_map(xml_dir: str, output_path: str):
    label_map = {}
    with open(output_path, "w") as f:
        for xml_file in glob.glob(xml_dir + '/*.xml'):
            tree = ET.parse(xml_file)
            root = tree.getroot()
            for member in root.findall('object'):
                class_name = member.find('name').text
                if class_name not in label_map:
                    label_map[class_name] = len(label_map) + 1
                    f.write(f"item {{\n  id: {label_map[class_name]}\n  name: '{class_name}'\n}}\n\n")
    return label_map

def xml_to_csv(path):
    """Iterates through all .xml files (generated by labelImg) in a given directory and combines
    them in a single Pandas dataframe.

    Parameters:
    ----------
    path : str
        The path containing the .xml files
    Returns
    -------
    Pandas DataFrame
        The produced dataframe
    """

    xml_list = []
    for xml_file in glob.glob(path + '/*.xml'):
        tree = ET.parse(xml_file)
        root = tree.getroot()
        filename = root.find('filename').text
        width = int(root.find('size').find('width').text)
        height = int(root.find('size').find('height').text)
        for member in root.findall('object'):
            bndbox = member.find('bndbox')
            value = (
                filename,
                width,
                height,
                member.find('name').text,
                int(bndbox.find('xmin').text),
                int(bndbox.find('ymin').text),
                int(bndbox.find('xmax').text),
                int(bndbox.find('ymax').text),
            )
            xml_list.append(value)
    column_name = ['filename', 'width', 'height', 'class', 'xmin', 'ymin', 'xmax', 'ymax']
    xml_df = pd.DataFrame(xml_list, columns=column_name)
    return xml_df

def class_text_to_int(labels_path, row_label):
    label_map = label_map_util.load_labelmap(labels_path)
    label_map_dict = label_map_util.get_label_map_dict(label_map)
    return label_map_dict[row_label]

def split(df, group):
    data = namedtuple('data', ['filename', 'object'])
    gb = df.groupby(group)
    return [data(filename, gb.get_group(x)) for filename, x in zip(gb.groups.keys(), gb.groups)]

def create_tf_example(group, path, labels_path):
    with tf.gfile.GFile(os.path.join(path, '{}'.format(group.filename)), 'rb') as fid:
        encoded_jpg = fid.read()
    encoded_jpg_io = io.BytesIO(encoded_jpg)
    image = Image.open(encoded_jpg_io)
    width, height = image.size

    filename = group.filename.encode('utf8')
    image_format = b'jpg'
    xmins = []
    xmaxs = []
    ymins = []
    ymaxs = []
    classes_text = []
    classes = []

    for index, row in group.object.iterrows():
        xmins.append(row['xmin'] / width)
        xmaxs.append(row['xmax'] / width)
        ymins.append(row['ymin'] / height)
        ymaxs.append(row['ymax'] / height)
        classes_text.append(row['class'].encode('utf8'))
        classes.append(class_text_to_int(labels_path, row['class']))

    tf_example = tf.train.Example(
        features=tf.train.Features(
            feature={
                'image/height': dataset_util.int64_feature(height),
                'image/width': dataset_util.int64_feature(width),
                'image/filename': dataset_util.bytes_feature(filename),
                'image/source_id': dataset_util.bytes_feature(filename),
                'image/encoded': dataset_util.bytes_feature(encoded_jpg),
                'image/format': dataset_util.bytes_feature(image_format),
                'image/object/bbox/xmin': dataset_util.float_list_feature(xmins),
                'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs),
                'image/object/bbox/ymin': dataset_util.float_list_feature(ymins),
                'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs),
                'image/object/class/text': dataset_util.bytes_list_feature(classes_text),
                'image/object/class/label': dataset_util.int64_list_feature(classes),
            }
        )
    )
    return tf_example

def generate_tfrecord(xml_dir: str, labels_path: str, output_path: str):
    writer = tf.python_io.TFRecordWriter(output_path)
    image_dir = xml_dir
    path = os.path.join(image_dir)
    examples = xml_to_csv(xml_dir)
    grouped = split(examples, 'filename')
    for group in grouped:
        tf_example = create_tf_example(group, path, labels_path)
        writer.write(tf_example.SerializeToString())
    writer.close()
    print('Successfully created the TFRecord file: {}'.format(output_path))

def file_extension(path):
    return os.path.splitext(path)[1]


from typing import List
from collections import defaultdict
from collections import deque
import heapq
class Twitter:

    def __init__(self):
        """
        Initialize your data structure here.
        """
        self.dictfee = defaultdict(set)
        self.dicttweet = defaultdict(deque)
        self.timer = 0

    def postTweet(self, userId: int, tweetId: int) -> None:
        """
        Compose a new tweet.
        """

        self.dicttweet[userId].append((self.timer,tweetId))
        self.timer+=1
        if len(self.dicttweet[userId])>10:
            self.dicttweet[userId].popleft()

    def getNewsFeed(self, userId: int) -> List[int]:
        """
        Retrieve the 10 most recent tweet ids in the user's news feed. Each item in the news feed must be posted by users who the user followed or by the user herself. Tweets must be ordered from most recent to least recent.
        """
        feed = list(self.dicttweet[userId])
        heapq.heapify(feed)
        for followee in self.dictfee[userId]:
            for time,tweet in self.dicttweet[followee]:
                if len(feed)< 10:
                    heapq.heappush(feed,(time,tweet))
                elif time > feed[0][0]:
                    heapq.heappushpop(feed,(time,tweet))
        ans = []
        while len(feed)>0:
            ans.append(heapq.heappop(feed)[1])
        return ans[::-1]

    def follow(self, followerId: int, followeeId: int) -> None:
        """
        Follower follows a followee. If the operation is invalid, it should be a no-op.
        """
        if followerId != followeeId:
            self.dictfee[followerId].add(followeeId)

    def unfollow(self, followerId: int, followeeId: int) -> None:
        """
        Follower unfollows a followee. If the operation is invalid, it should be a no-op.
        """
        if followeeId in self.dictfee[followerId]:
            self.dictfee[followerId].remove(followeeId)

# Your Twitter object will be instantiated and called as such:
# obj = Twitter()
# obj.postTweet(userId,tweetId)
# param_2 = obj.getNewsFeed(userId)
# obj.follow(followerId,followeeId)
# obj.unfollow(followerId,followeeId)
T= Twitter()
T.postTweet(1,5)
print(T.getNewsFeed(1))
T.follow(1,2)
T.postTweet(2,6)
print(T.getNewsFeed(1))
T.unfollow(1,2)
print(T.getNewsFeed(1))


import http.client as http
import json

import time

import utils

NETWORK_ERRORS = http.HTTPException, ConnectionError, OSError

HEADERS = {
    'User-Agent': "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0",
    'Accept-Charset': 'utf-8',
}

class Connection(object):

    def __init__(self, host=utils.BOARD_URL):
        self._host = host
        self._connect()

    def _repeat_on(exceptions):
        def _repeat_on_error(function):
            def _f(self, *args, **kwargs):
                while True:
                    try:
                        return function(self, *args, **kwargs)
                    except exceptions as e:
                        utils.inform(e, level=utils.WARNING)
                        self._connect()
                        time.sleep(1)
            return _f
        return _repeat_on_error

    @_repeat_on(NETWORK_ERRORS)
    def _connect(self):
        self._conn = http.HTTPSConnection(self._host)
        self._set_headers()
        utils.inform('Connected to {}'.format(self._host), level=utils.INFO)

    def _set_cookie(self):
        resp = self.get_response('/')
        self._headers['Cookie'] = resp.getheader('set-cookie')
        resp.read()

    def _set_headers(self):
        self._headers = HEADERS

    @_repeat_on(NETWORK_ERRORS)
    def _get_response(self, request):
        utils.inform('Requesting {}'.format(request), level=utils.INFO)
        self._conn.request('GET', request, headers=self._headers)
        resp = self._conn.getresponse()
        utils.inform('Response is {}: {}'.format(
            resp.status, resp.reason), level=utils.INFO)
        return resp

    def get_response(self, request):
        return self._get_response(request)

    def get_json(self, request):
        json_data = {}

        resp = self._get_response(request)
        data = resp.read()

        try:
            data = data.decode('utf8')
        except (AttributeError, UnicodeDecodeError):
            pass

        if resp.status == 200:
            try:
                json_data = json.loads(data)
            except ValueError as e:
                utils.inform(e, level=utils.WARNING)
            except TypeError as e:
                utils.inform(e, level=utils.WARNING)

        return resp.status, json_data

    def get_file(self, request):
        return self._get_response(request).read()


from django.shortcuts import render, HttpResponse, redirect
from account.forms import RegistrationForm, UserProfileForm
from django.contrib.auth.decorators import login_required
from django.db import transaction
from django.contrib import messages
from case.models import case , Task , Document
from case.forms import Reg_Case, Reg_Task , DocumentForm
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.http import HttpResponseRedirect
import os
from django.core.files.storage import default_storage

#python imports
import spacy
from spacy import displacy
from collections import Counter
import en_core_web_sm
nlp = spacy.load('en')
import speech_recognition as sr

from bs4 import BeautifulSoup
import requests
import re

def url_to_string(url):
    res = requests.get(url)
    html = res.text
    soup = BeautifulSoup(html, 'html.parser')
    for script in soup(["script", "style", 'aside']):
        script.extract()
    return " ".join(re.split(r'[\n\t]+', soup.get_text()))

def getfreq(temp_data):
    mp_codes={}
    mp_tags={}
    unique_codes=[]
    for i in range(len(temp_data)):
        if temp_data[i][0].lower() in mp_codes.keys():
            mp_codes[temp_data[i][0].lower()]=mp_codes[temp_data[i][0].lower()]+1
        else:
            mp_codes[temp_data[i][0].lower()]=1

        if temp_data[i][0].lower() in mp_tags.keys():
            continue
        else:
            mp_tags[temp_data[i][0].lower()]=temp_data[i][1].lower()

    return mp_codes,mp_tags

"""
ny_bb = url_to_string('https://www.coursera.org/instructor/andrewng')
article = nlp(ny_bb)
len(article.ents)
print(article)
"""

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
#from django.conf.settings import MEDIA_ROOT
#from django.urls import reverse
# Create your views here.

@transaction.atomic
def register(request):
	if request.method == 'POST':
		form = RegistrationForm(request.POST)
		profile_form = UserProfileForm(request.POST)
		if form.is_valid() and profile_form.is_valid():
			user = form.save()
			user.userprofile.designation = profile_form.cleaned_data.get('designation')
			user.userprofile.city = profile_form.cleaned_data.get('city')
			user.userprofile.unq = profile_form.cleaned_data.get('unq')
			user.userprofile.save()
			return redirect('/account/login')
		else:
			messages.error(request,form.errors)
			args = {'form':form, 'profile_form':profile_form}
			return render(request, 'account/reg_form.html', args)
	else:
		form = RegistrationForm()
		profile_form = UserProfileForm()
		args = {'form':form, 'profile_form':profile_form}
		return render(request, 'account/reg_form.html', args)

@login_required
def view_profile(request):
    if request.user.userprofile.designation == 'investigation head':
        new_cases = case.objects.filter(case_info = '',investigation_head = request.user.userprofile)
        ongoing_cases = case.objects.filter(status='Under Investigation',investigation_head=request.user.userprofile).count()
        completed_cases = case.objects.filter(status='Completed',investigation_head=request.user.userprofile).count()
        args = {'user': request.user,'new_cases':new_cases,'ongoing_cases':ongoing_cases,'completed_cases':completed_cases}
    elif request.user.userprofile.designation == 'member':
        form3 = DocumentForm()
        mp_codes={}
        mp_tags={}
        mp_places={}
        probability={}
        main_mp={}
        if request.method == 'POST':
            form3 = DocumentForm()
            print(request.user.userprofile.unq)
            t1 = Task.objects.filter(task_member=request.user.userprofile)
            print(t1)
            k = Document.objects.filter(unq = t1[0].case_number)
            print(t1[0].task_number)
            #1 file
            #print(k[0].file1)
            f = default_storage.open(os.path.join('', str(k[0].file1)), 'r')
            data = f.read()
            f.close()
            #print(data)
            article1 = nlp(data)
            k1 = [(X.text, X.label_) for X in article1.ents]
            print(len(k1))

            #2 file
            f = default_storage.open(os.path.join('', str(k[0].csvfile1)), 'r')
            data = f.read().splitlines()
            f.close()
            for i in range(len(data)):
                t1 = data[i].split(",")
                if t1[0].lower() in mp_places.keys():
                    continue
                else:
                    mp_places[t1[0].lower()]=t1[1].lower()

            print(mp_places)
            #article1 = nlp(data)
            print(len(k1))

            #1 csvfile1
            f = default_storage.open(os.path.join('', str(k[0].audio1)), 'rb')
            #data = f.read()
            #print(f)
            r = sr.Recognizer()
            harvard = sr.AudioFile(f)

            #1 audiofile1
            with harvard as source:
                audio = r.record(source)
            k = r.recognize_google(audio)
            article1 = nlp(k)
            k1.extend([(X.text, X.label_) for X in article1.ents])
            f.close()
            print([(X.text, X.label_) for X in article1.ents])
            print('hello')
            print('hello')
            #k1.extend([(X.text, X.label_) for X in article1.ents])

            print(len(k1))
            print(len(set(k1)))

            mp_codes, mp_tags = getfreq(k1)
            print(mp_codes)
            print(mp_tags)

            total =0
            for i in mp_tags:
                print("i ",mp_codes[i])
                total=total+mp_codes[i]

            for i in mp_codes:
                probability[i] = mp_codes[i]/total

            for i in mp_places:
                places = mp_places[i]
                for j in mp_tags:
                    if(i==j):
                        if places in main_mp.keys():
                            main_mp[places] = main_mp[places]+probability[j]
                        else:
                            main_mp[places] = probability[j]

            print("main ",main_mp)
            print("probabity" , probability)
            print("total ",total)
        form = Reg_Task()
        ongoing_task = Task.objects.filter()
        ongoing_cases = case.objects.filter(status='Under Investigation',members=request.user.userprofile).count()
        completed_cases = case.objects.filter(status='Completed',members=request.user.userprofile).count()
        args = {'mp_main':main_mp , 'mp_codes':mp_codes, 'mp_tags':mp_tags, 'user': request.user, 'ongoing_cases':ongoing_cases,'completed_cases':completed_cases , 'form':form, 'ongoing_task':ongoing_task,'form3':form3}
    else:
        args = {'user':request.user}
        if request.method == 'POST':
            form = Reg_Case(request.POST)
            if form.is_valid():
                form.save()
                return redirect('/account/profile')
            else:
                form = Reg_Case()
            args = {'user':request.user,'form':form}

    return render(request, 'account/profile.html', args)

"""
@login_required
def function1(request):
    form = NewInput1()
    if request.method=="POST":
        form = NewInput1(request.POST)

        if form.is_valid():

			model_instance = form.save(commit=False)
            article1=""
            article2="32"
            article3=""
            article4=""
            article5=""

            try:
                data1 = request.FILES['file1']
                f1 = data1.read().decode('utf-8')#.splitlines()
                article1 = nlp(f1)
                k1 = [(X.text, X.label_) for X in article1.ents]
            except:
                1

            try:
                data2 = request.FILES['file2']
                f2 = data2.read().decode('utf-8')#.splitlines()
                article2 = nlp(f2)
                print(article2)
                k2 = [(X.text, X.label_) for X in article2.ents]
            except:
                1

            try:
                data3 = request.FILES['file3']
                f3 = data3.read().decode('utf-8')#.splitlines()
                article3 = nlp(f3)
                k3 = [(X.text, X.label_) for X in article3.ents]
            except:
                1

            try:
                data4 = request.FILES['csvfile1']
                f4 = data4.read().decode('utf-8')#.splitlines()
                print(f4.split('\n')[3])
            except:
                1

            try:
                ny_bb4 = url_to_string(model_instance.url1)
                article4 = nlp(ny_bb4)
                print(set([(X.text, X.label_) for X in article4.ents]))
            except:
                1

            try:
                ny_bb5 = url_to_string(model_instance.url2)
                articler5 = nlp(ny_bb5)
            except:
                1

            return render(request , 'filetonlp/input1.html', {'form':form , 'article2':article4})

    return render(request , account/profile.html, {'form':form })
"""


import numpy as np
import matplotlib.pyplot as pt
import pandas as pd
from sklearn.tree import DecisionTreeClassifier

data = pd.read_csv("dataset/train.csv")
print(data.columns)
data = data.to_numpy()

classifier = DecisionTreeClassifier()

x_train = data[0:21000, 1:]
y_train = data[0:21000, 0]

classifier.fit(x_train, y_train)

# Now testing the dataset
x_test = data[21000:,1:]
y_test = data[21000:, 0]

d = x_test[8]
d.shape = (28, 28)
pt.imshow(255-d, cmap="gray")

pt.show()

# testing the accuracy
p = classifier.predict(x_test)
count = 0
for i in range(0, 21000):
    count+=1 if p[i] == y_test[i] else 0

print("Accuracy = ", (count/21000)*100)


import os
import sys
import numpy as np
import cv2
from PIL import Image
from matplotlib import pyplot as plt

from typing import Dict, Union, Sequence
from loggers import create_logger, error_traceback
logger = create_logger(logger_name='det_visualize')

__all__ = ['visualize_img', 'visualize_bbox', 'visualize_det', 'colormap']

def colormap(rgb: bool=False) -> np.ndarray:
    """获取可视化需要的颜色表
        desc(描述):
            Parameters:
                rgb: 图像是否为rgb格式(bool)
            Returns:
                色彩表
    Get colormap
    The code of this function is copied from https://github.com/facebookresearch/Detectron/blob/main/detectron/utils/colormap.py
    """
    color_list = np.array([
        0.000, 0.447, 0.741, 0.850, 0.325, 0.098, 0.929, 0.694, 0.125, 0.494,
        0.184, 0.556, 0.466, 0.674, 0.188, 0.301, 0.745, 0.933, 0.635, 0.078,
        0.184, 0.300, 0.300, 0.300, 0.600, 0.600, 0.600, 1.000, 0.000, 0.000,
        1.000, 0.500, 0.000, 0.749, 0.749, 0.000, 0.000, 1.000, 0.000, 0.000,
        0.000, 1.000, 0.667, 0.000, 1.000, 0.333, 0.333, 0.000, 0.333, 0.667,
        0.000, 0.333, 1.000, 0.000, 0.667, 0.333, 0.000, 0.667, 0.667, 0.000,
        0.667, 1.000, 0.000, 1.000, 0.333, 0.000, 1.000, 0.667, 0.000, 1.000,
        1.000, 0.000, 0.000, 0.333, 0.500, 0.000, 0.667, 0.500, 0.000, 1.000,
        0.500, 0.333, 0.000, 0.500, 0.333, 0.333, 0.500, 0.333, 0.667, 0.500,
        0.333, 1.000, 0.500, 0.667, 0.000, 0.500, 0.667, 0.333, 0.500, 0.667,
        0.667, 0.500, 0.667, 1.000, 0.500, 1.000, 0.000, 0.500, 1.000, 0.333,
        0.500, 1.000, 0.667, 0.500, 1.000, 1.000, 0.500, 0.000, 0.333, 1.000,
        0.000, 0.667, 1.000, 0.000, 1.000, 1.000, 0.333, 0.000, 1.000, 0.333,
        0.333, 1.000, 0.333, 0.667, 1.000, 0.333, 1.000, 1.000, 0.667, 0.000,
        1.000, 0.667, 0.333, 1.000, 0.667, 0.667, 1.000, 0.667, 1.000, 1.000,
        1.000, 0.000, 1.000, 1.000, 0.333, 1.000, 1.000, 0.667, 1.000, 0.167,
        0.000, 0.000, 0.333, 0.000, 0.000, 0.500, 0.000, 0.000, 0.667, 0.000,
        0.000, 0.833, 0.000, 0.000, 1.000, 0.000, 0.000, 0.000, 0.167, 0.000,
        0.000, 0.333, 0.000, 0.000, 0.500, 0.000, 0.000, 0.667, 0.000, 0.000,
        0.833, 0.000, 0.000, 1.000, 0.000, 0.000, 0.000, 0.167, 0.000, 0.000,
        0.333, 0.000, 0.000, 0.500, 0.000, 0.000, 0.667, 0.000, 0.000, 0.833,
        0.000, 0.000, 1.000, 0.000, 0.000, 0.000, 0.143, 0.143, 0.143, 0.286,
        0.286, 0.286, 0.429, 0.429, 0.429, 0.571, 0.571, 0.571, 0.714, 0.714,
        0.714, 0.857, 0.857, 0.857, 1.000, 1.000, 1.000
    ]).astype(np.float32)
    color_list = color_list.reshape((-1, 3)) * 255
    if not rgb:
        color_list = color_list[:, ::-1]
    return color_list

# 还未替换日志输出，仍然为标准输出
def visualize_img(img: Union[np.ndarray, str],
                  save_path: Union[None, str]=None,
                  show_img: bool=False,
                  is_rgb: bool=False) -> None:
    """利用pyplot可视化图像
        desc(描述):
            Parameters:
                img: 要显示图像的数据(numpy.ndarray)或要显示图像的路径(str)
                save_path: None或可视化结果图像的保存路径(str)
                           如果为None，则不保存
                show_img: 是否使用pyplot显示可视化结果(bool)
                is_rgb: 是否为RGB格式的图像数据，保证图像显示和保存时的色域正确
            Returns:
                None
    """
    if isinstance(img, str): # 为图像路径时
        if os.path.exists(img): # 路径存在
            img = Image.open(img) # 读取图像 -- rgb格式/gray格式
            is_rgb = True
        else: # 不存在的路径，抛出值异常
            try:
                raise ValueError()
            except:
                error_traceback(logger=logger,
                                lasterrorline_offset=9,
                                num_lines=6)
                logger.error('Summary: The img path does not exist.(path: {0})'.format(
                    img))
                sys.exit(1)
        img=np.array(img).astype('uint8') # 转换为numpy.ndarray数组数据-dtype为uint8
    elif isinstance(img, np.ndarray): # 为运行的图像数据
        img=img.astype('uint8') # 转换dtype为uint8
    else: # img参数类型异常，抛出类型异常
        try:
            raise ValueError()
        except:
            error_traceback(logger=logger,
                            lasterrorline_offset=22,
                            num_lines=19)
            logger.error('Summary: The img only support numpy.ndarray or str.(type: {0})'.format(
                type(img)))
            sys.exit(1)

    # 至少有一种可视化指令打开
    if show_img==False and save_path is None:
        try:
            raise ValueError()
        except:
            error_traceback(logger=logger,
                            lasterrorline_offset=6,
                            num_lines=3)
            logger.error('Summary: The function of visualize should open the show_img or enter the save_path.')
            sys.exit(1)

    if show_img:
        if is_rgb == False: # 输入图像数据为BGR格式
            # 三通道图像时，则将opencv的BGR转为RGB格式
            if len(img.shape)==3 and img.shape[2]==3:
                img = img[:, :, ::-1]
        # 可视化窗口显示图像
        plt.figure("visualize result") # 图像窗口命令
        plt.imshow(img) # 显示图像
        plt.axis("off") # 关闭坐标轴显示
        plt.title("result") # 图像标题
        plt.show() # 显示图像

    # 保存可视化图像
    if save_path: # 路径不为None
        if is_rgb == True: # 图像格式为RGB
            # 三通道图像时，则将RGB格式转为opencv的BGR格式
            if len(img.shape)==3 and img.shape[2]==3:
                img = img[:, :, ::-1]
        try: # 尝试保存图像
            cv2.imwrite(save_path, img)
            logger.info("The visualize result has save at: {0}.".format(save_path))
        except :
            logger.warning("The save_path is not a file.({0})".format(save_path))

def visualize_bbox(bboxs: np.ndarray,
                   draw_board: np.ndarray,
                   score_threshold: float=0.5,
                   labels: Union[Sequence[str], Dict[int, str]]=None,
                   use_rgb: bool=False) -> np.ndarray:
    """可视化bbox
        desc(描述):
            Parameters:
                bboxs: 要显示边界框的数据(numpy.ndarray)
                       shape: [N, 4] or [N, 6]
                       [0, 4] --> x1, y1, x2, y2
                       [0, 6] --> class, score, x1, y1, x2, y2
                draw_board: 画板背景图像(numpy.ndarray)或要显示图像的路径(str)
                score_threshold: 检测可视化输出的得分阈值(float)，小于该值的
                                 边界框不可视化
                                 value: [0., 1.)
                lables: 类别ID对应的类别名称(list|tuple|set|dict[int, str])
                use_rgb: 背景图像格式是否为rgb格式(bool)
            Returns:
                将边界框(当输入bboxs的shape为[N, 6]时包含类别)可视化
                后的图像(numpy.ndarray)
                shape: 与draw_board一致
    """
    # 判断bboxs是否含有类别与得分:
    # len(bboxs[0]) == 4，说明不包含
    include_cls = False if len(bboxs[0]) == 4 else True

    # 图像获取
    if isinstance(draw_board, str): # 为图像路径时
        if os.path.exists(draw_board): # 路径存在
            draw_board=Image.open(draw_board) # 读取图像
            use_rgb=True # PIL.Image读取的图像格式为RGB
        else: # 不存在的路径，抛出值异常
            try:
                raise ValueError()
            except:
                error_traceback(logger=logger,
                                lasterrorline_offset=10,
                                num_lines=7)
                logger.error('The draw_board path does not exist.(path: {0})'.format(
                    draw_board))
                sys.exit(1)
        draw_board=np.array(draw_board).astype('uint8') # 转换为numpy.ndarray数组数据-dtype为uint8
    elif isinstance(draw_board, np.ndarray): # 为运行的图像数据
        draw_board=draw_board.astype('uint8') # 转换dtype为uint8
    else: # img类型异常，抛出类型异常
        try:
            raise ValueError()
        except:
            error_traceback(logger=logger,
                            lasterrorline_offset=23,
                            num_lines=20)
            logger.error('The img only support numpy.ndarray or str.(type: {0})'.format(
                type(draw_board)))
            sys.exit(1)

    # 图像通道检查: 由于colormap为三通道的，所以强制所有可视化为三通道
    if len(draw_board.shape) == 2: # 单通道图像的处理方式
        draw_board = np.stack([draw_board]*3, axis=-1)
    elif len(draw_board.shape) == 3 and draw_board.shape[2] >= 4: # 多通道图像的处理方式
        draw_board = draw_board[:, :, :3]

    # 图像大小获取
    h, w, _ = draw_board.shape # 高、宽
    # 绘制参数控制
    scale_size = 512 # 字体以及线条大小缩放因子
    font_scale = max(min(max(h, w) / scale_size, 0.8), 0.5) # 字体scale
    font_thickness = int((max(h, w) / scale_size + 1) * 1.0) # 字体大小
    font_y_offset = font_thickness * 2 # 字体显示向上偏移一定像素
    rectangle_thickness = int((max(h, w) / scale_size + 1) * 1.0) # 线条大小

    # 获取颜色表
    _color_map = colormap(rgb=use_rgb)

    for idx, box in enumerate(bboxs): # 遍历每一个边界框并绘制
        # 类比id
        cls_id =  box[0].item() if include_cls else None
        # 类别得分
        score = box[1].item() if include_cls else None
        if score is not None and score < score_threshold: # 跳过不满足的bbox
            continue
        # 边界框
        bbox = box[2:] if include_cls else box
        # 绘制颜色
        _color = _color_map[int(cls_id)%80] if include_cls else _color_map[0]
        _color = tuple(int(i) for i in _color)

        # 绘制边界框
        cv2.rectangle(draw_board, [int(bbox[0]), int(bbox[1])], [int(bbox[2]), int(bbox[3])],
                      color=_color, thickness=rectangle_thickness)

        # 绘制类别描述: class_name + score
        if cls_id is not None: # 存在类别信息时
            # 可视化class_name的预处理
            # lables为None时直接使用class_id进行可视化
            cls_name = labels[int(cls_id)] + '-' + str(round(score, 2)) if \
                       labels is not None else str(int(cls_id)) + '-' + str(round(score, 2))
            cv2.putText(draw_board, cls_name,
                        (int(bbox[0]), int(bbox[1])-font_y_offset), fontFace=cv2.FONT_HERSHEY_SIMPLEX,
                        fontScale=font_scale, color=_color, thickness=font_thickness)

    return draw_board, use_rgb

def visualize_det(img, cls_and_bboxs,
                  score_threshold: float=0.5,
                  save_path: Union[None, str]=None,
                  labels: Union[Sequence[str], Dict[int, str]]=None,
                  use_rgb: bool=False,
                  show_img: bool=False) -> np.ndarray:
    """可视化检测模型的输出结果
        desc(描述):
            Parameters:
                img: 检测输入图像的数据(numpy.ndarray)或检测输入图像的路径(str)
                cls_and_bboxs: 检测输出边界框与对应类别的数据(numpy.ndarray)
                               shape: [N, 6]
                               [0, 6] --> class, score, x1, y1, x2, y2
                score_threshold: 检测可视化输出的得分阈值(float)，小于该值的
                                 边界框不可视化
                                 value: [0., 1.)
                save_path: None或可视化结果图像的保存路径(str)
                           如果为None，则不保存
                lables: 类别ID对应的类别名称(list|tuple|set|dict[int, str])
                use_rgb: 背景图像格式是否为rgb格式(bool)
                show_img: 是否使用pyplot显示可视化结果(bool)
            Returns:
                将边界框与类别可视化后的图像(numpy.ndarray)
                shape: 与输入真实图像一致
    """
    # 可视化bbox到目标图像
    visualize_result, _is_rgb = visualize_bbox(bboxs=cls_and_bboxs,
                                                draw_board=img,
                                                score_threshold=score_threshold,
                                                labels=labels,
                                                use_rgb=use_rgb)

    # 可视化窗口展示与可视化结果图像保存
    visualize_img(visualize_result, save_path=save_path, show_img=show_img, is_rgb=_is_rgb)

    # 返回后续可能会用上的可视化结果
    return visualize_result


import pandas as pd
import argparse
#cd Documents/education/galvanize/code/grader

parser = argparse.ArgumentParser(descrption='Find top students')
parser.add_argument('filename', metavar='F',type=str)
parser.add_argument('percentage', metavar='F',type=int)
args = parser.parse_args()
print('args:', args)
filename = args.filename
percentage = args.percentage

#df = pd.read_csv('data/grades.csv')

df = pd.read_csv(filename)
#df.head(2)
df1 = df.iloc[:,[0,1,3,5]]
#df1.head(2)
#df1.apply(lambda z: print(type(z)))
#df1.apply(lambda z: print(len(z)))
df1.apply(lambda row: (row[1]), axis=1)
#df1.head(2)
df1['final'] = df1.apply(lambda row: (row[1]+row[2]+(2*row[3]))/400, axis=1)
#df1.head(2)
#df1.head(2)
#num_students = int(len(df) * 0.01)
num_students = int(len(df1) * percentage/100)
#num_students
indicies = df1.final.argsort()
#indicies.head()
#df1.final[1424]
filtered_students = df.iloc[indicies[-num_students:]][::-1]
#filtered_students
filtered_students.student_id.values
ids = filtered_students.student_id.values
print(ids)


import json
from random import randint
from administrador_datos import cargar_datos
from  jugador import *

def jugar():
    pregunta = nueva_pregunta()

    if jugador["escalon"] != 8:
        print()
        print("Escalon N°", jugador["escalon"], "!!!!!")
        print(jugador["nombre"], "Puntaje:", jugador["puntaje"])

        if es_correcta(pregunta): # hace la pregunta y verifica y se contesta correctamente
            print("Es correcto")
            sumar_puntaje()
            subir_escalon()
            jugar()
        else:
            print("Es Incorrecto")
            subir_escalon()
            jugar()
    else:
        escalon_8(pregunta)

def nueva_pregunta():
    with open('preguntas.json', 'rt', encoding="utf-8") as json_preguntas:
        lista_preguntas = json.load(json_preguntas)
        preguntas_a_jugar = []

        for pregunta in lista_preguntas:
            if pregunta["escalon"] == jugador["escalon"]:
                preguntas_a_jugar.append(pregunta)

    a_preguntar = preguntas_a_jugar[randint(0,len(preguntas_a_jugar) - 1)]

    if jugador["escalon"] == 8:
        while a_preguntar in jugador["preguntas_hechas"]:
                a_preguntar = preguntas_a_jugar[randint(0,len(preguntas_a_jugar) - 1)]

        jugador["preguntas_hechas"].append(a_preguntar)

    return a_preguntar

def mostrar_opciones(pregunta):
    num_opcion = 1
    for i in pregunta["opciones"]:
        print(f"{num_opcion} - {i}")
        num_opcion += 1

def fin_juego():
    print("Fin del juego\n")

    print("FELICIDADES!!!\nOBTUVISTE PUNTAJE PERFECTO:", jugador["puntaje"]) if jugador["puntaje"] == 110 else print("Puntaje final:", jugador["puntaje"])

def es_correcta(pregunta):
    print(pregunta["pregunta"])
    mostrar_opciones(pregunta)
    respuesta = int(input())
    while respuesta < 1 or respuesta > 4:
        print("Opción incorrecta!!!")
        print()
        print(pregunta["pregunta"])
        mostrar_opciones(pregunta)
        respuesta = int(input())

    return pregunta["correcta"] == pregunta["opciones"][respuesta - 1]

def escalon_8(pregunta):
    contador = 0

    while contador < 4:
        print()
        print("Escalon N°", jugador["escalon"], " Final Boss!!!!!")
        print(jugador["nombre"], "Puntaje:", jugador["puntaje"])
        print("Contesta 4 preguntas para ganarle!!!")
        print(f"{contador}/4")
        pregunta = nueva_pregunta()
        if es_correcta(pregunta): # hace la pregunta y verifica y se contesta correctamente
            contador += 1
            print("Es correcto")
            sumar_puntaje()
        else:
            print("Es Incorrecto")
            contador = 5

    fin_juego()
    cargar_datos()


from rest_framework import serializers
from testapp.models import Employee

def multiples_of_1000(value):
    if value %100 !=0:
        raise serializers.ValidationError('Salary should be multiple of 1000')
class EmployeeSerializer(serializers.Serializer): #normal serializer
    eno = serializers.IntegerField()
    ename = serializers.CharField(max_length=64)
    esal = serializers.FloatField(validators=[multiples_of_1000,])
    eaddr = serializers.CharField(max_length=64)

    '''field level validation'''
    def validate_esal(self,value):
        if value<5000:
            raise serializers.ValidationError('Employee salary should be minimum 5000')
        return value

    '''object level validation'''
    def validate(self,data):
        ename=data.get('ename')
        esal=data.get('esal')
        if ename.lower()=='rucha':
            if esal<50000:
                raise serializers.ValidationError('Rucha salary should be 50000')
        return data
    '''create a data post()'''
    def create(self,validated_data):
        return Employee.objects.create(**validated_data)

    def update(self,instance,validated_data):
        instance.eno=validated_data.get('eno',instance.eno)
        instance.ename=validated_data.get('ename',instance.ename)
        instance.esal=validated_data.get('esal',instance.esal)
        instance.eaddr=validated_data.get('eaddr',instance.eaddr)
        instance.save()
        return instance


from water_in_jugs_qlearning import WaterInJugsQLearning, Jug
import itertools

def main():

    jugs = [Jug('Jug1', 8, 8), Jug('Jug2', 5, 0), Jug('Jug3', 3, 0)]
    actions = itertools.permutations(range(len(jugs)), 2)
    wj_arena = WaterInJugsQLearning(
        start_state=jugs,
        actions=actions,
        goal_qty=4,
        gamma=0.8,
        max_episodes=1000,
        epsilon_greedy=True)
    solution_steps, scores, eps_list = wj_arena.train()

    print('*** SOLUTION ***')
    for step in solution_steps:
        print(step)

    print('*** SCORES ***')
    for score, e in zip(scores, eps_list):
        print("{score:.2f};{epsilon:.2f}".format(score=score, epsilon=e))

if __name__ == "__main__":
    main()


import numpy as np
import logging

class Tracker():

    def __init__(self):

        self.current_tracks = []  ### List of all tracks
        self.temp_tracks = []  ### List of temporary tracks

        self.temp_track_len = 3  ### How much tracks to collect before merging into main track
        self.thresh_distance = 13  ### Euclidean distance threshold
        self.avg_dist = []

    def convert_to_center(self, result):
        return np.array([int((result[2] + result[0]) / 2), int((result[3] + result[1]) / 2)])

    def update(self, result):

        ### Check if we have any detections
        if len(result[0]) == 0:

            self.current_tracks.append(None)
            self.temp_tracks = []
            self.zero_res = result

        elif len(result[0]) == 1:
            self.single_res = result[0][0]

            logging.debug("SINGLE RES")

            ### If we don't have approved tracks or no good history
            if len(self.current_tracks) == 0 or self.current_tracks[-1] is None:
                logging.debug("NONE TRACK CONTINUED")
                ### if we don't temporary tracks we update temp tracks or we lost the track, else we check its distance to previous temp track
                if len(self.temp_tracks) == 0:
                    self.temp_tracks.append(result[0][0])
                    self.current_tracks.append(None)

                else:
                    curr_center = self.convert_to_center(result[0][0])
                    last_center = self.convert_to_center(self.temp_tracks[-1])
                    self.avg_dist.append(np.linalg.norm(curr_center - last_center))

                    if np.linalg.norm(curr_center - last_center) < self.thresh_distance:
                        logging.debug("DIST CRITERIA SATISFIED")
                        self.temp_tracks.append(result[0][0])

                        if len(self.temp_tracks) < self.temp_track_len:
                            self.current_tracks.append(None)
                        else:
                            logging.debug("CURRENT_TRACK UPDATED-------------------")
                            self.current_tracks.append(None)
                            self.current_tracks[-len(self.temp_tracks):] = self.temp_tracks.copy()
                            self.temp_tracks = []

                    else:
                        self.current_tracks.append(None)
                        self.current_tracks[-(len(self.temp_tracks) + 1):] = [None for _ in
                                                                              range(len(self.temp_tracks) + 1)]
                        self.temp_tracks = []

            else:
                #                 self.temp_tracks = []
                ### Now, we work with tracks that have history greater than 5
                logging.debug("CURRENT TRACK CONTINUED")
                curr_center = self.convert_to_center(result[0][0])
                last_canter = self.convert_to_center(self.current_tracks[-1])

                if np.linalg.norm(curr_center - last_canter) < self.thresh_distance:
                    logging.debug("GOOD TRACK APPENDED")
                    self.avg_dist.append(np.linalg.norm(curr_center - last_canter))
                    self.current_tracks.append(result[0][0])
                else:
                    logging.debug("BAD TRACK APPENDED")
                    self.current_tracks.append(None)

        elif len(result[0]) > 1:
            self.multi_res = result

            logging.debug("MULTI RES")
            #             self.temp_tracks = []

            if len(self.current_tracks) == 0 or self.current_tracks[-1] is None:

                if len(self.temp_tracks) > 0:
                    appended = False

                    for i in range(len(result[0])):

                        curr_res = result[0][i]
                        curr_center = self.convert_to_center(curr_res)
                        last_center = self.convert_to_center(self.temp_tracks[-1])
                        if np.linalg.norm(curr_center - last_center) < self.thresh_distance:

                            self.temp_tracks.append(result[0][0])
                            if len(self.temp_tracks) < self.temp_track_len:
                                self.current_tracks.append(None)
                            else:
                                logging.debug("CURRENT_TRACK UPDATED-------------------")
                                self.current_tracks.append(None)
                                self.current_tracks[-len(self.temp_tracks):] = self.temp_tracks.copy()
                                self.temp_tracks = []

                            appended = True
                            break

                    if not appended:
                        self.current_tracks.append(None)
                        self.current_tracks[-(len(self.temp_tracks) + 1):] = [None for _ in
                                                                              range(len(self.temp_tracks) + 1)]
                        self.temp_tracks = []

                else:
                    self.current_tracks.append(None)

            else:
                #                 self.current_tracks.append(None)
                self.temp_tracks = []
                appended = False
                for i in range(len(result[0])):

                    curr_res = result[0][i]
                    curr_center = self.convert_to_center(curr_res)
                    last_center = self.convert_to_center(self.current_tracks[-1])
                    if np.linalg.norm(curr_center - last_center) < self.thresh_distance:
                        appended = True
                        self.current_tracks.append(curr_res)
                        break

                if not appended:
                    self.current_tracks.append(None)

    def calc_missing_intervals(self, length=2):

        i = 0
        j = 0
        misses = 0
        while i < len(self.current_tracks) and j < len(self.current_tracks):
            if self.current_tracks[i] is not None:
                i += 1
                j = i
            elif self.current_tracks[i] is None and self.current_tracks[j] is None:
                j += 1
            elif self.current_tracks[i] is None and self.current_tracks[j] is not None and 0 < (j - i) <= length and i==0:
                i = j
            elif self.current_tracks[i] is None and self.current_tracks[j] is not None and 0 < (j - i) <= length:
                interp_tracks = self.interpolate_bboxes(self.current_tracks[i - 1:j + 1])
                self.current_tracks[i - 1:j + 1] = interp_tracks
                i = j
                misses += 1
            elif self.current_tracks[i] is None and self.current_tracks[j] is not None and (j - i) > length:
                i = j

        if 0 < (j - i) <= length:
            misses += 1

        return misses

    def interpolate_bboxes(self, tracks):

        first = tracks[0]
        last = tracks[-1]

        first_center = [(first[2] + first[0]) / 2, (first[3] + first[1]) / 2]
        last_center = [(last[2] + last[0]) / 2, (last[3] + last[1]) / 2]

        half_width = ((abs(first[2] - first[0]) + abs(last[2] - last[0])) / 2) / 2
        half_height = ((abs(first[3] - first[1]) + abs(last[3] - last[1])) / 2) / 2

        i = 1

        while i < len(tracks) - 1:
            new_x = first_center[0] + ((i + 1) / len(tracks)) * (last_center[0] - first_center[0])
            new_y = first_center[1] + ((i + 1) / len(tracks)) * (last_center[1] - first_center[1])

            new_bbox = np.array(
                [new_x - half_width, new_y - half_height, new_x + half_width, new_y + half_height, first[-1]])

            tracks[i] = new_bbox

            i += 1

        return tracks


import pandas as pd
import glob
import seaborn
import matplotlib.pyplot as plt

pd.set_option('display.width', 500)
pd.set_option('display.max_columns', 36)
files = glob.glob('C:/*.csv')
df = pd.concat((pd.read_csv(file) for file in files), ignore_index=True)
df = df.apply(lambda x: x.str.replace('"', '')).apply(lambda x: x.str.replace('=', ''))

def to_datetime(columnname):
    for items in list(columnname):
        df[items] = pd.to_datetime(df[items])

to_datetime(['Entered', 'Received', 'Completed'])

# create new column
df['Entered - Received'] = df['Entered'] - df['Received']

df = df.stack().reset_index()  # stack is pivot

new_df = pd.DataFrame(df)
new_df = new_df.xs(['level_1', 0], axis=1, drop_level=True)  # keep the level you want
new_df = new_df.sort_values(0)  # sort value
new_df = new_df.drop_duplicates(subset=0)  # drop duplicate, subset is the column that you want to drop duplicates on
new_df['allocation'] = new_df[0].diff()  # subtract from previous row for the timestamp column
new_df = new_df.rename(columns={'level_1': 'function', 0: 'timestamp'})  # rename column

new_df['minutes'] = pd.to_timedelta(new_df['allocation']).astype('timedelta64[m]')  # convert time to minutes
new_df2 = pd.DataFrame(data=new_df[['function', 'minutes']])
new_df2 = new_df2.groupby('function', as_index=False).sum()  # groupby function
new_df2 = pd.DataFrame(new_df2)

graph = seaborn.barplot(x=new_df2['function'], y=new_df2['minutes'])

for p in graph.patches:
    graph.annotate(str(p.get_height()), (p.get_x() * 1.005, p.get_height() * 1.005))

plt.legend(['Entered', 'Completed'], )
graph.get_legend().legendHandles[0].set_color('blue')
graph.get_legend().legendHandles[1].set_color('yellow')
plt.xlabel('Function')
plt.ylabel('Minutes')
plt.style.use('seaborn')
plt.title('Minutes per function')
plt.show()


import json
import argparse
import urllib2
from urlparse import urljoin

class ServoClient(object):
    def __init__(self, host, port):
        self.host = host
        self.port = port
        self.entryPoint = "http://%s:%s" % (host, port)
        motorsUrl = self.query("/")["motors"]["url"]
        motors = self.query(motorsUrl)
        self.motorurl = motors[0]["url"]

    def query(self, url, verb="GET", json_data = None):
        data = None
        data_content_type = None
        # Encode json data object
        if json_data is not None:
            data = json.dumps(json_data)
            data_content_type = 'text/json'

        headers = {}
        if data_content_type:
            headers['Content-Type'] = data_content_type

        # create the url
        url = urljoin(self.entryPoint, url)

        # prepare the request
        req = urllib2.Request(url, data, headers=headers)
        req.get_method = lambda: verb

        # issue the request and return the response
        resp = urllib2.urlopen(req)
        body = resp.read()

        if body:
            return json.loads(body)

    def isOn(self):
        return self.query(self.motorurl)["status"] == "on"

    def set_state(self, st):
        self.query(self.motorurl, "PUT", {"status":st})

    def pulse(self, dt):
        self.query(self.motorurl, "PUT", {"status":"on", "duration": dt})

    def start(self):
        self.set_state("on")

    def stop(self):
        self.set_state("off")

if __name__=='__main__':
    parser = argparse.ArgumentParser(description='HTTP Server')
    parser.add_argument('host', type=str, help='The host to connect too')
    parser.add_argument('port', type=int, help='Listening port for HTTP Server', default=8080, nargs='?')
    parser.add_argument('state', help='The new state of the servo',  type=str)
    args = parser.parse_args()

    s = ServoClient(args.host, args.port)
    s.set_state(args.state)


import itertools, sys

with open(sys.argv[1]) as f:
    lines = f.readlines()

    g1, g2 = {}, {}
    for line in lines:
        a,b,c,d = [int(x) for x in line.replace(" -> ", ",").split(",")]
        steph = 1 if c>=a else -1
        stepv = 1 if d>=b else -1
        for x,y in itertools.zip_longest(range(a,c+steph, steph), range(b,d+stepv,stepv)):
            if x is None: x = a
            if y is None: y = b
            if a==c or b==d: g1[(x,y)] = 1 if (x,y) in g1 else 0
            g2[(x,y)] = 1 if (x,y) in g2 else 0

print(sum(g1.values()))
print(sum(g2.values()))


import datetime
import os
import pprint

from pymongo import MongoClient
from bson.objectid import ObjectId

client = MongoClient(MONGODB_URI)

db = client.bank

accounts_collection = db.accounts

select_by_balance = {"$match": {"balance":{"$lt":1000}}}

select_by_avg = {
    "$group": {
        "_id":"$account_type",
        "avg_balance":{"$avg":"balance"}
    }
}

pipeline  =[select_by_balance,select_by_avg]

result = accounts_collection.aggregate(pipeline)

for item in result:
    pprint.pprint(item)

client.close()


import pytest
import random
import requests
import json
from Tools import Request
from Tools import RandomParam

def result_judge(data):
    try:
        message = json.loads(data.text)
        assert message['code'] == 0 or  message['code'] == 200, json.loads(data.text)
    except:
        assert False, data.text

@pytest.mark.repeat(5)
def test_selectJiHuiChangeLog():
    url = 'http://localhost:8089/crm-manager-master/sys/jiHuiZongKu/selectJiHuiChangeLog'
    data = RandomParam.get_param(['userGroupId|int|1|'])
    result = Request.get_request(url, data)
    result_judge(result)

@pytest.mark.repeat(5)
def test_selectGouTongJiLuInDifferentPeriod():
    url = 'http://localhost:8089/crm-manager-master/sys/jiHuiZongKu/selectGouTongJiLuInDifferentPeriod'
    data = RandomParam.get_param(['tableId|int|0|','userGroupId|int|1|','page|int|1|1','step|int|1|10'])
    result = Request.post_request(url, data)
    result_judge(result)


# from system
import logging
from beaker.session import Session, SessionObject
from beaker.middleware import SessionMiddleware
from pylons import tmpl_context as c

# Put after imports so log is not overwridden by an imported module.
log = logging.getLogger(__name__)

class KwmoSession(Session):
    def __init__(self, request, environ=None, id=None, invalidate_corrupt=False,
             use_cookies=True, type=None, data_dir=None,
             key='beaker.session.id', timeout=None, cookie_expires=True,
             cookie_domain=None, secret=None, secure=False,
             namespace_class=None, **namespace_args):

        self.environ = environ

        # Call parent constructor.
        Session.__init__(self, request, id, invalidate_corrupt,
             use_cookies, type, data_dir,
             key, timeout, cookie_expires,
             cookie_domain, secret, secure,
             namespace_class, **namespace_args)

    def _create_id(self):
        # Call parent _create_id method.
        Session._create_id(self)
        if self.environ and self.environ.has_key('wsgiorg.routing_args') and (len(self.environ['wsgiorg.routing_args']) > 1):
            if 'workspace_id' in self.environ['wsgiorg.routing_args'][1]:
                if self.environ['wsgiorg.routing_args'][1]['controller'] in ['admin_teambox', 'admin_file_download']:
                    self.id = 'wsadminid' + self.environ['wsgiorg.routing_args'][1]['workspace_id'] + '__' + self.id
                    log.debug('kwmo admin session id created ' + self.id)
                else:
                    self.id = 'wsid' + self.environ['wsgiorg.routing_args'][1]['workspace_id'] + '__' + self.id
                    log.debug('kwmo session id created ' + self.id)
            else:
                log.info('_failed to create kwmo session id, creating session with cookie id ' + self.id)
        else:
            log.warn('__failed to create kwmo session id, creating session with cookie id ' + self.id)

    def load(self):
        if not self.is_new:
            if self.environ and self.environ.has_key('wsgiorg.routing_args') and (len(self.environ['wsgiorg.routing_args']) > 1):
                if 'workspace_id' in self.environ['wsgiorg.routing_args'][1]:
                    if self.environ['wsgiorg.routing_args'][1]['controller'] in ['admin_teambox', 'admin_file_download']:
                        self.id = 'wsadminid' + self.environ['wsgiorg.routing_args'][1]['workspace_id'] + '__' + self.id
                        log.debug('kwmo admin session loaded ' + self.id)
                    else:
                        self.id = 'wsid' + self.environ['wsgiorg.routing_args'][1]['workspace_id'] + '__' + self.id
                        #log.debug('kwmo session loaded ' + self.id)

                else:
                    log.info('_failed to load kwmo session, loading session with cookie id ' + self.id)
            else:
                log.warn('__failed to load kwmo session, loading session with cookie id ' + self.id)

        #call parent load
        Session.load(self)

    def save(self):
        # Save permissions in session if needed.
        if c.perms.dirty:
            # Save permissions.
            log.debug("Permissions have changed: saving them into session.")
            self['perms'] = c.perms.to_dict()

        # Call parent save method.
        log.debug("Saving session.")
        Session.save(self)

class KwmoSessionObject(SessionObject):
    def _session(self):
        """Lazy initial creation of session object"""
        if self.__dict__['_sess'] is None:
            params = self.__dict__['_params']
            environ = self.__dict__['_environ']
            self.__dict__['_headers'] = req = {'cookie_out':None}
            req['cookie'] = environ.get('HTTP_COOKIE')
            if params.get('type') == 'cookie':
                self.__dict__['_sess'] = CookieSession(req, **params)
            else:
                self.__dict__['_sess'] = KwmoSession(req, environ, use_cookies=True,
                                                 **params)
        return self.__dict__['_sess']

class KwmoSessionMiddleware(SessionMiddleware):
    def __call__(self, environ, start_response):
        session = KwmoSessionObject(environ, **self.options)
        if environ.get('paste.registry'):
            if environ['paste.registry'].reglist:
                environ['paste.registry'].register(self.session, session)
        environ[self.environ_key] = session
        environ['beaker.get_session'] = self._get_session

        def session_start_response(status, headers, exc_info = None):
            if session.dirty() or (session.accessed() and self.options.get('auto')):
                session.persist()
                if session.__dict__['_headers']['set_cookie']:
                    cookie = session.__dict__['_headers']['cookie_out']
                    if cookie:
                        headers.append(('Set-cookie', cookie))
            return start_response(status, headers, exc_info)
        return self.wrap_app(environ, session_start_response)


def debugExceptions():
    '''
    From http://code.activestate.com/recipes/65287/
    After this is run, attach a hook to exception events to launch debugger in post-mortem mode upon error
    '''
    import sys
    def info(type, value, tb):
       if hasattr(sys, 'ps1') or not sys.stderr.isatty():
          # we are in interactive mode or we don't have a tty-like
          # device, so we call the default hook
          sys.__excepthook__(type, value, tb)
       else:
          import traceback, pdb
          # we are NOT in interactive mode, print the exception...
          traceback.print_exception(type, value, tb)
          print
          # ...then start the debugger in post-mortem mode.
          pdb.pm()
    sys.excepthook = info


import csv

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

from model.datasets import SampleDataset

def collect_data(data: dict, csv_path, is_tp):
    # recording_id,species_id,songtype_id,t_min,f_min,t_max,f_max
    reader = csv.DictReader(open(csv_path, "r"))
    fieldnames = reader.fieldnames

    for row in reader:
        species_id = row['species_id''']
        songtype_id = row['songtype_id']
        t_min = float(row['t_min'])
        t_max = float(row['t_max'])
        f_min = float(row['f_min'])
        f_max = float(row['f_max'])
        duration = t_max - t_min

        key = f'{species_id}|{songtype_id}|{is_tp}'
        if key not in data:
            data[key] = {'count': 0, 'duration': [], 'f_min': [], 'f_max': []}

        data[key]['count'] = data[key]['count'] + 1
        data[key]['duration'].append(duration)
        data[key]['f_min'].append(f_min)
        data[key]['f_max'].append(f_max)

def collect_mean_values(data):
    def _get_stat(data_item):
        return np.min(data_item), np.max(data_item), np.mean(data_item)

    for key, value in data.items():
        value['duration'] = _get_stat(value['duration'])
        value['f_min'] = _get_stat(value['f_min'])
        value['f_max'] = _get_stat(value['f_max'])

def analyze_data(path, is_tp):
    data = pd.read_csv(path)
    data[SampleDataset.k_key] = data['species_id'].astype(str) + '|' + data['songtype_id'].astype(str)
    data[SampleDataset.k_duration] = data['t_max'] - data['t_min']
    # data = data.sort_values([SampleDataset.k_species_id, SampleDataset.k_songtype_id], ascending=[True, True])

    gr_min = data.groupby(SampleDataset.k_key).min(numeric_only=True)
    gr_mean = data.groupby(SampleDataset.k_key).mean(numeric_only=True)
    gr_max = data.groupby(SampleDataset.k_key).max(numeric_only=True)

    gr_min = gr_min.sort_values([SampleDataset.k_species_id, SampleDataset.k_songtype_id])  # type:pd.DataFrame
    gr_mean = gr_mean.sort_values([SampleDataset.k_species_id, SampleDataset.k_songtype_id])  # type:pd.DataFrame
    gr_max = gr_max.sort_values([SampleDataset.k_species_id, SampleDataset.k_songtype_id])  # type:pd.DataFrame

    gr_orig = gr_min[[SampleDataset.k_species_id, SampleDataset.k_songtype_id]]
    gr_orig['tp'] = is_tp

    columns = [SampleDataset.k_f_min, SampleDataset.k_f_max, SampleDataset.k_duration]
    gr_min = gr_min[columns].add_suffix('_min')
    gr_mean = gr_mean[columns].add_suffix('_mean')
    gr_max = gr_max[columns].add_suffix('_max')

    new_data = pd.concat([gr_orig, gr_min, gr_mean, gr_max], axis=1)
    columns = [
        SampleDataset.k_species_id, SampleDataset.k_songtype_id, 'tp',
        SampleDataset.k_f_min + '_min', SampleDataset.k_f_min + '_mean', SampleDataset.k_f_min + '_max',
        SampleDataset.k_f_max + '_min', SampleDataset.k_f_max + '_mean', SampleDataset.k_f_max + '_max',
        SampleDataset.k_duration + '_min', SampleDataset.k_duration + '_mean', SampleDataset.k_duration + '_max',
    ]
    new_data = new_data[columns]

    return new_data

def main():
    statistics_data = {}
    statistics_data_tp = {}
    statistics_data_fp = {}

    # tips = sns.load_dataset("tips")
    # sns.displot(tips, x="size")
    # plt.show()

    # collect_data(statistics_data, r'd:\Projects\Kaggle\rfcx-species-audio-detection_data\train_tp.csv', 1)
    # collect_data(statistics_data, r'd:\Projects\Kaggle\rfcx-species-audio-detection_data\train_tp.csv', 1)
    # collect_data(statistics_data_tp, r'd:\Projects\Kaggle\rfcx-species-audio-detection_data\train_tp.csv', 1)
    # collect_data(statistics_data_fp, r'd:\Projects\Kaggle\rfcx-species-audio-detection_data\train_fp.csv', 0)

    data_tp = analyze_data(r'../data/train_tp.csv', 1)
    data_fp = analyze_data(r'../data/train_fp.csv', 0)
    data_all = pd.concat([data_tp, data_fp])

    data_all.to_csv(r'../data/stat.csv', index=False)
    exit(0)

    # recording_id,species_id,songtype_id,t_min,f_min,t_max,f_max
    csv_tp = pd.read_csv(r'../data/train_tp.csv')
    csv_tp['key'] = csv_tp['species_id'].astype(str) + '|' + csv_tp['songtype_id'].astype(str)
    csv_tp['duration'] = csv_tp['t_max'] - csv_tp['t_min']
    csv_tp.sort_values(['f_min', 'f_max'], ascending=[True, True])

    csv_fp = pd.read_csv(r'../data/train_fp.csv')
    csv_fp['key'] = csv_fp['species_id'].astype(str) + '|' + csv_fp['songtype_id'].astype(str)
    csv_fp['duration'] = csv_fp['t_max'] - csv_fp['t_min']
    csv_fp.sort_values(['f_min', 'f_max'], ascending=[True, True])

    species_id_tp = sorted(set(csv_tp['species_id']))
    species_id_fp = sorted(set(csv_fp['species_id']))
    keys_tp = sorted(set(csv_tp['key']))
    keys_fp = sorted(set(csv_fp['key']))

    min_tp = csv_tp['f_min'].min()
    mean_tp = csv_tp['f_min'].mean()
    max_tp = csv_tp['f_max'].max()
    min_fp = csv_fp['f_min'].min()
    mean_fp = csv_fp['f_min'].mean()
    max_fp = csv_fp['f_max'].max()

    d_min_tp = csv_tp['duration'].min()
    d_mean_tp = csv_tp['duration'].mean()
    d_max_tp = csv_tp['duration'].max()
    d_min_fp = csv_fp['duration'].min()
    d_mean_fp = csv_fp['duration'].mean()
    d_max_fp = csv_fp['duration'].max()

    df_1 = csv_tp[['key', 'f_min']]
    df_1['f_type'] = 'min'
    df_1.rename(columns={'f_min': 'f'}, inplace=True)
    df_2 = csv_tp[['key', 'f_max']]
    df_2['f_type'] = 'max'
    df_2.rename(columns={'f_max': 'f'}, inplace=True)

    df = pd.concat([df_1, df_2])
    df = df.sort_values(['f'], ascending=[False])
    # grouped = csv_tp.groupby("key")
    # users_sorted_average = pd.DataFrame({col: vals['key'] for col, vals in grouped}).mean().sort_values(
    #     ascending=True)

    # sns.displot(csv_tp, x='key', y='duration', height=5, aspect=3)
    sns.displot(csv_tp, x='key', y='duration', height=5, aspect=3)
    plt.show()

    # sns.catplot(x="key", y="f_min",data=csv_tp)
    sns.catplot(x="key", y="f", hue='f_type', data=df)
    plt.show()
    exit(1)

    collect_mean_values(statistics_data_tp)

    keys = sorted(statistics_data_fp.keys())
    counts = [statistics_data_fp[k]['count'] for k in keys]

    sns.displot(x=keys, y=counts, height=5, aspect=3)
    plt.rcParams["figure.figsize"] = (10, 5)
    plt.show()

if __name__ == '__main__':
    main()