"""Attempt to calculate percent chance of getting into a fatal car crash given a latitude and longitude input for Allegheny County."""

import warnings
warnings.filterwarnings("ignore", message="X does not have valid feature names, but StandardScaler was fitted with feature names")

import tensorflow as tf
import pandas as pd
import re
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

# Load data from 2017-2021
filenames = ['CRASH_ALLEGHENY_2017.csv', 'CRASH_ALLEGHENY_2018.csv', 'CRASH_ALLEGHENY_2019.csv', 'CRASH_ALLEGHENY_2020.csv', 'CRASH_ALLEGHENY_2021.csv']
dfs = []
for filename in filenames:
    df = pd.read_csv(filename)
    dfs.append(df)
df = pd.concat(dfs, ignore_index=True)

def dms_to_degrees(dms_str):
    parts = re.split(':|\s+', dms_str)
    if len(parts) != 3:
        return None
    try:
        degrees = float(parts[0])
        minutes = float(parts[1])
        seconds = float(parts[2])
    except ValueError:
        return None
    total_degrees = degrees + minutes/60 + seconds/3600
    return total_degrees

df['LATITUDE'] = df['LATITUDE'].astype(str).apply(dms_to_degrees)
df['LONGITUDE'] = -1 * df['LONGITUDE'].astype(str).apply(dms_to_degrees)
df = df.dropna(subset=['LATITUDE', 'LONGITUDE'])

# Split data into training and testing sets
X = df[['LATITUDE', 'LONGITUDE']]
y = df['FATAL_COUNT']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Standardize the features
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

# Define the model
model = tf.keras.Sequential([
    tf.keras.layers.Dense(32, activation='relu', input_shape=(2,)),
    tf.keras.layers.Dense(16, activation='relu'),
    tf.keras.layers.Dense(1, activation='sigmoid')
])

# Compile the model
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

# Train the model
model.fit(X_train, y_train, epochs=25, batch_size=32, validation_data=(X_test, y_test))

# Test the model with a single set of latitude and longitude
test_data = [[40.434043, -79.992462]] # adjust the values here to test different locations
scaled_test_data = scaler.transform(test_data)
prediction = model.predict(scaled_test_data)[0][0] * 100
print(f"The percentage chance of getting into a fatal crash at {test_data[0][0]}, {test_data[0][1]} is {prediction:.2f}%")

# # Test the model to find the highest % fatality by latitude and longitude, scaled to 1 decimal point
# max_prediction = 0
# max_lat = None
# max_lon = None
# for lat in range(40000000, 42000000, 100000): # To increase decimal and exponentially increase time the for loop runs, remove zeros on the step size (100000)
#     for lon in range(-81000000, -79000000, 100000): # To increase decimal and exponentially increase time the for loop runs, remove zeros on the step size (100000)
#         test_data = [[lat / 1000000, lon / 1000000]]
#         scaled_test_data = scaler.transform(test_data)
#         prediction = model.predict(scaled_test_data)[0][0] * 100
#         if prediction > max_prediction:
#             max_prediction = prediction
#             max_lat = lat / 1000000
#             max_lon = lon / 1000000
# print(f"The highest prediction of {max_prediction:.2f}% is for latitude {max_lat:.6f} and longitude {max_lon:.6f}.")


from django.db.models.signals import post_delete, post_save
from django.dispatch import receiver

from .models import Post

@receiver(post_save, sender=Post)
def category_games_amount_post_save(sender, instance, created, *args, **kwargs):
    """
    После сохранения экземпляра :model:`blog.Post`,
    связанную с :model:`blog.Category`.

    Значение поля :model:`blog.Category` увеличивается на единицу.
    """
    if created:
        instance.category.post_amount += 1
        instance.category.save()

@receiver(post_delete, sender=Post)
def category_games_amount_post_delete(sender, instance, *args, **kwargs):
    """
    После удаления экземпляра :model:`blog.Post`
    связанную с :model:`blog.Category`.

    Значение поля :model:`blog.Category` уменьшается на единицу.
    """
    instance.category.post_amount -= 1
    instance.category.save()


###############################################################################
# Project: ApRES
# Purpose: Classes to encapsulate an ApRES file
# Author:  Paul M. Breen
# Date:    2018-09-24
###############################################################################

__version__ = '0.1.1'

import re
import os
import sys
import warnings

import numpy as np
from netCDF4 import Dataset

class ApRESBurst(object):
    """
    Class for reading and writing ApRES bursts
    """

    DEFAULTS = {
        'autodetect_file_format_version': True,
        'forgive': True,
        'header_markers': {
            'start': '\r\n*** Burst Header ***',
            'end': '\r\n*** End Header ***'
        },
        'end_of_header_re': '\*\*\* End Header',
        'header_line_delim': '=',
        'header_line_eol': '\r\n',
        'data_type_key': 'Average',
        'data_types': ['<u2','<f4','<u4'],
        'data_dim_keys': ['NSubBursts', 'N_ADC_SAMPLES'],
        'data_dim_order': 'C'
    }

    ALTERNATIVES = [{
            'header_line_delim': '=',
            'data_dim_keys': ['NSubBursts', 'N_ADC_SAMPLES']
        }, {
            'header_line_delim': ':',
            'data_dim_keys': ['SubBursts in burst', 'Samples']
        }
    ]

    def __init__(self, fp=None):
        """
        Constructor

        * fp is not None: Then header_start will be set to the file's current
        position.  This allows reading through a file one burst block at a
        time.  This is the normal behaviour.
        * fp is None: Allows the burst object to be populated by other means,
        rather than reading from a file

        :param fp: The open file object of the input file
        :type fp: file object
        """

        self.header_start = 0
        self.data_start = -1
        self.header_lines = []
        self.header = {}
        self.data_shape = ()
        self.data_type = '<u2'
        self.data = None

        self.fp = fp

        if self.fp:
            self.header_start = self.fp.tell()

    def reset_init_defaults(self):
        """
        Reset the initial DEFAULTS parsing tokens from the ALTERNATIVES list
        """

        for key in self.ALTERNATIVES[0]:
            self.DEFAULTS[key] = self.ALTERNATIVES[0][key]

    def read_header_lines(self):
        """
        Read the raw header lines from the file

        :returns: The raw header lines
        :rtype: list
        """

        self.data_start = -1
        self.header_lines = []

        self.fp.seek(self.header_start, 0)
        line = self.fp.readline()
        self.header_lines.append(line.rstrip())

        while(line):
            # The data section follows the end of header marker
            if re.match(self.DEFAULTS['end_of_header_re'], line):
                self.data_start = self.fp.tell()
                break

            line = self.fp.readline()
            self.header_lines.append(line.rstrip())

        return self.header_lines

    def determine_file_format_version(self):
        """
        Determine the file format version from the read header lines

        The header lines are inspected to determine the file format version,
        and the DEFAULTS parsing tokens are setup accordingly
        """

        # Create a list of the first data dimension key from the alternative
        # file formats
        dim_keys = [item['data_dim_keys'][0] for item in self.ALTERNATIVES]

        # Search the header lines for one of the keys, to determine version
        for line in self.header_lines:
            for i, key in enumerate(dim_keys):
                if re.match(key, line):
                    for key in self.ALTERNATIVES[i]:
                        self.DEFAULTS[key] = self.ALTERNATIVES[i][key]
                    return

    def store_header(self):
        """
        Store the read header lines as parsed key/value pairs

        :returns: The parsed header
        :rtype: dict
        """

        self.header = {}

        for line in self.header_lines:
            item = line.split(self.DEFAULTS['header_line_delim'], maxsplit=1)

            if len(item) > 1 and item[0]:
                self.header[item[0].strip()] = item[1].strip()

        return self.header

    def define_data_shape(self):
        """
        Parse the data dimensions from the header to define the data shape

        If the Average header item is non-zero, then that indicates that the
        subbursts have been aggregated to a single value, so we rewrite that
        dimension as 1

        :returns: The data shape
        :rtype: tuple
        """

        self.data_shape = ()
        data_shape = []

        for key in self.DEFAULTS['data_dim_keys']:
            data_shape.append(int(self.header[key]))

        if int(self.header[self.DEFAULTS['data_type_key']]) > 0:
            data_shape[0] = 1

        self.data_shape = tuple(data_shape)

        return self.data_shape

    def define_data_type(self):
        """
        The Average item from the header is used to define the data type

        Average = 0: Record contains all subbursts as 16-bit ints
        Average = 1: Record contains the averaged subbursts as a 16-bit int
        Average = 2: Record contains the stacked subbursts as a 32-bit int

        :returns: The data type
        :rtype: str
        """

        try:
            self.data_type = self.DEFAULTS['data_types'][int(self.header[self.DEFAULTS['data_type_key']])]
        except IndexError as e:
            raise IndexError('Unsupported Averaging/Stacking configuration option found in header: {}'.format(int(self.header[self.DEFAULTS['data_type_key']])))

        return self.data_type

    def configure_from_header(self):
        """
        Configure the object from the raw header lines

        :returns: The parsed header
        :rtype: dict
        """

        if self.DEFAULTS['autodetect_file_format_version']:
            self.determine_file_format_version()

        self.store_header()
        self.define_data_shape()
        self.define_data_type()

        return self.header

    def read_header(self):
        """
        Read the header from the file

        * The raw header lines are available in the header_lines list
        * The parsed header is available in the header dict
        * The file offset to the start of the data is available in data_start
        * The data shape is available in the data_shape tuple
        * The data type is available in data_type

        :returns: The parsed header
        :rtype: dict
        """

        # The header lines are used to configure this object
        self.read_header_lines()
        self.configure_from_header()

        return self.header

    def reshape_data(self):
        """
        Reshape the data according to the data_shape tuple

        If the data read from the file don't conform to the expected
        data_shape as determined from the header, then when reshaping the data,
        an exception will occur:

        * If the reshaping fails because the data array is shorter than
        expected, then we will reraise the exception.
        * If forgive mode is True, then we will emit a warning and then
        truncate the data to conform to the data_shape, otherwise we will
        reraise the exception.

        :returns: The data
        :rtype: numpy.array
        """

        try:
            self.data = np.reshape(self.data, self.data_shape, order=self.DEFAULTS['data_dim_order'])
        except ValueError as e:
            expected_len = self.data_shape[0] * self.data_shape[1]

            if self.data.size < expected_len:
                warnings.warn("Data array read from file doesn't match data_shape as read from the file header: {}. It is shorter than expected. Cannot continue.")
                raise
            elif self.data.size > expected_len:
                warnings.warn("Data array read from file doesn't match data_shape as read from the file header. It is longer than expected.")

                if self.DEFAULTS['forgive']:
                    warnings.warn("{}. Will truncate data to fit to data_shape.".format(e))
                    self.data = np.reshape(self.data[:expected_len], self.data_shape, order=self.DEFAULTS['data_dim_order'])
                else:
                    raise

        return self.data

    def read_data(self):
        """
        Read the data from the file

        The data are available in the data array, shaped according to the
        data_shape tuple

        :returns: The data
        :rtype: numpy.array
        """

        if self.data_start == -1:
            self.read_header()

        count = self.data_shape[0] * self.data_shape[1]
        self.fp.seek(self.data_start, 0)
        self.data = np.fromfile(self.fp, dtype=np.dtype(self.data_type), count=count)
        self.reshape_data()

        return self.data

    def format_header_line(self, key, value):
        """
        Format a raw header line from the given key and value

        :param key: The header item key
        :type key: str
        :param value: The header item value
        :type value: str
        :returns: The formatted raw header line
        :rtype: str
        """

        return '{}{}{}'.format(key, self.DEFAULTS['header_line_delim'], value)

    def reconstruct_header_lines(self):
        """
        Reconstruct the raw header lines from the parsed header

        :returns: The reconstructed raw header lines
        :rtype: list
        """

        self.header_lines = [self.format_header_line(k,v) for k,v in self.header.items()]
        self.header_lines.insert(0, self.DEFAULTS['header_markers']['start'])
        self.header_lines.append(self.DEFAULTS['header_markers']['end'])

        return self.header_lines

    def write_header(self, fp, subbursts=None, samples=None):
        """
        Write the header to the given file object

        :param fp: The open file object of the output file
        :type fp: file object
        :param subbursts: A range specifying the subbursts to be written
        :type subbursts: range object
        :param samples: A range specifying the samples to be written
        :type samples: range object
        """

        if self.data_start == -1:
            self.read_header()

        eol = self.DEFAULTS['header_line_eol']

        for line in self.header_lines:
            # Ensure requested subbursts & samples are reflected in the header
            if subbursts and re.match(self.DEFAULTS['data_dim_keys'][0], line):
                line = self.format_header_line(self.DEFAULTS['data_dim_keys'][0], len(subbursts))
            if samples and re.match(self.DEFAULTS['data_dim_keys'][1], line):
                line = self.format_header_line(self.DEFAULTS['data_dim_keys'][1], len(samples))

            fp.write(line + eol)

    def write_data(self, fp, subbursts=None, samples=None):
        """
        Write the data to the given file object

        :param fp: The open file object of the output file
        :type fp: file object
        :param subbursts: A range specifying the subbursts to be written
        :type subbursts: range object
        :param samples: A range specifying the samples to be written
        :type samples: range object
        """

        if self.data_start == -1:
            self.read_data()

        if not subbursts:
            subbursts = range(self.data_shape[0])
        if not samples:
            samples = range(self.data_shape[1])

        fp.write(np.asarray(self.data[subbursts.start:subbursts.stop:subbursts.step, samples.start:samples.stop:samples.step], order=self.DEFAULTS['data_dim_order']))

class ApRESFile(object):
    """
    Context manager for reading and writing ApRES files
    """

    DEFAULTS = {
        'file_encoding': 'latin-1',
        'apres_suffix': '.dat',
        'netcdf_suffix': '.nc',
        'netcdf_add_history': True,
        'netcdf_group_name': 'burst{:d}',
        'netcdf_var_name': 'data',
        'netcdf_attrs': {
            'units': '1',
            'long_name': 'Sub Burst ADC Samples'
        }
    }

    def __init__(self, path=None):
        """
        Constructor

        :param path: Path to the file
        :type path: str
        """

        self.path = path
        self.fp = None
        self.file_size = -1
        self.bursts = []

    def __enter__(self):
        """
        Enter the runtime context for this object

        The file is opened

        :returns: This object
        :rtype: ApRESFile
        """

        return self.open(self.path)

    def __exit__(self, exc_type, exc_value, exc_traceback):
        """
        Exit the runtime context for this object

        The file is closed

        :returns: False
        :rtype: bool
        """

        self.close()

        return False         # This ensures any exception is re-raised

    def open(self, path=None, mode='r'):
        """
        Open the given file

        :param path: Path to the file
        :type path: str
        :param mode: Mode in which to open the file
        :type mode: str
        :returns: This object
        :rtype: ApRESFile
        """

        if path:
            self.path = path

        self.fp = open(self.path, mode, encoding=self.DEFAULTS['file_encoding'])
        self.file_size = os.fstat(self.fp.fileno()).st_size

        return self

    def close(self):
        """
        Close the file

        :returns: This object
        :rtype: ApRESFile
        """

        self.fp.close()
        self.fp = None

        return self

    def eof(self):
        """
        Report whether the end-of-file has been reached

        :returns: True if EOF has been reached, False otherwise
        :rtype: bool
        """

        return self.fp.tell() >= self.file_size

    def read_burst(self):
        """
        Read a burst block from the file

        :returns: The burst
        :rtype: ApRESBurst
        """

        burst = ApRESBurst(fp=self.fp)
        burst.read_data()

        return burst

    def read(self):
        """
        Reads all burst blocks from the file

        The bursts are available in the bursts list

        :returns: The bursts
        :rtype: list
        """

        while not self.eof():
            burst = self.read_burst()
            self.bursts.append(burst)

        return self.bursts

    def to_apres_dat(self, path, bursts=None, subbursts=None, samples=None):
        """
        Write the bursts to the given file path as an ApRES .dat file

        When rewriting the ApRES data to a new file, the bursts, subbursts,
        and the ADC samples for those selected subbursts, can be subsetted.
        The bursts, subbursts, and samples keyword arguments must specify a
        range object

        :param path: The path of the output file
        :type path: str
        :param bursts: A range specifying the bursts to be written
        :type bursts: range object
        :param subbursts: A range specifying the subbursts to be written
        :type subbursts: range object
        :param samples: A range specifying the samples to be written
        :type samples: range object
        """

        if not self.bursts:
            self.read()

        if not bursts:
            bursts = range(len(self.bursts))

        # We append each burst, so ensure file is empty if it already exists
        with open(path, 'w') as fout:
            pass

        # The ApRES .dat file format is a mixed mode file.  The header is
        # text, and the data section is binary
        for burst in self.bursts[bursts.start:bursts.stop:bursts.step]:
            with open(path, 'a') as fout:
                burst.write_header(fout, subbursts=subbursts, samples=samples)

            with open(path, 'ab') as fout:
                burst.write_data(fout, subbursts=subbursts, samples=samples)

    def burst_to_nc_object(self, burst, nco):
        """
        Map the given burst to the given netCDF object

        The netCDF object can either be a Dataset or a Group

        :param burst: The burst
        :type burst: ApRESBurst
        :param nco: A netCDF object (one of Dataset or Group)
        :type nco: netCDF4._netCDF4.Dataset or netCDF4._netCDF4.Group
        :returns: The netCDF object
        :rtype: netCDF4._netCDF4.Dataset or netCDF4._netCDF4.Group
        """

        # Write the burst header items as netCDF object attributes
        for key in burst.header:
            nco.setncattr(key, burst.header[key])

        for j, key in enumerate(burst.DEFAULTS['data_dim_keys']):
            nco.createDimension(key, burst.data_shape[j])

        data = nco.createVariable(self.DEFAULTS['netcdf_var_name'], burst.data_type, tuple(burst.DEFAULTS['data_dim_keys']))
        data.setncatts(self.DEFAULTS['netcdf_attrs'])
        data[:] = burst.data

        return nco

    def nc_object_to_burst(self, nco):
        """
        Map the given netCDF object to an ApRESBurst object

        The netCDF object can either be a Dataset or a Group

        :param nco: A netCDF object (one of Dataset or Group)
        :type nco: netCDF4._netCDF4.Dataset or netCDF4._netCDF4.Group
        :returns: The burst object
        :rtype: ApRESBurst
        """

        # We make a copy, otherwise data is invalid after file is closed
        data = nco.variables[self.DEFAULTS['netcdf_var_name']][:]
        attrs = vars(nco).copy()

        burst = ApRESBurst()
        burst.data_start = 0              # Stop data being read from file

        # Remove any attributes that weren't part of the original burst's header
        try:
            del attrs['history']
        except KeyError:
            pass

        # Reconstruct the original burst from the parsed header and data.  We
        # initially set the header_lines to be the parsed header which allows us
        # to determine the burst format version, and thus setup the DEFAULTS
        # parsing tokens accordingly.  This then allows us to correctly
        # reconstruct the raw header lines with the appropriate header line
        # delimiter for the burst format version
        burst.data = data
        burst.header = attrs
        burst.header_lines = burst.header
        burst.determine_file_format_version()
        burst.reconstruct_header_lines()
        burst.configure_from_header()

        return burst

    def to_netcdf(self, path=None, mode='w'):
        """
        Write the bursts to the given file path as a netCDF4 file

        The default netCDF file path is the same as the input file, but with
        a .nc suffix

        :param path: The path of the output file
        :type path: str
        :param mode: Mode in which to open the file
        :type mode: str
        """

        if not self.bursts:
            self.read()

        path = path or os.path.splitext(self.path)[0] + self.DEFAULTS['netcdf_suffix']

        ncfile = Dataset(path, mode)

        # Add the command line invocation to global history attribute
        if self.DEFAULTS['netcdf_add_history']:
            ncfile.history = ' '.join(sys.argv)

        # Write each burst as a netCDF4/HDF5 group
        for i, burst in enumerate(self.bursts):
            group = ncfile.createGroup(self.DEFAULTS['netcdf_group_name'].format(i))
            self.burst_to_nc_object(burst, group)

        ncfile.close()


from flask import Flask
from flask import request
from flask import jsonify
from amap_api import ip2province
from province2bitmap import province2bitmap

MapuinoApp = Flask(__name__)

database = {}

@MapuinoApp.route("/frontendAPI", methods=['GET'])
def putdate():
    if request.method == 'GET':
        print(request)
        if request.headers.getlist("X-Forwarded-For"):
            request_ip = request.headers.getlist("X-Forwarded-For")[0]
        else:
            return "NO_IP_ADDR"
        print(request_ip)
        province = ip2province(request_ip)
        if province == "error":
            return "IP_ADDR_ERROR"

        bitmap = province2bitmap(province[:2])
        print(bitmap)
        segment = bitmap[0]
        bit = bitmap[1]

        if not(request.args.get('UUID') in database):
            database[request.args.get('UUID')] = {
                "SEG0": 0x00000000,
                "SEG1": 0x00000000,
            }

        if segment == 0:
            database[request.args.get('UUID')]["SEG0"] |= (1 << bit)
        else:
            database[request.args.get('UUID')]["SEG1"] |= (1 << bit)

        return "OK"

@MapuinoApp.route("/hardwareAPI", methods=['GET'])
def getdata():
    if request.method == 'GET':
        try:
            resp = database[request.args.get('UUID')]
            resp['CODE'] = 0
            resp_json = jsonify(resp)
            database[request.args.get('UUID')]["SEG0"] = 0x00000000
            database[request.args.get('UUID')]["SEG1"] = 0x00000000
        except Exception as e:
            print(e)
            resp_json = jsonify({'CODE': 1})

        return resp_json

if __name__ == "__main__":
    MapuinoApp.run("0.0.0.0", 7767)


from typing import Union
from neuroseg.models import ResUNET2D, ResUNET3D
from neuroseg.config.optimizer_cfg import OptimizerConfigurator
from neuroseg.config import TrainConfig, PredictConfig

class ModelConfigurator:
    def __init__(self,
                 config: Union[TrainConfig, PredictConfig]) -> None:
        self.config = config
        self.model_name = self.config.model
        self.loss = self.config.loss
        self.track_metrics = self.config.track_metrics
        self.optimizer_cfg = OptimizerConfigurator(self.config)
        self.optimizer = self.optimizer_cfg.optimizer

        self.model_cfg = self._get_model_cfg()
        self.model = self.model_cfg.model
        return

    def _get_model_cfg(self):
        if self.model_name == "resunet2d":
            return ResUNET2D(self.config)
        elif self.model_name == "resunet3d":
            return ResUNET3D(self.config)
        else:
            raise NotImplementedError(self.model_name)


#!/usr/bin/python3
from nis import cat
from ssl import HAS_ECDH, _create_unverified_context
import sys
from pathlib import Path
import matplotlib.pyplot as plt
import numpy as np
import cv2
import torch
import torch.backends.cudnn as cudnn
from absl import app
from absl.flags import FLAGS

import rospy
import message_filters
from sensor_msgs.msg import Image
from rospy.numpy_msg import numpy_msg
from geometry_msgs.msg import Vector3
from std_msgs.msg import Header, Int8

# ADD YOLOv5 LIBS #
FILE = Path(__file__).resolve()
ROOT = FILE.parents[0]  # YOLOv5 root directory
if str(ROOT) not in sys.path:
    sys.path.append(str(ROOT))  # add ROOT to PATH

from utils.augmentations import letterbox
from models.experimental import attempt_load
from utils.general import non_max_suppression, scale_coords
from utils.plots import Annotator, colors

coco_id_mapping = {
    1: 'person'
}  # pyformat: disable

coco_ths_mapping = {
     1: 0.7
}  # pyformat: disable

#################################################################################
# Implementation of a CV_Bridge function for compatibility with ROS Melodic
## this is not a complete implementation
## some code lines dealing with exceptions have been removed for convenience

numpy_type_to_cvtype = {'uint8': '8U', 'int8': '8S', 'uint16': '16U',
                        'int16': '16S', 'int32': '32S', 'float32': '32F', 'float64': '64F'}

def cv2_to_imgmsg(cvim, encoding="passthrough", header=None):
    if not isinstance(cvim, (np.ndarray, np.generic)):
        raise TypeError('Your input type is not a numpy array')

    img_msg = Image()
    img_msg.height = cvim.shape[0]
    img_msg.width = cvim.shape[1]

    if header is not None:
        img_msg.header = header

    if cvim.ndim < 3:
        n_channels = 1
    else:
        n_channels = cvim.shape[2]

    cv_type = '%sC%d' % (numpy_type_to_cvtype[cvim.dtype.name], n_channels)

    if encoding == "passthrough":
        img_msg.encoding = cv_type
    else:
        img_msg.encoding = encoding

    if cvim.dtype.byteorder == '>':
        img_msg.is_bigendian = True

    #img_msg.data = cvim.tostring() # Deprecated since numpy version 1.19.0.
    img_msg.data = cvim.tobytes()
    img_msg.step = len(img_msg.data) // img_msg.height

    return img_msg

#################################################################################

class Detector(object):
    def __init__(self):

        # define class variables and set initial values
        ## variables for runnig yolov5 model
        self.image_size = (int(720), int(1280))
        self.conf_thres = 0.15
        self.iou_thres = 0.45
        self.agnostic_nms= False
        self.max_det = 100
        self.max_dist = 3
        self.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

        ## variables for topic management
        self.topic_name = {}
        self.topic_name["rgb"] = "/zed2/zed_node/left/image_rect_color"
        self.topic_name["depth"] = "/zed2/zed_node/depth/depth_registered"
        self.raw_img = {}
        self.raw_img["rgb"] = np.zeros( self.image_size + (3,) )
        self.raw_img["depth"] = np.zeros( self.image_size )
        self.is_received = {}
        self.is_received["rgb"] = False
        self.is_received["depth"] = False
        self.rcv_time = {}
        self.rcv_time["rgb"] = None
        self.rcv_time["depth"] = None
        self.proc_time = rospy.get_time()

        ## variables for controlling module status
        self.model = None
        self.is_initialized = False
        ## variables for camera to world transformation
        # self.color_intrinsic = np.array([[337.2084410968044, 0.0,               320.5],
        #                                  [0.0,               337.2084410968044, 240.5],
        #                                  [0.0,               0.0,               1.0  ]])
        # self.inverse_intrinsic = np.linalg.inv(self.color_intrinsic)
        # self.cam_to_body = np.array([[ 0,  0,  1],
        #                              [-1,  0,  0],
        #                              [ 0, -1,  0]])

        # SUBSCRIBERS
        self.sub_rgb = message_filters.Subscriber(self.topic_name["rgb"], numpy_msg(Image))
        self.sub_depth = message_filters.Subscriber(self.topic_name["depth"], numpy_msg(Image))
        self.sub_ts = message_filters.TimeSynchronizer([self.sub_rgb, self.sub_depth], queue_size=10)
        self.sub_ts.registerCallback(self.sync_callback)
        # PUBLISHERS
        self.pub_img = rospy.Publisher("/detection/image_rect", Image, queue_size=1)
        self.pub_crowd = rospy.Publisher("/detection/num_people", Int8, queue_size=1)

        # initialize the yolov5 model
        self._init_model()

    @torch.no_grad()
    def _init_model(self):
        # initialize model and load pretrained weight
        print('Initializing model...')
        self.model = attempt_load('yolov5m.pt', map_location=self.device)  # load FP32 model
        self.stride = int(self.model.stride.max())  # model stride
        self.names = self.model.module.names if hasattr(self.model, 'module') else self.model.names  # get class names
        self.model.eval()
        print('model loaded')
        if self.is_initialized == False:
            self.is_initialized = True

    def _preprocess_data(self, input_img):
        ## padding and resizing
        tmp_img = letterbox(input_img, self.image_size, stride=self.stride, auto=True)[0]
        ## switching axes
        tmp_img = tmp_img.transpose((2, 0, 1))[::-1]  # HWC to CHW, BGR to RGB
        tmp_img = np.ascontiguousarray(tmp_img)
        ## noramlization
        tmp_img = tmp_img / 255.0
        ## convert data format from numpy to tensor
        input_tensor = torch.from_numpy(tmp_img).unsqueeze(dim=0).to(self.device).float()
        return input_tensor

    def _make_prediction(self, input_tensor):
        # make prediction for the current input
        pred, _ = self.model(input_tensor)
        pred = non_max_suppression(pred, self.conf_thres, self.iou_thres, \
                None, self.agnostic_nms, max_det=self.max_det)
        ## get detection result of the first sample
        ## since only one sample is given as input
        result = pred[0]

        return result

    def _measure_distance(self, box, depth_img):
        H, W = self.image_size
        # find 3d position of the center point of detected box
        box = [int(val) for val in box]
        x_0, y_0, x_1, y_1 = box

        ## slice the center box
        img_slice = depth_img[y_0:y_1, x_0:x_1]
        ## if over 50% of values in the center box is NaN, skip the current box
        nan_idx = np.isnan(img_slice)
        if np.mean(nan_idx) < 0.5:
            distance = np.mean(img_slice[np.invert(nan_idx)])
        else:
            distance = np.nan

        return distance

    def _process_detections(self, annotator, detections, tensor_size, depth_img):
        # initilaize a variable to store number of people in the image
        num_people = 0
        ## scale detection box cooridnates to fit the input image size
        detections[:, :4] = scale_coords(tensor_size, detections[:, :4], self.image_size).round()
        ## draw detection boxes
        for *xyxy, conf, obj in reversed(detections):
            # get the class index
            c = int(obj)
            c_coco = c+1

            # check if the detected class is in the coco_id_mapping
            if not (c_coco in coco_id_mapping.keys()):
                continue
            # get the name for the detected class
            name = coco_id_mapping[c_coco]
            # get the confidence threshold
            conf_ths = coco_ths_mapping[c_coco]
            # check if the confidence score is over the threshold
            if conf.data.cpu() < conf_ths:
                continue

            # measure distance values in the box
            distance = self._measure_distance(xyxy, depth_img)
            if distance > self.max_dist:
                continue

            # draw a detection box
            color = colors(c, bgr=True)
            annotator.box_label(xyxy, name, color=color)
            num_people += 1

        return num_people

    def sync_callback(self, rgb, depth):
        try:
            # store the rgb message
            self.raw_img["rgb"] = np.frombuffer(rgb.data, dtype=np.uint8).reshape(self.image_size+(3,))
            self.is_received["rgb"] = True
            self.rcv_time["rgb"] = rgb.header.stamp
            # store the depth message
            self.raw_img["depth"] = np.frombuffer(depth.data, dtype=np.float32).reshape(self.image_size)
            self.is_received["depth"] = True
            self.rcv_time["depth"] = depth.header.stamp

        except Exception as e:
            print(e)

    def run_model(self):
        # if no image has been received, skip the loop
        if (self.is_received["rgb"] and self.is_received["depth"]) is False:
            return

        # measure the start time of current loop
        t0 = rospy.get_time()

        # if the last process time has not been outdated than a second, skip the loop
        # if (t0 - self.proc_time) < 1.0:
        #     return

        # copy the recent images
        tmp_img = np.copy(self.raw_img["rgb"][:, :, :3])
        tmp_depth = np.copy(self.raw_img["depth"])
        tmp_stamp = self.rcv_time["rgb"]
        ## store the time stamp into a header
        header = Header()
        header.stamp = tmp_stamp

        # prepare tensor input data
        tmp_img_ = self._preprocess_data(tmp_img)

        # make prediction
        result = self._make_prediction(tmp_img_)

        # process the detection result
        ## define a annotator object for drawing detection boxes
        annotator = Annotator(tmp_img, line_width=1, pil=False)
        num_people = self._process_detections(annotator, result, tmp_img_.shape[2:], tmp_depth)
        ## get the annotation result
        output_img = annotator.result()

        # measure the time when the detection is finished
        t1 = rospy.get_time()

        # print("inference time: %4.3f"%(t1-t0))

        # publish the detection result
        self.pub_img.publish(cv2_to_imgmsg(output_img.astype(np.uint8), "rgb8", header))
        self.pub_crowd.publish(Int8(data=num_people))

        # update the last process time
        self.proc_time = t1

def main(_argv):
    rospy.init_node('simple_crowd_detector')

    detector = Detector()
    rate = rospy.Rate(10)
    while not rospy.is_shutdown():
        if detector.is_initialized is True:
            detector.run_model()
            rate.sleep()

if __name__ == '__main__':
    try:
        app.run(main)
    except SystemExit as e:
        print(e)


import os
import pathlib
from typing import Any, Dict, List
from unittest.mock import MagicMock

import pytest
from utils import filter_travels, parse_refunds

def read_from_file(file_path: str):
    content: str = ""
    with open(file_path, "r") as file:
        content = file.read()
    return content

@pytest.fixture
def text_to_parse():
    parent = str(pathlib.Path(__file__).parent)
    path = os.path.sep.join([parent, "fixtures", "refunds.txt"])
    return read_from_file(path)

def test_parse_refunds(text_to_parse: str):
    travels = parse_refunds(text_to_parse)
    assert len(travels) == 2

Travel = MagicMock
empty_dict: Dict[str, Any] = dict()

@pytest.mark.parametrize(
    "travels, filters, expected_count",
    [
        (
            [Travel(origin="La Habana"), Travel(origin="Matanzas")],
            dict(origin="La Habana"),
            1,
        ),
        (
            [
                Travel(destination="La Habana", seat=11),
                Travel(destination="Matanzas", seat=11),
            ],
            dict(seat=11),
            2,
        ),
        (
            [
                Travel(destination="La Habana", seat=11),
                Travel(destination="Matanzas", seat=11),
            ],
            empty_dict,
            2,
        ),
    ],
)
def test_filter_travels(
        travels: List[Travel],
        filters: Any,
        expected_count: int
        ):
    assert len(filter_travels(travels, **filters)) == expected_count


import platform
from twisted.internet import reactor
from twisted.internet.task import LoopingCall
from MMPProtocol import MMPClient
from BitcoinMiner import *

class MMPBitcoinMiner(BitcoinMiner):
	def __init__(self, device, host, user, password, port=8332, *args, **kwargs):
		self.client = MMPClient(self)
		self.clientargs = (host, port, user, password)

		self.name = kwargs.get('name','')

		self.lastRateUpdate = 0.0
		self.serverMsgBanner = False
		self.workRequested = False
		return BitcoinMiner.__init__(self, device, host, user, password, port, *args, **kwargs)

	def say(self, *args):
		self.serverMsgBanner = False
		BitcoinMiner.say(self, *args)

	def hashrate(self, rate):
		if time() > (self.lastRateUpdate + 30.0):
			self.client.setMeta('rate', rate)
			self.lastRateUpdate = time()
		BitcoinMiner.hashrate(self, rate)

	def mine(self):
		self.stop = False
		Thread(target=self.miningThread).start()

		self.client.setMeta('device', self.device.name.replace('\x00',''))
		self.client.setMeta('version', 'poclbm v%s by m0mchil' % (VERSION,))
		if self.name:
			self.client.setMeta('name', self.name)
		self.client.setMeta('os', platform.system() + ' ' + platform.version())
		self.client.setMeta('cores', self.device.max_compute_units)

		self.client.connect(*self.clientargs)
		LoopingCall(self.checkQueues).start(0.5)
		reactor.run()

	def onConnect(self):
		self.sayLine('connected')
	def onDisconnect(self):
		self.sayLine('lost connection')

	def onMsg(self, message):
		if not self.serverMsgBanner:
			self.sayLine('server message(s):')
		self.say(message + '\n')
		self.serverMsgBanner = True

	def onWork(self, wu):
		self.workRequested = False
		work = {
			'midstate': pack('I'*8, *sha256(STATE, np.array(unpack('I'*64, wu.data[:64] + '\x00'*192), dtype=np.uint32)).tolist()).encode('hex'),
			'data': wu.data.encode('hex') + '00000080' + ('00'*40) + '80020000',
			'target': wu.target.encode('hex'),
			'mask': wu.mask
		}
		with self.lock:
			self.queueWork(work)

	def sendResult(self, result):
		for i in xrange(OUTPUT_SIZE):
			if result['output'][i]:
				h = hash(result['state'], result['data'][0], result['data'][1], result['data'][2], result['output'][i])
				if h[7] != 0:
					self.failure('Verification failed, check hardware!')
				else:
					self.diff1Found(bytereverse(h[6]), result['target'][6])
					if belowOrEquals(h[:7], result['target'][:7]):
						if result['work'] is None:
							return
						d = result['work']['data']
						d = ''.join([d[:136], pack('I', long(result['data'][1])).encode('hex'), d[144:152], pack('I', long(result['output'][i])).encode('hex')])
						deferred = self.client.sendResult(d.decode('hex'))
						def callback(accepted):
							self.blockFound(pack('I', long(h[6])).encode('hex'), accepted)
						deferred.addCallback(callback)

	def checkQueues(self):
		if self.stop:
			reactor.stop()
			return
		with self.lock:
			if self.update and not self.workRequested:
				self.workRequested = True
				self.client.requestWork()
			if not self.resultQueue.empty():
				self.sendResult(self.resultQueue.get(False))


#!/usr/bin/env python
import operator
import os
import random
import re
import sys
from collections import defaultdict
from datetime import datetime
from subprocess import Popen, PIPE
from tarfile import TarFile

from cluster.common import Cluster
from cluster.config import USER, LOG_PATH, USER_LABEL, PBS_ARCHIVE_PATH, HOME
from cluster.tools import confirm_delete, parse_timearg, truncate_str, cache_cmd, print_table

class TimeDeltaError(Exception):
    """Custom error thrown when parsing time delta"""

class TimeDelta:
    """Makes filtering job list by arbitrary constraints simpler"""

    def __init__(self, arg, newer=True):
        self.compare = operator.ge if newer else operator.le

        if re.match(r'^\d{4}-\d{2}-\d{2}$', arg):
            self.field = 'date'
            self.value = datetime.strptime(arg, '%Y-%m-%d')
        elif re.match(r'^\d+[a-cn-u.]*-*\d*[a-cn-u.]*$', arg):
            self.field = 'job_id'
            if '-' in arg:
                self.value_min = int(arg.split('-')[0].split('.')[0])
                self.value_max = int(arg.split('-')[1].split('.')[0])
            else:
                self.value_min = int(arg.split('.')[0])
        elif ',' in arg:
            self.compare = operator.contains
            self.field = 'job_id_list'
            self.value = [int(j) for j in arg.split(',')]
        elif re.match(r'^\d+[hdw]$', arg):
            self.field = 'date'
            self.value = parse_timearg(arg)
        else:
            raise TimeDeltaError("Unable to parse: %s" % arg)

    def filter(self, jobs):
        for job in jobs:
            if self.field == 'date':
                if job.finished:
                    if self.compare(job.finished, self.value):
                        yield job
                elif not job.qstat and job.start_time:
                    if self.compare(job.start_time, self.value):
                        yield job
            if self.field == 'job_id_list':
                if self.compare(self.value, job.job_id):
                    yield job
            elif self.field == 'job_id':
                if self.compare(job.job_id, self.value_min):
                    if hasattr(self, 'value_max'):
                        if not operator.le(job.job_id, self.value_max):
                            continue
                    yield job

def read_qstat():
    """Parses the brief qstat output for all users and makes 3 separate summaries: users, queues, total

    :return: Job summaries for users, queues and total
    :rtype: tuple[dict, dict, dict]
    """
    qstat = cache_cmd('/usr/bin/qstat')

    user_stats = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
    queue_stats = defaultdict(lambda: defaultdict(int))
    total_stats = defaultdict(int)

    for line in qstat.split('\n')[2:]:  # skip first two rows of header
        if not line:
            continue

        job_id, name, user, time, status, queue = line.strip().split()

        user = USER_LABEL if user == USER else user
        user_stats[user][queue][status] += 1
        queue_stats[queue][status] += 1
        total_stats[status] += 1

    return user_stats, queue_stats, total_stats

def print_all_jobs():
    """Print a short summary of running/queued jobs. Identical to the old jobstatus script."""
    user_stats, queue_stats, total_stats = read_qstat()

    print("=========================================================")
    print("%-15s %-10s %-10s %-10s %-10s" % ('User', 'Queue', 'Running', 'Queued', 'Exiting'))
    print("---------------------------------------------------------")

    statuses = ('R', 'Q', 'E')

    for user in sorted(user_stats):
        for queue in sorted(user_stats[user]):
            row = tuple([user, queue] + [user_stats[user][queue].get(s, 0) for s in statuses])
            print("%-15s %-10s %-10s %-10s %-10s" % row)

    print("---------------------------------------------------------")

    for queue in sorted(queue_stats):
        row = tuple(['', queue] + [queue_stats[queue].get(s, 0) for s in statuses])
        print("%-15s %-10s %-10s %-10s %-10s" % row)

    print("                -----------------------------------------")

    row = tuple(['', 'totals'] + [total_stats.get(s, 0) for s in statuses])
    print("%-15s %-10s %-10s %-10s %-10s" % row)

def details(args):
    """Print job details for current user. Output format can be fine-tuned with args argument.

    :param args: Arguments from argparse
    :type args: argparse.Namespace
    """
    # Don't cache commands if we're deleting jobs, we need fresh status
    cluster = Cluster(jobs_qstat=True, jobs_log=True, jobs_pbs=True, cached=not args.delete)
    jobs = cluster.jobs_list()

    filtering = True in (args.print_running, args.print_queued, args.print_completed, args.print_failed)

    # We're about to delete some jobs, make sure to sanitize other arguments to make sense with delete action
    if args.delete:
        # Override to table view when deleting jobs, you want to see what'll get killed
        if args.output != 'table':
            sys.stderr.write('Warning: Output format forced from "%s" to "table" for delete action.\n' % args.output)
            args.output = 'table'

        # We can delete only running and queued, make sure they're both included if either is not explicitly enabled
        if not args.print_queued and not args.print_running:
            args.print_queued = True
            args.print_running = True

        filtering = True

        # Get rid of completed and failed if they were explicitly defined
        if args.print_completed or args.print_failed:
            sys.stderr.write('Warning: Ignore completed and failed jobs for delete action.\n')
            args.print_completed = False
            args.print_failed = False

        # Limiting by number of jobs makes no sense for deleting, get rid of it
        if args.limit_output and args.limit_output.isdigit() and int(args.limit_output) < 10000:
            sys.stderr.write('Warning: Filtering by number of jobs (%s) ignored.\n' % args.limit_output)
            args.limit_output = None

    if filtering:
        if not args.print_running:
            jobs = [job for job in jobs if not job.state.startswith('R')]
        if not args.print_queued:
            jobs = [job for job in jobs if not job.state.startswith('Q')]
        if not args.print_completed:
            jobs = [job for job in jobs if not job.state.startswith('C')]
        if not args.print_failed:
            jobs = [job for job in jobs if not (job.state.startswith('F') or job.state == '?')]

    if args.limit_output:
        if args.limit_output.isdigit():
            if int(args.limit_output) < 10000:
                jobs = jobs[:int(args.limit_output)]
            else:
                limit_check = TimeDelta(args.limit_output)
                jobs = limit_check.filter(jobs)
        else:
            try:  # filter by time
                limit_check = TimeDelta(args.limit_output)
                jobs = limit_check.filter(jobs)
            except TimeDeltaError:  # try filtering by name
                jobs = [job for job in jobs if job.name == args.limit_output]

    if args.output == 'jobid':
        jobids = [str(job.job_id) for job in jobs]
        print(' '.join(jobids))
    elif args.output == 'cmd':
        for job in jobs:
            print(job.cmd)
    else:
        data = []
        for job in jobs:
            data.append(
                [str(job.job_id), truncate_str(job.name, 20), job.state, job.exit_status, job.start, job.runtime,
                 job.memory, job.cmd])

        print_table(
            ['Job ID', 'Name', 'Status', 'Exit', 'Start Time', 'Elapsed/Total Time', 'Used Memory', 'Command'],
            data
        )

    if args.delete:
        jobs = list(jobs)
        if not len(jobs):
            print("\n\nNo jobs to delete.")
            return

        print("\n\nDANGER ZONE!")
        if confirm_delete('Are you sure you want to delete %s jobs listed above?' % len(jobs), str(len(jobs))):
            ids = [str(j.job_id) for j in jobs]
            proc = Popen('qdel %s' % ' '.join(ids), shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True,
                         universal_newlines=True)
            qdel, err = proc.communicate()
            if err:
                raise Exception("Can't run qdel: %s" % err)
            print("Deleted %d jobs." % len(ids))
        else:
            print("Wrong answer, not deleting anything.")

def archive(args):
    """Archive old finished jobs, save them in a gzipped file

    :param args: Arguments from argparse
    :type args: argparse.Namespace
    """
    timefilter = TimeDelta(args.age, newer=False)

    cluster = Cluster(jobs_qstat=True, jobs_log=True, jobs_pbs=True)
    jobs = cluster.jobs_list()

    jobs_to_archive = []

    for job in timefilter.filter(jobs):
        if job.qstat:
            # Do not delete running jobs!
            continue

        if job.pbs_output or job.pbs_log:
            jobs_to_archive.append(job)

    if not jobs_to_archive:
        # Bail, nothing to do here
        return

    # Delete files only when we know they are all safely zipped
    delete_list = []
    archived_job_ids = set()

    if not os.path.exists(PBS_ARCHIVE_PATH):
        os.mkdir(PBS_ARCHIVE_PATH)

    tar_file = '%s_%032x.tar.gz' % (datetime.now().strftime('%Y-%m-%d'), random.getrandbits(128))
    tar_path = os.path.join(PBS_ARCHIVE_PATH, tar_file)

    with TarFile.open(tar_path, 'w:gz') as tar:
        for job in jobs_to_archive:
            if job.pbs_output:
                tar.add(job.pbs_output, arcname=job.pbs_output.replace(HOME, '').lstrip('/'))
                delete_list.append(job.pbs_output)

            if job.pbs_log:
                archived_job_ids.add(job.job_id)

            print('Archived job %s' % job.job_id)

    with open(LOG_PATH + '_bkp', 'w') as log:
        for job in jobs[::-1]:
            if job.pbs_log and job.job_id not in archived_job_ids:
                log.write(job.pbs_log)

    for f in delete_list:
        os.remove(f)

    os.rename(LOG_PATH + '_bkp', LOG_PATH)

def subcommand_header(name):
    header = r"""
/-------------{pad}-\
| Subcommand: {name} |
\-------------{pad}-/""".format(pad='-' * len(name), name=name)
    return header

def main():
    if len(sys.argv) == 1:  # Python 2 argparse hack... don't judge me
        print_all_jobs()
        return

    # noinspection PyCompatibility
    import argparse
    timedelta_help = 'Must be either a date (YYYY-MM-DD), Job ID (numeric part) or a time delta (2w, 3h or 1d). ' \
                     'Time delta unit can be one of: h(hours), d(days) or w(weeks)'

    parser = argparse.ArgumentParser(
        description='Check job status. If no subcommand is specified it prints out a summary of all jobs.',
        add_help=False
    )
    parser.add_argument('-h', '--help', action='store_true', dest='help', default=False)

    command_parsers = parser.add_subparsers(title='Available subcommands',
                                            dest='command',
                                            description='For detailed subcommand help run: <subcommand> -h.')

    details_parser = command_parsers.add_parser('details', help='Show details of my jobs.')
    details_parser.add_argument('-r', '--print-running', action='store_true', help='Print running jobs.')
    details_parser.add_argument('-q', '--print-queued', action='store_true', help='Print queued jobs.')
    details_parser.add_argument('-c', '--print-completed', action='store_true', help='Print completed jobs.')
    details_parser.add_argument('-f', '--print-failed', action='store_true', help='Print failed jobs.')
    details_parser.add_argument('-d', '--delete', action='store_true', help='Delete listed jobs.')
    details_parser.add_argument('-l', '--limit-output', default='50',
                                help='Limit output to either: number of lines, Job ID, time delta or name. '
                                     'The default is 50 lines. '
                                     'Job ID can be in a form of range (i.e. 28327149-28327165) or a comma separated '
                                     'list of ids. Time delta unit can be one of: h(hours), d(days) or w(weeks).')
    details_parser.add_argument('-o', '--output', default='table',
                                help='Choose how to display output: table, jobid or cmd (default: table). '
                                     'TABLE dislays all available information about the job. '
                                     'JOBID displays space-separated job IDs which is useful for deleting jobs. '
                                     'CMD displays the commands which is useful for resubmitting jobs.')
    details_parser.set_defaults(func=details)

    archive_parser = command_parsers.add_parser('archive', help='Archive finished jobs.')
    archive_parser.add_argument('age', default='1w', nargs='?',
                                help='Archive finished jobs older than AGE (default: 1 week). ' + timedelta_help)
    archive_parser.set_defaults(func=archive)

    if len(sys.argv) == 2 and sys.argv[1] in ('-h', '--help'):
        # Another py2 hack, for a global help
        parser.print_help()

        for subp in (details_parser, archive_parser):
            print(subcommand_header(subp.prog))
            subp.print_help()

        parser.exit()

    args = parser.parse_args()

    args.func(args)

if __name__ == '__main__':
    main()


import logging
import sys
from enum import Enum

from nio.util.nio_time import get_nio_time

class NIOAdapter(logging.LoggerAdapter):

    """ The base logger adapter class for NIO.  """

    detail_errors = True

    def __init__(self, logger, extra={}):
        super().__init__(logger, extra)

    def setLevel(self, level):
        """ Set the specified level on the underlying logger.  """
        log_level = level.value if isinstance(level, Enum) else level
        super().setLevel(log_level)

    def exception(self, msg, *args, **kwargs):
        """ Log an exception inside of a handler.

        Delegate an exception call to the underlying logger and log exception
        or error depending of running flag "detail_errors"
        """
        if self.__class__.detail_errors:
            super().exception(msg, *args, **kwargs)
        else:
            _, exc_value, _ = sys.exc_info()
            msg = "{0}, (Exception details: {1})".format(
                msg, self.exc_info(exc_value))
            super().error(msg, *args, **kwargs)

    @classmethod
    def exc_info(cls, ex):
        """ Returns string information for an exception.  """
        try:
            if isinstance(ex, Exception) and hasattr(ex, 'message'):
                ex_str = "{0}{1}{2}".format(type(ex).__name__,
                                            " - " if ex.message else "",
                                            ex.message)
            else:
                ex_str = str(ex)
                ex_str = "{0}{1}{2}".format(
                    type(ex).__name__, " - " if ex_str else "", ex_str)
        except:
            ex_str = repr(ex)
        return ex_str

    def process(self, msg, kwargs):
        """ Add some additional context to our log record

        Args:
            msg (str): The message to be logged.

        Kwargs:
            see LoggerAdapter.process

        Returns:
            The (possibly modified) versions of the arguments passed in.
        """

        self.extra["niotime"] = get_nio_time()
        self.extra["context"] = self.logger.name
        return super().process(msg, kwargs)

    @classmethod
    def name_to_level(cls, name):
        """ Converts from level name to level identifier

        Args:
            name (str): Log name

        Returns:
            log level identifier
        """
        return getattr(logging, name.upper(), None)


__author__ = 'sstober'

import logging
log = logging.getLogger(__name__)

import numpy as np

KEYSTROKE_BASE_ID = 2000

def get_event_id(stimulus_id, condition):
    return stimulus_id * 10 + condition

def decode_event_id(event_id):
    if event_id < 1000:
        stimulus_id = event_id / 10
        condition = event_id % 10
        return stimulus_id, condition
    else:
        return event_id

def get_event_string(event_id):
    if event_id < 1000:
        """
        Event Ids < 1000 are trial labels
        with the last digit indicating the condition
                1 : 'perception',
                2 : 'cued imag',
                3 : 'imag fix cross',
                4 : 'imagination',
        and the remaining digits referring to the stimulus id.
        """
        stimulus_id, condition = decode_event_id(event_id)
        return 'stimulus {}, condition {}'.format(stimulus_id, condition)
    else:
        return {
            1000: 'audio onset',
            1111: 'noise',
            KEYSTROKE_BASE_ID: 'imagination failed',
            KEYSTROKE_BASE_ID+1: 'imagination okay'
        }[event_id]

def print_event_type_counts(event_types, decoder=None):
    types, counts = np.unique(event_types, return_counts=True)
    for i in xrange(len(types)):
        label = types[i]
        if decoder is not None:
            label = decoder(label)
        log.debug('{}: {}'.format(label, counts[i]))


from flask import Flask,request,send_from_directory,jsonify
from utils import UPLOAD_COMPRESSED_FILE,UPLOAD_DECOMPRESSED_FILE,UPLOAD_TXT_FILE
import os
from werkzeug.utils import secure_filename
from huffman import huffmanCoding
from flask_cors import CORS

app = Flask(__name__)
CORS(app)

app.config['UPLOAD_COMPRESSED_FILE'] = UPLOAD_COMPRESSED_FILE
app.config['UPLOAD_DECOMPRESSED_FILE'] = UPLOAD_DECOMPRESSED_FILE
app.config['UPLOAD_TXT_FILE'] = UPLOAD_TXT_FILE

@app.route('/')
def index():
    return jsonify({'hello':'world'})

@app.route('/compress',methods=['GET','POST'])
def compress():

    if request.method == 'POST':

        file = request.files.get('file')

        _,ext = os.path.splitext(file.filename)

        if ext != '.txt':
            return jsonify({"success":False,"message":'Only Text files can be Decompressed'})

        filename = secure_filename(file.filename)
        path = os.path.join(app.config['UPLOAD_TXT_FILE'], filename)
        file.save(path)

        huffmanCoding.path = path
        huffmanCoding.compression()

        return jsonify({"success":True,"message":'File Successfully Compressed'})

    else:
        return jsonify({"success":False,"message":'Invalid Method'})

@app.route('/decompress/<path:filename>')
def decompress(filename):

    fileName,_ = os.path.splitext(filename)
    path = os.path.join(app.config['UPLOAD_TXT_FILE'], filename)

    huffmanCoding.path = path
    huffmanCoding.compression()
    huffmanCoding.decompression(fileName)

    return jsonify({"success":True,"message":'File Successfully Decompressed'})

@app.route('/download/file/<filename>/<type>')
def sendFile(filename,type):
    fileName,_ = os.path.splitext(filename)
    if type == "binary":
        return send_from_directory(app.config['UPLOAD_COMPRESSED_FILE'],path=fileName+'.bin',as_attachment=True)
    else:
        return send_from_directory(app.config['UPLOAD_DECOMPRESSED_FILE'],path=fileName+'_decompressed.txt',as_attachment=True)

if __name__ == '__main__':
    app.run(host='0.0.0.0',port=5000)


# This code is supporting material for the book
# Building Machine Learning Systems with Python
# by Willi Richert and Luis Pedro Coelho
# published by PACKT Publishing
#
# It is made available under the MIT License

import os
import sys
import collections
import csv
import json

from matplotlib import pylab
import numpy as np

DATA_DIR = "data"
CHART_DIR = "charts"

if not os.path.exists(DATA_DIR):
    raise RuntimeError("Expecting directory 'data' in current path")

if not os.path.exists(CHART_DIR):
    os.mkdir(CHART_DIR)

def tweak_labels(Y, pos_sent_list):
    pos = Y == pos_sent_list[0]
    for sent_label in pos_sent_list[1:]:
        pos |= Y == sent_label

    Y = np.zeros(Y.shape[0])
    Y[pos] = 1
    Y = Y.astype(int)

    return Y

def load_sanders_data(dirname=".", line_count=-1):
    count = 0

    topics = []
    labels = []
    tweets = []

    with open(os.path.join(DATA_DIR, dirname, "corpus.csv"), "r") as csvfile:
        metareader = csv.reader(csvfile, delimiter=',', quotechar='"')
        for line in metareader:
            count += 1
            if line_count > 0 and count > line_count:
                break

            topic, label, tweet_id = line

            tweet_fn = os.path.join(
                DATA_DIR, dirname, 'rawdata', '%s.json' % tweet_id)
            try:
                tweet = json.load(open(tweet_fn, "r"))
            except IOError:
                print(("Tweet '%s' not found. Skip." % tweet_fn))
                continue

            if 'text' in tweet and tweet['user']['lang'] == "en":
                topics.append(topic)
                labels.append(label)
                tweets.append(tweet['text'])

    tweets = np.asarray(tweets)
    labels = np.asarray(labels)

    return tweets, labels

def plot_pr(auc_score, name, phase, precision, recall, label=None):
    pylab.clf()
    pylab.figure(num=None, figsize=(5, 4))
    pylab.grid(True)
    pylab.fill_between(recall, precision, alpha=0.5)
    pylab.plot(recall, precision, lw=1)
    pylab.xlim([0.0, 1.0])
    pylab.ylim([0.0, 1.0])
    pylab.xlabel('Recall')
    pylab.ylabel('Precision')
    pylab.title('P/R curve (AUC=%0.2f) / %s' % (auc_score, label))
    filename = name.replace(" ", "_")
    pylab.savefig(os.path.join(CHART_DIR, "pr_%s_%s.png" %
                               (filename, phase)), bbox_inches="tight")

def show_most_informative_features(vectorizer, clf, n=20):
    c_f = sorted(zip(clf.coef_[0], vectorizer.get_feature_names()))
    top = list(zip(c_f[:n], c_f[:-(n + 1):-1]))
    for (c1, f1), (c2, f2) in top:
        print("\t%.4f\t%-15s\t\t%.4f\t%-15s" % (c1, f1, c2, f2))

def plot_log():
    pylab.clf()
    pylab.figure(num=None, figsize=(6, 5))

    x = np.arange(0.001, 1, 0.001)
    y = np.log(x)

    pylab.title('Relationship between probabilities and their logarithm')
    pylab.plot(x, y)
    pylab.grid(True)
    pylab.xlabel('P')
    pylab.ylabel('log(P)')
    filename = 'log_probs.png'
    pylab.savefig(os.path.join(CHART_DIR, filename), bbox_inches="tight")

def plot_feat_importance(feature_names, clf, name):
    pylab.clf()
    coef_ = clf.coef_
    important = np.argsort(np.absolute(coef_.ravel()))
    f_imp = feature_names[important]
    coef = coef_.ravel()[important]
    inds = np.argsort(coef)
    f_imp = f_imp[inds]
    coef = coef[inds]
    xpos = np.array(list(range(len(coef))))
    pylab.bar(xpos, coef, width=1)

    pylab.title('Feature importance for %s' % (name))
    ax = pylab.gca()
    ax.set_xticks(np.arange(len(coef)))
    labels = ax.set_xticklabels(f_imp)
    for label in labels:
        label.set_rotation(90)
    filename = name.replace(" ", "_")
    pylab.savefig(os.path.join(
        CHART_DIR, "feat_imp_%s.png" % filename), bbox_inches="tight")

def plot_feat_hist(data_name_list, filename=None):
    pylab.clf()
    num_rows = 1 + (len(data_name_list) - 1) / 2
    num_cols = 1 if len(data_name_list) == 1 else 2
    pylab.figure(figsize=(5 * num_cols, 4 * num_rows))

    for i in range(num_rows):
        for j in range(num_cols):
            pylab.subplot(num_rows, num_cols, 1 + i * num_cols + j)
            x, name = data_name_list[i * num_cols + j]
            pylab.title(name)
            pylab.xlabel('Value')
            pylab.ylabel('Density')
            # the histogram of the data
            max_val = np.max(x)
            if max_val <= 1.0:
                bins = 50
            elif max_val > 50:
                bins = 50
            else:
                bins = max_val
            n, bins, patches = pylab.hist(
                x, bins=bins, normed=1, facecolor='green', alpha=0.75)

            pylab.grid(True)

    if not filename:
        filename = "feat_hist_%s.png" % name

    pylab.savefig(os.path.join(CHART_DIR, filename), bbox_inches="tight")

def plot_bias_variance(data_sizes, train_errors, test_errors, name):
    pylab.clf()
    pylab.ylim([0.0, 1.0])
    pylab.xlabel('Data set size')
    pylab.ylabel('Error')
    pylab.title("Bias-Variance for '%s'" % name)
    pylab.plot(
        data_sizes, train_errors, "-", data_sizes, test_errors, "--", lw=1)
    pylab.legend(["train error", "test error"], loc="upper right")
    pylab.grid()
    pylab.savefig(os.path.join(CHART_DIR, "bv_" + name + ".png"))

def load_sent_word_net():

    sent_scores = collections.defaultdict(list)
    sentiwordnet_path = os.path.join(DATA_DIR, "SentiWordNet_3.0.0_20130122.txt")

    if not os.path.exists(sentiwordnet_path):
        print("Please download SentiWordNet_3.0.0 from http://sentiwordnet.isti.cnr.it/download.php, extract it and put it into the data directory")
        sys.exit(1)

    with open(sentiwordnet_path, 'r') as csvfile:
        reader = csv.reader(csvfile, delimiter='\t', quotechar='"')
        for line in reader:
            if line[0].startswith("#"):
                continue
            if len(line) == 1:
                continue

            POS, ID, PosScore, NegScore, SynsetTerms, Gloss = line
            if len(POS) == 0 or len(ID) == 0:
                continue
            # print POS,PosScore,NegScore,SynsetTerms
            for term in SynsetTerms.split(" "):
                # drop #number at the end of every term
                term = term.split("#")[0]
                term = term.replace("-", " ").replace("_", " ")
                key = "%s/%s" % (POS, term.split("#")[0])
                sent_scores[key].append((float(PosScore), float(NegScore)))
    for key, value in sent_scores.items():
        sent_scores[key] = np.mean(value, axis=0)

    return sent_scores

def log_false_positives(clf, X, y, name):
    with open("FP_" + name.replace(" ", "_") + ".tsv", "w") as f:
        false_positive = clf.predict(X) != y
        for tweet, false_class in zip(X[false_positive], y[false_positive]):
            f.write("%s\t%s\n" %
                    (false_class, tweet.encode("ascii", "ignore")))

if __name__ == '__main__':
    plot_log()


import math
from builtins import enumerate

from sklearn.metrics import confusion_matrix,roc_auc_score
from tensorflow.keras.regularizers import l2
from tensorflow.keras.layers import Lambda
import tensorflow as tf
import numpy as np
from tensorflow.keras.models import save_model, load_model
import random
from imblearn.keras import BalancedBatchGenerator
from tensorflow.keras.preprocessing.image import ImageDataGenerator as IDG
from sklearn.metrics import confusion_matrix
from tensorboard.plugins.hparams import api as hp
from imblearn.keras import BalancedBatchGenerator, balanced_batch_generator
from scipy.ndimage.filters import median_filter
from tensorflow.keras import regularizers
from tensorflow.keras.models import Sequential
from tensorflow.keras.callbacks import ReduceLROnPlateau, LearningRateScheduler, TensorBoard
from tensorflow.keras.layers import concatenate, Dense, Dropout, Input, Activation, Flatten, Conv2D, MaxPooling2D, \
    AveragePooling2D, GlobalAveragePooling2D, BatchNormalization
from modules.Sampler import Sampler
from modules import LoadData as ld
from modules.LoadData import load_data
from modules.Generators import *
from modules.CallBacks import *
import os
import pickle

class DMTL:
    def stage_model(self,stage,second_model,params,loss,base_model):
      Model=base_model
      optimizer = params["optimizer"]
      if stage==2:
          metrics = ["accuracy"]
          labels = ['h','v','a']
          label_index=self.label_index
          for l in second_model.layers:
            l.trainable = False
          y=second_model.layers[-2].output
          out=[]
          for k in range(len(labels)):
              x=y
              for i in range(params["number_of_dense"][k]):
                  if params["regularizition"][k] != 0.0:
                      x = Dense(params["nn"][k], activation=params["dense_activation"][k],trainable=(self.label_index==k),
                                kernel_regularizer=l2(params["regularizition"][k]), name="dense_" + labels[k] + str(i))(x)
                  else:
                      x = Dense(params["nn"][k], activation=params["dense_activation"][k],trainable=(self.label_index==k),
                                name="dense_" + labels[k] + str(i))(x)
                  if params["dropout"][k] != 0.0:
                      x = Dropout(params["dropout"][k],trainable=(self.label_index==k), name="dropout_" + labels[k] + str(i))(x)
              x = Dense(1, activation="sigmoid",trainable=(self.label_index==k),name="class_"+labels[k])(x)
              out.append(x)
          model = tf.keras.Model(second_model.input,out)
          model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
          return model
      else:
          metrics = [Meric.a, Meric.v, Meric.h, Meric.loss_a, Meric.loss_v, Meric.loss_h]

          inp = Input((64, 64, 1))
          con = concatenate([inp, inp, inp])

          model = Model(include_top=False, weights='imagenet', input_tensor=con)

          y=Flatten()(model.layers[-1].output)
          for i in range(params["number_of_dense"]):
              x=y
              if params["regularizition"]!=0.0:

                  x = Dense(params["nn"], activation=params["dense_activation"],kernel_regularizer=l2(params["regularizition"]), name="BigDense")(x)
              else:

                  x = Dense(params["nn"], activation=params["dense_activation"],name="BigDense")(x)
          if params["dropout"]!=0.0:
            x = Dropout(params["dropout"], name="BigDrop")(x)
          x = Dense(3, activation="sigmoid",name="classification")(x)
          model = tf.keras.Model(inp,x)
          model.compile(optimizer=optimizer, loss=loss, metrics=["accuracy"])
          return model
    def __init__(self,params, base_model, label, loss, second_model,phase,data=None):
        if data==None:
          data=self.prepare_data(phase,params)
        self.phase=phase
        labels = ['h','v','a']
        self.label_index = labels.index(label) if phase==2 else -1
        default_params = {"agumentation": False, "scale": False, "dense_activation": "relu", "regularizition": 0.0
            , "dropout": 0.0, "optimizer": "adam", "number_of_dense": 1, "balancer": "None", "batch_size": 32,
                          "nn": 1024}
        default_params.update(params)

        params = default_params

        self.agumnetation = params["agumentation"]

        ############ Creating CNN ##############
        optimizer = params["optimizer"]

        model=self.stage_model(phase,second_model,params,loss,base_model)
        self.__model = model
        self.__data = data
        self.balancer = params["balancer"]
        self.batch_size=params["batch_size"]
        self.__number_of_dense = params["number_of_dense"]
        tmp = list(params.values())
        keys = list(params.keys())
        if phase==2:
          for i in range(len(keys)):
              if keys[i] in ["dense_activation", "regularizition", "dropout", "number_of_dense"]:
                  tmp[i] = tmp[i][self.label_index]

        self.details = [keys[i] + ":" + str(tmp[i]) for i in range(len(keys))]
    def train(self, epochs, load_best_weigth, verbose, TensorB, name_of_best_weight, phase,save_m):
        if phase == "train" and self.agumnetation:
            self.__data["x_train"] = self.__data["x_train_128"]
        # self.__data["x_val"] = self.__data["x_val_128"]
        batch_size = self.batch_size
        balancer = self.balancer
        callbacks = [DMTL_ModelCheckpoint(self.__data["x_val"], self.__data["y_val"], self.__model, name_of_best_weight,self.label_index)]
        if TensorB:  # save History of train
            tb = TensorBoard(log_dir="log_train/" + "#".join(self.details))
            callbacks.append(tb)
        '''
        Here we have two kinds of sampler.
        online,offline
        online:for fix data on every epoch(we will use it in fit_generator)
        offline:for fix data before train

        onlines 1-batch_balancer:we can just use Data Agumentation with it
                2-DySa:nothing can't combine with it
        offlines:if balamcer be one of {smote,adasyn,None},then we should just our balancer to fix data and then use agumentation or not.

        '''
        ##batch_balancer
        if balancer == "batch_balancer":
            S = [[], []]
            for i in range(len(self.__data["x_train"])):
                S[self.__data["y_train"][i][0]].append(self.__data["x_train"][i])
            S = np.array(S)
            generator = BatchBalancer(S, self.agumnetation, batch_size)
            hist = self.__model.fit_generator(generator, validation_data=(self.__data["x_val"], self.__data["y_val"]),
                                              shuffle=True, callbacks=callbacks, steps_per_epoch=1000 / batch_size,
                                              epochs=epochs, verbose=verbose)
        else:  ###It means that we will use offline balancers
            self.__data = self.__getBalancedData(balancer)  # fixing data
            if self.agumnetation:
                generator = Agumentation(self.__data, batch_size)
                hist = self.__model.fit_generator(generator,
                                                  validation_data=(self.__data["x_val"], self.__data["y_val"]),
                                                  shuffle=True, callbacks=callbacks, steps_per_epoch=1000 / batch_size,
                                                  epochs=epochs, verbose=verbose)
            ###It means that we will use offline balancers
            else:

                hist = self.__model.fit(self.__data["x_train"], self.__data["y_train"], epochs=epochs,
                                        batch_size=batch_size,
                                        validation_data=(self.__data["x_val"], self.__data["y_val"]), shuffle=True,
                                        callbacks=callbacks, verbose=verbose)
        if load_best_weigth:
            self.__model.load_weights(name_of_best_weight)
        if save_m:
            save_model(self.__model, "model_" + name_of_best_weight)

        # cleaning model from GPU

    def clear(self):
        tf.keras.backend.clear_session()
        del self.__model
        del self.__data

    def __getBalancedData(self, name):
        if name == "None":  # no sampler
            return self.__data
        elif name == "smote":
            return Sampler("smote", 10, self.__data).run()
        elif name == "adasyn":
            return Sampler("adasyn", 10, self.__data).run()

    def prepare_data(self,phase,params):
        d_a = load_data(label="a", phase="search")
        d_h = load_data(label="h", phase="search")
        d_v = load_data(label="v", phase="search")
        y = {"y_val", "y_train", "y_test"}
        x = {"x_val", "x_train", "x_test", "x_val_128", "x_train_128"}
        data = {}
        for t in y:
            if phase == 1:
                data[t] = np.array([[d_a[t][i][0], d_v[t][i][0], d_h[t][i][0]] for i in range(len(d_a[t]))])
            else:
                data[t] = [d_a[t], d_v[t], d_h[t]]
        for t in x:
            data[t] = d_a[t]
        if params['agumentation']:
            data["x_val"] = ld.normalize(data["x_val"])
            data["x_test"] = ld.normalize(data["x_test"])
        elif params["scale"]:
            data["x_val"] = ld.normalize(data["x_val"])
            data["x_test"] = ld.normalize(data["x_test"])
            data["x_train"] = ld.normalize(data["x_train"])
        return data

    def evaluate(self):

        X = self.__data["x_test"]
        y = self.__data["y_test"][:,self.label_index] if self.phase==1 else self.__data["y_test"][self.label_index]
        y_pred1 = self.__model.predict(X)
        y_pred = y_pred1[:,self.label_index] > 0.5 if self.phase==1 else y_pred1[self.label_index] > 0.5
        y_pred = y_pred * 1
        c_matrix = confusion_matrix(y, y_pred)
        precision = c_matrix[0, 0] / sum(c_matrix[:,0])
        recall = c_matrix[0, 0] / sum(c_matrix[ 0])
        acc = np.sum(c_matrix.diagonal()) / np.sum(c_matrix)
        f_half = 1.25 * precision * recall / (.25 * precision + recall)
        g_mean = math.sqrt(precision * recall)
        TP = c_matrix[0][0]
        TN = c_matrix[1][1]
        FN = c_matrix[0][1]
        FP= c_matrix[1][0]
        mcc = (TP * TN - FP * FN) / math.sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))
        auc = roc_auc_score(y, y_pred)
        return [acc, precision, recall, f_half, g_mean, auc, mcc]

    @staticmethod
    def k_fold(k,epochs, params1,params2, load_best_weigth, verbose, TensorB, name_of_best_weight,base_model):
        data_tmp = [None, None, None]
        flag = None
        if params2['agumentation']:
            data_tmp[0] = ld.load_data('a', phase="aug_evaluation")
            data_tmp[1] = ld.load_data('v', phase="aug_evaluation")
            data_tmp[2] = ld.load_data('h', phase="aug_evaluation")
            flag = True
        else:
            data_tmp[0] = ld.load_data('a', phase="evaluation")
            data_tmp[1] = ld.load_data('v', phase="evaluation")
            data_tmp[2] = ld.load_data('h', phase="evaluation")
            flag = False
        data = {}
        data['x'] = data_tmp[0]['x']
        data['y'] = [data_tmp[0]['y'], data_tmp[1]['y'], data_tmp[2]['y']]
        size = len(data['x']) // k
        tmp_idx = np.arange(data['x'].shape[0])
        np.random.shuffle(tmp_idx)
        x = data['x'][tmp_idx]
        y = [data['y'][0][tmp_idx], data['y'][1][tmp_idx], data['y'][2][tmp_idx]]
        acc_vec = []
        np.save("x.npy",x)
        np.save("y.npy",y)
        for i in range(k):
            x_test = x[i * size:(i + 1) * size]

            y_test = [y[0][i * size:(i + 1) * size], y[1][i * size:(i + 1) * size], y[2][i * size:(i + 1) * size]]
            x_train = np.append(x[0:i * size], x[(i + 1) * size:], axis=0)
            y_train = [np.append(y[0][0:i * size], y[0][(i + 1) * size:], axis=0),
                       np.append(y[1][0:i * size], y[1][(i + 1) * size:], axis=0),
                       np.append(y[2][0:i * size], y[2][(i + 1) * size:], axis=0)]

            y_test1 = np.array([[y[0][i][0],y[1][i][0],y[2][i][0]] for i in range(i * size,(i + 1) * size)])
            y_train1=[[y[0][i][0], y[1][i][0], y[2][i][0]] for i in range(i * size)]+[[y[0][i][0], y[1][i][0], y[2][i][0]] for i in range((i + 1) * size,len( y[2]))]
            y_train1=np.array(y_train1)
            tmp = random.sample(range(len(x_train)), 232)
            x_val = []
            y_val = [[], [], []]
            y_val1=[]
            for j in tmp:
                x_val.append(x_train[j])
                y_val[0].append(y_train[0][j])
                y_val[1].append(y_train[1][j])
                y_val[2].append(y_train[2][j])
                y_val1.append([y_train[0][j][0],y_train[1][j][0],y_train[2][j][0]])
            y_val1=np.array(y_val1)
            x_val = np.array(x_val)

            x_train = np.delete(x_train, tmp, axis=0)
            y_train[0] = np.delete(y_train[0], tmp, axis=0)
            y_train[1] = np.delete(y_train[1], tmp, axis=0)
            y_train[2] = np.delete(y_train[2], tmp, axis=0)
            y_train1=np.delete(y_train1, tmp, axis=0)
            for j in range(3):
                y_val[j]=np.array(y_val[j])
            ##########fixing data#########
            data2 = ld.fix_data(flag, x_train, y_train, x_val, y_val, x_test, y_test)
            data1 = ld.fix_data(flag, x_train, y_train1, x_val, y_val1, x_test, y_test1)

            model = DMTL(params=params1,base_model=base_model,label="",loss='binary_crossentropy',second_model=None,phase=1,data=data1)

            model.train(1, False, verbose, TensorB, name_of_best_weight + str(i) + ".h5", "k_fold",save_m=True)

            epochs=[1,1,1]
            labels=['a','v','h']
            for l in range(3):
                second_model = load_model("model_"+name_of_best_weight + str(i) + ".h5",custom_objects={
                  "a":Meric.a,"v":Meric.v,"h":Meric.h,"loss_a":Meric.loss_a,"loss_v":Meric.loss_v,"loss_h":Meric.loss_h
                })

                model = DMTL(params=params2,base_model=base_model,label=labels[l],loss='binary_crossentropy',second_model=second_model,phase=2,data=data2)
                model.train(epochs[l], load_best_weigth, verbose, TensorB, name_of_best_weight + str(i) + ".h5", "k_fold",save_m=False)
                ans =model.evaluate()
                acc_vec.append(ans)
                print(ans)
                model.clear()
                del model
        return acc_vec

class Meric:
    def loss_a(y_true, y_pred):
        a_pred = y_pred[:, 0]
        a_true = y_true[:, 0]
        bce = tf.keras.losses.BinaryCrossentropy()
        return bce(a_true, a_pred)

    def loss_v(y_true, y_pred):
        a_pred = y_pred[:, 1]
        a_true = y_true[:, 1]
        bce = tf.keras.losses.BinaryCrossentropy()
        return bce(a_true, a_pred)

    def loss_h(y_true, y_pred):
        a_pred = y_pred[:, 2]
        a_true = y_true[:, 2]
        bce = tf.keras.losses.BinaryCrossentropy()
        return bce(a_true, a_pred)

    def a(y_true, y_pred):
        a_pred = y_pred[:, 0]
        a_true = y_true[:, 0]
        bce = tf.keras.losses.BinaryCrossentropy()
        return tf.keras.metrics.binary_accuracy(a_true, a_pred)

    def v(y_true, y_pred):
        a_pred = y_pred[:, 1]
        a_true = y_true[:, 1]
        bce = tf.keras.losses.BinaryCrossentropy()
        return tf.keras.metrics.binary_accuracy(a_true, a_pred)

    def h(y_true, y_pred):
        a_pred = y_pred[:, 2]
        a_true = y_true[:, 2]
        bce = tf.keras.losses.BinaryCrossentropy()
        return tf.keras.metrics.binary_accuracy(a_true, a_pred)


from os import path

from common import ARCH_X64, ARCH_X86

INSTALL_KIND_ENTERPRISE = "Enterprise"
INSTALL_KIND_PROFESSIONAL = "Professional"
INSTALL_KIND_COMMUNITY = "Community"

# Ordered by which install to use, from top to bottom
INSTALL_KINDS = (
  INSTALL_KIND_ENTERPRISE,
  INSTALL_KIND_PROFESSIONAL,
  INSTALL_KIND_COMMUNITY)

_DEFAULT_VS_DIRECTORY = "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017"

_CARGO_TARGET = {
  ARCH_X86: "i686-pc-windows-msvc",
  ARCH_X64: "x86_64-pc-windows-msvc" }

def command_line_parser():
  import argparse

  parser = argparse.ArgumentParser(add_help=False)

  parser.add_argument(
    "--vs-directory",
    dest="vs_directory",
    action="store",
    metavar="DIR",
    required=False,
    help=
      "Where Visual Studio is located - by default, "
      f"`{_DEFAULT_VS_DIRECTORY}\\[Kind]'")

  return parser

def cargo_target(args):
  return _CARGO_TARGET[args.host]

def cmake_generator(args):
  return "Ninja"

def environment(args):
  import re

  from subprocess import run

  vcvars_dir = path.join(find_vc_install(args), "VC", "Auxiliary", "Build")

  if args.host == ARCH_X64:
    vcvars = path.join(vcvars_dir, 'vcvars64.bat')
  elif args.host == ARCH_X86:
    vcvars = path.join(vcvars_dir, 'vcvars32.bat')
  else:
    assert False

  cmd_line = (vcvars, '&', 'set')

  out = run(cmd_line, shell=True, capture_output=True)
  env = {}
  if out.returncode != 0:
    assert False
  else:
    matcher = re.compile("^([a-zA-Z0-9]*)=(.*)$")
    stdout = out.stdout.decode('utf-8')
    for line in stdout.splitlines():
      match = matcher.match(line)
      if match:
        env[match[1]] = match[2]

  env['CC'] = 'cl'
  env['CXX'] = 'cl'

  return env

def find_vc_install(args):
  if args.vs_directory:
    return args.vs_directory

  base = _DEFAULT_VS_DIRECTORY

  for kind in INSTALL_KINDS:
    kind_path = path.join(base, kind)
    if path.exists(kind_path):
      return kind_path

  assert False


# -*- coding: utf-8 -*-
"""
Created on Mon Jan 27 16:41:04 2014

@author: dim1
"""
import numpy as np
from scipy.interpolate import interp1d

def getGainsForGNLSE(freqs,fiber):
    baseDir = 'O:\\OFM\\Maser\\PM for OPO and DFG\\'
    fiberRoot = str(baseDir)
    if fiber == 'er7pm':
        fiberDataFile = 'er7pm_gain.csv'
    fiberData = np.genfromtxt(str(fiberRoot+fiberDataFile),delimiter=',')
    freqAxis = fiberData[0,:]
    gainInterp = interp1d(freqAxis[::-1], fiberData[1,::-1], kind='linear',bounds_error=False,fill_value=0)
    fractGain = gainInterp(freqs)

    return fractGain


#!/usr/bin/env python3.9
import time
import youtube_dl
from youtube_dl.utils import DownloadError, ExtractorError
from pathlib import Path
import requests
import pdftotext
import get_wayback_machine
from bs4 import BeautifulSoup
from moviepy.editor import VideoFileClip
from newspaper import Article
from ebooklib import epub

from user_settings import average_word_length, headers, wpm
from src.backend.log import log_

def extract_epub(url):
    "extracts book duration and title from epub file"
    try:
        book = epub.read_epub(url)
        items = [x for x in book.get_items()]
        text_content = ""
        for item in items:
            if hasattr(item, "get_body_content"):
                 html = item.get_body_content()
                 soup = BeautifulSoup(html, "html.parser")
                 text_content += soup.text.replace("\n", " ")

        total_words = len(text_content) / average_word_length
        estimatedReadingTime = str(round(total_words / wpm, 1))

        res = {"type": "epub",
               "length": estimatedReadingTime,
               "title": book.title.replace("\n", ""),
               "url": url}
    except Exception as e:
        log_(f"Error: {e}", False)
        res = {"type": "epub (exception)",
               "url": url}
    return res

def extract_youtube(url):
    "extracts video duration in minutes from youtube link, title etc"
    res = {}
    with youtube_dl.YoutubeDL({"quiet": True}) as ydl:
        try:
            video = ydl.extract_info(url, download=False)
        except (KeyError, DownloadError, ExtractorError) as e:
            log_(f"ERROR: Video link skipped because : error during information \
extraction from {url} : {e}", False)
            res.update({"type": "video not found", "url": url})
            return res

        title = video['title'].strip()
        res = {"type": "video",
               "title": title.replace("\n", ""),
               "url": url}

        if "duration" in video:
            length = str(round(video['duration'] / 60, 1))
            res.update({"length": length})
            res.update({"channel": video["uploader"]})
        else:
            log_("Youtube playlist detected.:", False)
            res["type"] = "video (youtube playlist)"
            if "playlist" not in res["title"].lower():
                res["title"] += " - Playlist"
            length = 0
            channels = []
            for ent in video["entries"]:
                length += ent["duration"]
                channels.append(ent["uploader"])
            res["length"] = str(round(length / 60, 1))
            if len(list(set(channels))) == 1:
                res["channel"] = channels[0]
            else:
                if "channel" in res:
                    del res["channel"]
    return res

def extract_local_video(link):
    """
    extracts video duration in minutes from local files
    https://stackoverflow.com/questions/3844430/how-to-get-the-duration-of-a-video-in-python
    """
    vid = Path(link)
    if not vid.exists():
        link = link.replace("\\", "")
        vid = Path(link)
    if not vid.exists():
        log_(f"ERROR : Thought this was a local video but file was not found! \
: {link}", False)
        return {"type": "local video not found",
                "url": link}
    clip = VideoFileClip(link)
    duration = round(clip.duration / 60, 1)
    title = clip.filename.strip()
    dic = {"type": "local video",
           "title": title.replace("\n", ""),
           "length": duration,
           "url": link}
    return dic

def extract_pdf_url(url):
    "extracts reading time from an online pdf"
    try:
        downloaded = requests.get(url, headers=headers, timeout=5)
    except requests.exceptions.ConnectTimeout as e:
        log_(f"Connection timed out, skipping url: {url}", False)
        temp_dic = {}
        temp_dic["type"] = "connection timed out"
        temp_dic["url"] = url
        return temp_dic
    open("./.temporary.pdf", "wb").write(downloaded.content)
    temp_dic = extract_pdf_local("./.temporary.pdf")
    temp_dic["type"] = "online pdf"
    temp_dic["url"] = url
    del temp_dic["title"]  # can't yet scrap pdf title reliably
    Path("./.temporary.pdf").unlink()
    return temp_dic

def extract_pdf_local(path):
    "extracts reading time from a local pdf file"
    if not Path(path).exists():
        path = path.replace(" ", "\\ ")
    try:
        with open(path, "rb") as f:
            text_content = pdftotext.PDF(f)
            text_content = " ".join(text_content).replace("\n", " ")
    except FileNotFoundError:
        log_(f"ERROR: Cannot find {path}, I thought it was a PDF", False)
        return {"type": "pdf not found",
                "url": path}

    total_words = len(text_content) / average_word_length
    estimatedReadingTime = str(round(total_words / wpm, 1))

    title = path.split(sep="/")[-1].strip()
    res = {"type": "local pdf",
           "length": estimatedReadingTime,
           "title": title.replace("\n", ""),
           "url": path}
    return res

def extract_txt(path):
    "extracts reading time from a text file"
    try:
        txt_file = Path(path)
        with txt_file.open() as f:
            lines = f.readlines()
        text_content = ' '.join(lines).replace("\n", " ")

        total_words = len(text_content) / average_word_length
        estimatedReadingTime = str(round(total_words / wpm, 1))

        title = path.split(sep="/")[-1].strip()
        res = {"type": "text",
               "length": estimatedReadingTime,
               "url": path,
               "title": title.replace("\n", "")}
        return res

    except ValueError as e:
        log_(f"ERROR: Cannot find {path} : {e}", False)
        res = {"type": "txt file not found",
               "url": path}
        return res

def extract_webpage(url, simple_method=True):
    """
    extracts reading time from a webpage, output is a dictionnary containing
    estimation of the reading time ; title of the page ; if the wayback
    machine was used
    """
    wayback_used = 0
    if simple_method:
        article = Article(url)
        try:
            article.download()
            article.parse()
            title = article.title
            if title == "WWW Error Blocked Diagnostic":
                raise Exception("Exception, found title: 'WWW Error Blocked Diagnostic'")
            text_content = " ".join(article.text.replace("\n", " ").split())
        except Exception as e:
            log_(f"Error: {e}")
            simple_method = False
    if simple_method is False:
        log_("Using fallack extractor")
        try:
            req = requests.get(url, headers=headers, timeout=5)
        except requests.exceptions.ConnectTimeout as e:  # if timed out
            log_("Connection timed out, retrying once in 5 seconds...", False)
            time.sleep(5)
            try:
                req = requests.get(url, headers=headers, timeout=10)
            except requests.exceptions.ConnectTimeout as e:
                log_(f"Connection timed out again, skipping url: {url}", False)
                res = {"type": "connection timed out",
                       "url": url}
                return res
        except requests.exceptions.ConnectionError: # if url is dead use wayback machine
            log_(f"Using the wayback machine for {url}", False)
            wayback_used = 1
            wb = get_wayback_machine.get(url)
            try:
                url = wb.links['last memento']['url']
                req = requests.get(url, headers=headers, timeout=5)
            except (requests.exceptions.ConnectionError, AttributeError) as e:
                log_(f"ERROR: url could not be found even using wayback machine : \
{url} : {e}", False)
                res = {"type": "web page not found",
                       "url": url,
                       "used_wayback_machine": "wayback url not found"}
                return res
            except requests.exceptions.ConnectTimeout as e:
                log_(f"Connection timed out on wayback machine, skipping url: {url}", False)
                res = {"type": "connection timed out",
                       "url": url}
                return res

        # extracting divs and p elements and keeping the the smallest text
        html_page = req.content
        soup = BeautifulSoup(html_page, 'html.parser')
        parsed_text_trial = []
        parsed_text_trial.append(' '.join([x.text.replace("\n", " ")
                                           for x in soup.find_all("div")]))
        parsed_text_trial.append(soup.get_text().replace("\n", " "))
        parsed_text_trial.append(" ".join([x.text.replace("\n", " ")
                                           for x in soup.find_all('p')]))
        parsed_text_trial = [" ".join(x.split()) for x in parsed_text_trial]
        parsed_text_trial.sort(key=lambda x: len(x))
        text_content = parsed_text_trial[0]

        titles = soup.find_all('title')
        if len(titles) != 0:
            title = soup.find_all('title')[0].get_text()
        else:
            title = "No title found"

    total_words = len(text_content) / average_word_length
    estimatedReadingTime = str(round(total_words / wpm, 1))
    title = title.replace("\n", "").replace("\\n", "").strip()
    log_(f"Found title: {title}")
    res = {"title": title,
           "type": "webpage",
           "length": estimatedReadingTime,
           "used_wayback_machine": wayback_used,
           "url": url}
    return res


# -*- coding: utf-8 -*-
# @Time : 2021/07/09 13:08
# @Author : yunshan
# @File : creat_yaml.py
import os
from ruamel.yaml import YAML
import yaml

# create yaml object
yaml1 =YAML()
py_dict = {
    'name':'xiao ming',
    'age':[18,19]
}

with open('./new_yaml_file.yaml','w',encoding='utf-8') as file:
    yaml1.dump(py_dict,file)

# load yaml file
with open('./new_yaml_file.yaml','r',encoding='utf-8') as yaml_data:
    data = yaml_data.read()

result = yaml.load(data)
print(result)
# print(result)
# for data in result:
#     print(data)