import importlib

import re

from datetime import datetime

from typing import Any

from xml.etree.ElementTree import Element

import xml.etree.ElementTree as ET

#from gcs_jsonlogger.gcs_logger import GcsLogger

from jsonpath_ng.ext import parse

import json

#from layers.common.error.exceptions import ExceptionHandler

#LOGGER = GcsLogger("common.mapping.mapping_framework")

TARGET_ELEMENT = "targetElement"

TARGET_COLUMN_NAME = "targetColumnName"

TARGET_STRUCT_NAME = "targetStructName"

TARGET_DATA_TYPE = "targetDataType"

NUMBERING_PLAN = "numberingPlan"

PHONE = "phone"

CURRENCY = "currency"

ISO4217CODE = "iso4217Code"

USECOMMAS = "useCommas"

FORMAT = "format"

USD = "USD"

EUR = "EUR"

GBP = "GBP"

SEPARATOR = "separator"

ATTRIBUTES = "attributes"

ARTIFACT = "artifact"

CHILDREN = "children"

MAPPINGS = "mappings"

SOURCE = "source"

ACQUIRED_COLUMN_NAME = "acquiredColumnName"

ACQUIRED_COLUMN_KEY_NAME = "acquiredColumnKeyName"

TAXONOMY = "taxonomy"

PARTIAL_MATCH = "partialMatch"

RECORD = "record"

SPLIT = "split"

CONSTANT = "constantValue"

CONVERTER = "converter"

CONVERTER_FUNCTION = "converterFunction"

CONCAT = "concat"

TAXONOMY_NAME = "taxonomyName"

CUSTOM = "custom"

DATE_CONVERSION = "dateConversion"

ISO_DATE = "isoDate"

ROOT_ELEMENT = "root_element"

JSON_PATH = "jsonPath"

CONFIGURATION = "configuration"

INPUT_FORMAT = "inputFormat"

MULTIPLY = "multiply"

STRING_REPLACE = "stringReplace"

TARGET_COLUMN_STRUCT_NAME = "targetColumnStructName"

ARRAY = "array"

Record_file="XML Automation\\FBN Dallas\\record.json"
Mapping_file="XML Automation\\FBN Dallas\\mapping.json"
records = json.load(open("Record_file"))
#print(records)
mapping = json.load(open("Mapping_file"))

class MappingFramework:

    @staticmethod

    #@ExceptionHandler.error_handler

    def handle_mapping(record: Any, mapping_dict: dict, taxonomies: dict) -> dict:

        response = {}

        target_record = {}

        #LOGGER.info(f'handle_mapping: record={record}')

        array_of_structs = []

        root_element = MappingFramework.create_root_element(mapping_dict.get(ROOT_ELEMENT))

        config = MappingFramework.get_configuration(mapping_dict)

        input_format = MappingFramework.get_config_value(config, INPUT_FORMAT)

        # If the input data is xml, we want to convert it to an ElementTree once

        # if input_format == "xml":

        #     xml_record: ElementTree = ET.ElementTree(ET.fromstring(record))

        #     record = xml_record

        mappings = mapping_dict.get(MAPPINGS)

        for mapping in mappings:

            if TARGET_ELEMENT in mapping:

                target_element_name = mapping[TARGET_ELEMENT]

                if ATTRIBUTES in mapping:

                    target_element_attr = MappingFramework.handle_iso_date_conversion(record, mapping[ATTRIBUTES])

                    target_element_attr = target_element_attr if target_element_attr else mapping[ATTRIBUTES]

                    target_element = Element(target_element_name, target_element_attr)

                else:

                    target_element = Element(target_element_name)

                if SOURCE in mapping:

                    print(target_element)

                    MappingFramework.handle_source(mapping[SOURCE], target_element, record, taxonomies)

                    target_element_value = MappingFramework.handle_data_conversion(target_element.text,

                                                                                mapping.get(TARGET_DATA_TYPE))

                    if FORMAT in mapping:

                        target_element_value = MappingFramework.format_data(target_element_value, mapping[FORMAT])

                    if target_element.text is not None and len(target_element.text) > 0:

                        target_element.text = str(target_element_value)

                        root_element.append(target_element)

                    if TARGET_COLUMN_NAME in mapping:

                        target_column_name = mapping[TARGET_COLUMN_NAME]

                        target_record[target_column_name] = target_element_value

                if CHILDREN in mapping:

                    if ARRAY == mapping.get(TARGET_DATA_TYPE):

                        if TARGET_COLUMN_STRUCT_NAME in mapping:

                            struct_dict = {mapping.get(TARGET_COLUMN_STRUCT_NAME): {}}

                        MappingFramework.handle_array(mapping.get(CHILDREN), target_element, record, taxonomies,

                                                      target_record, array_of_structs, mapping[TARGET_COLUMN_NAME], struct_dict, False)

                        array_of_structs.append(struct_dict)

                        if struct_dict.values():

                            target_record[mapping[TARGET_COLUMN_NAME]] = array_of_structs

                    else:

                        MappingFramework.handle_children(mapping.get(CHILDREN), target_element, record, taxonomies,

                                                         target_record)

                    if MappingFramework.has_children(target_element):

                        root_element.append(target_element)

        response[ARTIFACT] = root_element

        response[RECORD] = target_record

        return response

    @staticmethod

    def handle_data_conversion(value, _format):

        if not _format:

            return value

        if not value:

            return value

        try:

            if _format == 'int':

                return int(value)

            elif 'datetime' in _format:

                try:

                    _format = _format.split('~')[1]

                    if len(value) > 10:

                        return datetime.strptime(value, _format)

                    else:

                        formatted_date = datetime.strptime(value, _format)

                        return formatted_date.date()

                except ValueError as ve:

                    return value

            return value

        except Exception as ex:

            raise ex

    @staticmethod

    def format_data(value, format_dict):

        if 'phone' in format_dict:

            return MappingFramework.handle_phone_format(value, format_dict[PHONE][NUMBERING_PLAN],

                                                        format_dict[PHONE][SEPARATOR])

        elif 'currency' in format_dict:

            return MappingFramework.handle_currency_format(value, format_dict[CURRENCY][ISO4217CODE],

                                                           format_dict[CURRENCY][USECOMMAS])

    @staticmethod

    def handle_phone_format(phone_number, _format, split_char=' '):

        try:

            ph = re.sub('[^0-9]+', '', phone_number)

            formatted_phone_number = re.sub("(\d)(?=(\d{3})+(?!\d))", fr"\1{split_char}", "%s" % ph[:-1]) + ph[-1]

            return formatted_phone_number if _format == 'NANP' else '+' + formatted_phone_number.replace(split_char,

                                                                                ' ', 1)

        except Exception as ex:

            raise ex

    @staticmethod

    def handle_currency_format(currency_amount, _format, use_commas=True):

        try:

            if _format == USD:

                _format = "$"

            elif _format == EUR:

                _format = "€"

            elif _format == GBP:

                _format = "£"

            else:

                _format = ""

            currency_amount, decimal_count = (float(currency_amount), 2) if '.' in str(currency_amount) else (

                int(currency_amount), 0)

            prefix = '-' if currency_amount < 0 else ''

            comma = ',' if use_commas else ''

            formatted_amount = '%s%s%s' % (

                prefix, _format, '{:{}.{}f}'.format(abs(currency_amount), comma, decimal_count))

            return formatted_amount

        except Exception as ex:

            raise ex

    @staticmethod

    #@ExceptionHandler.error_handler

    def get_configuration(mapping: dict):

        if "configuration" in mapping:

            return mapping.get("configuration")

        return None

    @staticmethod

    def get_config_value(config: dict, key: str):

        if config is not None:

            if key in config:

                return str(config.get(key)).lower()

        return None

    @staticmethod

    #@ExceptionHandler.error_handler

    def handle_children(children, parent_element: Element, record: dict, taxonomies: dict, target_record: dict):

        for child in children:

            if TARGET_ELEMENT in child:

                target_element_name = child[TARGET_ELEMENT]

                if ATTRIBUTES in child:

                    target_element_attr = MappingFramework.handle_iso_date_conversion(record, child[ATTRIBUTES])

                    target_element_attr = target_element_attr if target_element_attr else child[ATTRIBUTES]

                    target_element = Element(target_element_name, target_element_attr)

                else:

                    target_element = Element(target_element_name)

                if SOURCE in child:

                    # has_source = True

                    MappingFramework.handle_source(child[SOURCE], target_element, record, taxonomies)

                    target_element_value = MappingFramework.handle_data_conversion(target_element.text,

                                                                                child.get(TARGET_DATA_TYPE))

                    if FORMAT in child:

                        target_element_value = MappingFramework.format_data(target_element_value, child[FORMAT])

                    if target_element.text is not None and len(target_element.text) > 0:

                        target_element.text = str(target_element_value)

                        parent_element.append(target_element)

                        # has_children = True

                    if TARGET_COLUMN_NAME in child:

                        target_column_name = child[TARGET_COLUMN_NAME]

                        target_record[target_column_name] = target_element_value

                if CHILDREN in child:

                    MappingFramework.handle_children(child[CHILDREN], target_element, record, taxonomies,

                                                     target_record)

                    if MappingFramework.has_children(target_element):

                        parent_element.append(target_element)

    @staticmethod

   # @ExceptionHandler.error_handler

    def handle_array(children, parent_element: Element, record: dict, taxonomies: dict, target_record: dict, array: list, target_column_name, struct_dict: dict, nested_children: bool):

        for child in children:

            if TARGET_ELEMENT in child:

                target_element_name = child[TARGET_ELEMENT]

                if ATTRIBUTES in child:

                    target_element_attr = MappingFramework.handle_iso_date_conversion(record, child[ATTRIBUTES])

                    target_element_attr = target_element_attr if target_element_attr else child[ATTRIBUTES]

                    target_element = Element(target_element_name, target_element_attr)

                else:

                    target_element = Element(target_element_name)

                if TARGET_COLUMN_STRUCT_NAME in child:

                    if child[TARGET_COLUMN_STRUCT_NAME] not in struct_dict:

                        child_struct = {child[TARGET_COLUMN_STRUCT_NAME]: {}}

                        struct_dict[parent_element.tag].update(child_struct)

                if SOURCE in child:

                    # has_source = True

                    MappingFramework.handle_source(child[SOURCE], target_element, record, taxonomies)

                    target_element_value = MappingFramework.handle_data_conversion(target_element.text,

                                                                                child.get(TARGET_DATA_TYPE))

                    if FORMAT in child:

                        target_element_value = MappingFramework.format_data(target_element_value, child[FORMAT])

                    if target_element.text is not None and len(target_element.text) > 0:

                        target_element.text = str(target_element_value)

                        parent_element.append(target_element)

                        # has_children = True

                    if TARGET_STRUCT_NAME in child:

                        target_struct_name = child[TARGET_STRUCT_NAME]

                        if target_element_value:

                            key = list(struct_dict.keys())[0]

                            # if key == parent_element.tag:

                            if key == parent_element.tag or not nested_children:

                                struct_dict[key][target_struct_name] = target_element_value

                            else:

                                struct_dict[key][parent_element.tag][target_struct_name] = target_element_value

                                # struct_dict['individual']['person'][target_struct_name] = target_element_value

                if CHILDREN in child:

                    nested_children = True

                    MappingFramework.handle_array(child[CHILDREN], target_element, record, taxonomies, target_record, array, target_column_name, struct_dict, nested_children)

                    if MappingFramework.has_children(target_element):

                        parent_element.append(target_element)

        # if struct_dict:

           # target_record[target_column_name] = array

    @staticmethod

    #@ExceptionHandler.error_handler

    def has_children(element: Element) -> bool:

        if len(element.findall('*')) > 0:

            return True

        else:

            return False

    @staticmethod

    #@ExceptionHandler.error_handler

    def handle_source(source_dict, element: Element, record: dict, taxonomies: dict):

        if ACQUIRED_COLUMN_NAME in source_dict:

            MappingFramework.handle_acquired_column_name(record, element, source_dict[ACQUIRED_COLUMN_NAME])

        elif TAXONOMY in source_dict:

            MappingFramework.handle_taxonomy(source_dict[TAXONOMY], element, taxonomies, record)

        elif SPLIT in source_dict:

            MappingFramework.handle_split(record, element, source_dict[SPLIT])

        elif CONVERTER in source_dict:

            MappingFramework.handle_converter(record, element, source_dict[CONVERTER])

        elif CONCAT in source_dict:

            MappingFramework.handle_concat(record, element, source_dict[CONCAT], taxonomies)

        elif CONSTANT in source_dict:

            MappingFramework.handle_constant(element, source_dict[CONSTANT])

        elif CUSTOM in source_dict:

            MappingFramework.handle_custom_converter(record, element, source_dict[CUSTOM], taxonomies)

        elif JSON_PATH in source_dict:

            print(element)

            MappingFramework.handle_json_path_expression(record, element, source_dict[JSON_PATH])

        elif MULTIPLY in source_dict:

            MappingFramework.handle_multiply(record, element, source_dict[MULTIPLY])

        elif STRING_REPLACE in source_dict:

            MappingFramework.handle_string_replace(record, element, source_dict[STRING_REPLACE])

    @staticmethod

    def handle_constant(element: Element, constant: str):

        element.text = constant

    @staticmethod

    def handle_taxonomy(taxonomy_info: dict, element, taxonomies: dict, record: dict):

        element.text = MappingFramework.get_taxonomy_value(taxonomy_info, taxonomies, record)

    @staticmethod

   # @ExceptionHandler.error_handler

    def get_taxonomy_value(taxonomy_info: dict, taxonomies: dict, record: dict) -> str:

        taxonomy_value = ''

        taxonomy_table_name = taxonomy_info[TAXONOMY_NAME]

        taxonomy_key_column_name = taxonomy_info[ACQUIRED_COLUMN_KEY_NAME]

        if taxonomy_key_column_name in record:

            if PARTIAL_MATCH in taxonomy_info:

                for key in taxonomies[taxonomy_table_name]:

                    if record[taxonomy_key_column_name].find(key) > 0:

                        taxonomy_value = taxonomies[taxonomy_table_name].get(key)

            else:

                taxonomy_value = taxonomies[taxonomy_table_name].get(str(record[taxonomy_key_column_name]).strip())

        return taxonomy_value

    @staticmethod

    #@ExceptionHandler.error_handler

    def handle_split(record: dict, element: Element, split: dict):

        key_column_name = split[ACQUIRED_COLUMN_NAME]

        split_char = split["splitChar"]

        index_pos = split["indexPos"]

        if key_column_name in record:

            element.text = MappingFramework.split(record[key_column_name], split_char, index_pos)

    @staticmethod

   # @ExceptionHandler.error_handler

    def split(value: str, separator: chr, index: int):

        if value is not None and separator in value:

            split_list = value.split(separator)

            if index < len(split_list):

                return value.split(separator)[index]

        else:

            return ''

    @staticmethod

   #@ExceptionHandler.error_handler

    def handle_concat(record: dict, element: Element, concat_config: dict, taxonomies: dict):

        concat_value = ''

        for concat_item in concat_config:

            if TAXONOMY in concat_item:

                concat_value = concat_value + MappingFramework.get_taxonomy_value(concat_item[TAXONOMY], taxonomies,

                                                                                record)

            elif ACQUIRED_COLUMN_NAME in concat_item:

                concat_value = concat_value + MappingFramework.get_acquired_column_value(record, concat_item[

                    ACQUIRED_COLUMN_NAME])

        element.text = concat_value

    @staticmethod

    #@ExceptionHandler.error_handler

    def handle_converter(record: dict, parent_element: Element, converter: dict):

        acquired_column_name = converter[ACQUIRED_COLUMN_NAME]

        converter_function = converter[CONVERTER_FUNCTION]

        if converter_function == "yesno" and acquired_column_name in record:

            parent_element.text = MappingFramework.yesno_converter(str(record[acquired_column_name]))

        elif converter_function == "streetCleanup" and acquired_column_name in record:

            parent_element.text = MappingFramework.street_converter(str(record[acquired_column_name]))

    @staticmethod

    #@ExceptionHandler.error_handler

    def yesno_converter(status: str) -> str:

        temp_status = status.lower().strip()

        if temp_status == 'y':

            status = 'YES'

        elif temp_status == 'n':

            status = 'NO'

        else:

            status = 'NO'

        return status

    @staticmethod

   # @ExceptionHandler.error_handler

    def street_converter(street: str) -> str:

        # add spaces around # and -

        street = re.sub(r"(#)", r" \1 ", street)

        # add spaces between certain dwelling types and alpha-numberic

        street = re.sub(r"\b(APT|UNIT|BSMT|BLDG?|DEPT|FRNT|HNGR|LBBY|LOT|LOWR|OFFICE|OFC|PIER|ROOM|RM|SPC|STOP|SUITE|TRLR|UPPR|SIDE|SLIP|REAR|KEY|STE|FL|PH|HWY)\s?(\W*|\w*)", r"\1 \2", street, 2)

        # cleanup multiple spaces added by previous regex's

        street = re.sub(' +', ' ', street).rstrip()

        return street

    @staticmethod

    #@ExceptionHandler.error_handler

    def handle_acquired_column_name(record: dict, element: Element, acquired_column_name):

        if acquired_column_name in record:

            element.text = MappingFramework.get_acquired_column_value(record, acquired_column_name)

    @staticmethod

   # @ExceptionHandler.error_handler

    def handle_custom_converter(record: dict, element: Element, custom_config: dict, taxonomies: dict):

        module_name = custom_config['moduleName']

        class_name = custom_config['className']

        method_name = custom_config['methodName']

        custom_data = custom_config.get('customData')

        element.text = MappingFramework.call_custom_method(module_name, method_name, class_name, record, element,

                                                           taxonomies, custom_data)

    @staticmethod

  #  @ExceptionHandler.error_handler

    def call_custom_method(module_name, method_name, class_name, record: dict, element: Element, taxonomies: dict,

                           custom_data: str):

        module = importlib.import_module(module_name)

        clazz = getattr(module, class_name)

        func = getattr(clazz, method_name)

        if custom_data:

            record['custom_config_data'] = custom_data

        return func(record, element, taxonomies)

    @staticmethod

   # @ExceptionHandler.error_handler

    def get_acquired_column_value(record, acquired_column_name):

        if acquired_column_name in record:

            return MappingFramework.handle_zero_value(str(record[acquired_column_name]).strip())

        else:

            return ''

    @staticmethod

    def handle_zero_value(value):

        if value == '0':

            return None

        else:

            return value

    @staticmethod

   # @ExceptionHandler.error_handler

    def handle_date_conversion(value):

        iso_date = datetime.strptime(value, '%m/%d/%Y').date()

        return iso_date

    @staticmethod

  #  @ExceptionHandler.error_handler

    def create_root_element(config: dict):

        root = Element("artifact")

        root.set('xmlns:xsi', 'http://www.w3.org/2001/XMLSchema-instance')

        # need to make these as parameter in the configuration

        for k, v in config.items():

            root.set(k, v)

        return root

    @staticmethod

  #  @ExceptionHandler.error_handler

    def handle_json_path_expression(record: dict, element, json_path):

        json_path_list = []

        try:

            if isinstance(json_path, str):

                json_path_list.append(json_path)

            elif isinstance(json_path, list):

                json_path_list = json_path

            for path in json_path_list:

                jsonpath_expr = parse(path)

                get_match = jsonpath_expr.find(record)

                if len(get_match) > 0:

                    if get_match[0].value and get_match[0].value != 'None':

                        element.text = str(get_match[0].value)

                        break

                # else:

                #     element.text = None

        except Exception:

            #LOGGER.exception(f'Failed to convert record using json expression {json_path}', exc_info=True)

            element.text = None

            raise

    @staticmethod

  #.error_handler

    def handle_multiply(record: dict, element: Element, multiply: dict):

        acquired_column_name = multiply[ACQUIRED_COLUMN_NAME]

        if acquired_column_name in record:

            value = int(record[acquired_column_name])

            factor = int(multiply[CONSTANT])

            element.text = str(value * factor)

    @staticmethod

  #  @ExceptionHandler.error_handler

    def handle_string_replace(record: dict, element: Element, replace_string: dict):

        acquired_column_name = replace_string[ACQUIRED_COLUMN_NAME]

        if acquired_column_name in record:

            element.text = str(record[acquired_column_name].replace(

                replace_string['searchString'],

                replace_string['replaceString'])).strip()

    @staticmethod

   #.error_handler

    def handle_iso_date_conversion(record: dict, attr):

        if DATE_CONVERSION in attr:

            if ISO_DATE in attr[DATE_CONVERSION]:

                source_date = record[attr[DATE_CONVERSION][ISO_DATE]]

                conv_iso_date = MappingFramework.handle_date_conversion(source_date)

                return {ISO_DATE: str(conv_iso_date).replace('-', '')}

artifact = MappingFramework.handle_mapping(records,mapping,{})["artifact"]

print(artifact)

tree = ET.ElementTree(artifact)

tree.write('output1.xml')


import cv2
import math
import mediapipe as mp
from mediapipe.framework.formats.detection_pb2 import Detection
from dtos import FaceRegions
# mp_face_detection = mp.solutions.face_detection
# mp_drawing = mp.solutions.drawing_utils

class face_detection():
    def __init__(self, image):
        self.image = image
        self.image_width = image.shape[1]
        self.image_height = image.shape[0]
        self.RELATIVE_BOUNDING_BOX = 2
        self.mp_face_detection = mp.solutions.face_detection
        self.regions = []

    #detect faces
    def detect_faces(self):
        """
        Function to detect faces
        -------
        Returns:
            NONE
        """
        # model_selction = 0 -> 속도 빠른 모델, 1 -> 정확도 높은 속도 느린 모델
        # 모델 불러와서 face detect
        with self.mp_face_detection.FaceDetection(
            model_selection=0, min_detection_confidence=0.6) as face_detection:
            # To improve performance, optionally mark the image as not writeable to
            # pass by reference.
            self.image.flags.writeable = False
            self.image = cv2.cvtColor(self.image, cv2.COLOR_BGR2RGB)
            self.results = face_detection.process(self.image)

            self.image.flags.writeable = True
            self.image = cv2.cvtColor(self.image, cv2.COLOR_RGB2BGR)

            self.detections = self.results.detections

    # 다른 노드에 디텍션 정보 종합하기 위함
    # localization_to_region
    # dtos.FaceRegions
    def localization_to_region(self):
        """
        Function to detect objects
        -------
        Returns
            face_full : list
                face regions detected
            core_landmark : list, optional
                key landmarks of face (eyes, nose, mouth)
            all_landmark : list, optional
                all landmarks of face (including ears)
        """
        # 얼굴이 없는 경우도 고려해야 함.
        self.face_full = []
        self.core_landmark = []
        self.all_landmark = []

        if self.detections:
            for detection in self.detections:

                location = detection.location_data
                box = detection.location_data.relative_bounding_box

                # error 발생시켜야 함
                if(location.format != self.RELATIVE_BOUNDING_BOX):
                    print(location.format)
                    print("Face detection input is lacking required relative_bounding_box()")

                # error 발생시켜야 함
                # if(detection.location_data.relative_keypoints.size != 6):
                #     print("Face detection input expected 6 keypoints, has", location.relative_keypoints.size)

                # detection된 얼굴이 여러개인 경우 고려하여
                # 한 클래스를 한 명의 디텍션 정보로 구성하고 이를 리스트로 묶음.

                # face_full
                x = max(float(0), box.xmin)
                y = max(float(0), box.ymin)
                width = min(box.width - abs(x - box.xmin), 1 - x)
                height = min(box.height - abs(y - box.ymin), 1 - y)
                SignalType = "FACE_FULL"
                visual_score = detection.score[0]

                region = FaceRegions(x, y, width, height, SignalType, visual_score)
                self.face_full.append(region)

                # landmark 총 6개 -> 각 랜드마크 당 클래스 하나
                # core는 앞에 4개만
                core_landmark_region = []
                all_landmark_region = []

                for i in range(4):
                    keypoint = location.relative_keypoints[i]
                    x = keypoint.x
                    y = keypoint.y
                    w = self.normalize_x(1)
                    h = self.normalize_y(1)
                    SignalType = "FACE_LANDMARK"

                    region = FaceRegions(x, y, w, h, SignalType, visual_score)
                    region = self.extend_salient_region_with_point(keypoint.x, keypoint.y, region)

                    core_landmark_region.append(region)
                    all_landmark_region.append(region)

                for i in range(4, 6):
                    keypoint = location.relative_keypoints[i]
                    x = keypoint.x
                    y = keypoint.y
                    w = self.normalize_x(1)
                    h = self.normalize_y(1)
                    SignalType = "FACE_LANDMARK"

                    region = FaceRegions(x, y, w, h, SignalType, visual_score)

                    region = self.extend_salient_region_with_point(keypoint.x, keypoint.y, region)

                    all_landmark_region.append(region)

                self.core_landmark.append(core_landmark_region)
                self.all_landmark.append(all_landmark_region)

        return self.face_full, self.core_landmark, self.all_landmark

    def extend_salient_region_with_point(self, x, y, region):
        """
        Function to extend regions in consideration of error
        -------
        Parameters
            x : float
                x of face keypoint
            y : float
                y of face keypoint
            region : dataclass
                class containing face regions
        -------
        Returns
            region : dataclass
                updated regions

        """
        if(region.w is None):
            region.set_w(self.normalize_x(1))
        elif(x < region.x):
            region.set_w(region.w + region.x - x)
        elif(x > region.x + region.w):
            region.set_w(x - region.x)

        if(region.h is None):
            region.set_h(self.normalize_y(1))
        elif(y < region.y):
            region.set_h(region.h + region.y - y)
        elif(y > region.y + region.h):
            region.set_h(y - region.y)

        if(region.x is None):
            region.set_x(x)
        else:
            region.set_x(min(x, region.x))

        if(region.y is None):
            region.set_y(y)
        else:
            region.set_y(min(y, region.y))

        return region

    def normalize_x(self, pixel):
        return pixel / float(self.image_width)

    def normalize_y(self, pixel):
        return pixel / float(self.image_height)


# -*- coding: utf-8 -*-
"""
Created on Sat Dec 19 20:21:52 2020

@author: helle
"""

import matplotlib.pyplot as plt
import numpy as np

from index import *

Nerls=[NE, NP, NIp, NIh, NIi]

def popsum2d(pops, Nerls=Nerls):
    index = indexFunction(Nerls)
    popSum = np.zeros((16,pops.shape[1]))
    popSum[0] = pops[index['S__']]
    popSum[1] = np.sum(pops[1:(Nerls[0]+1)], axis = 0)            # E__
    popSum[2] = np.sum(pops[  (Nerls[0] +1):(2*Nerls[0]+1)], axis = 0)       # Et_
    popSum[3] = np.sum(pops[                (2*Nerls[0]+1):(3*Nerls[0]+1)], axis = 0)     # Es_
    popSum[4] = np.sum(pops[                               (3*Nerls[0]+1):(3*Nerls[0]+Nerls[1]+1)], axis = 0)     # P__
    popSum[5] = np.sum(pops[                                              (3*Nerls[0]+Nerls[1]+1):(3*Nerls[0]+2*Nerls[1]+1)], axis = 0)     # Pt_
    popSum[6] = np.sum(pops[                                                                      (3*Nerls[0]+2*Nerls[1]+1):(3*Nerls[0]+3*Nerls[1]+1)], axis = 0)     # Ps_
    popSum[7] = np.sum(pops[                                                                                (3*Nerls[0]+3*Nerls[1]+1):(3*Nerls[0]+3*Nerls[1]+Nerls[2]+1)], axis = 0)     # I_p
    popSum[8] = np.sum(pops[                                                                                (3*Nerls[0]+3*Nerls[1]+Nerls[2]+1):(3*Nerls[0]+3*Nerls[1]+Nerls[2]+Nerls[3]+1)], axis = 0)     # I_h
    popSum[9] = np.sum(pops[                                                                                (3*Nerls[0]+3*Nerls[1]+Nerls[2]+Nerls[3]+1):(3*Nerls[0]+3*Nerls[1]+Nerls[2]+Nerls[3]+Nerls[4]+1)], axis = 0)     # I_i
    popSum[10] = np.sum(pops[                                                                                (3*Nerls[0]+3*Nerls[1]+Nerls[2]+Nerls[3]+Nerls[4]+1):(3*Nerls[0]+3*Nerls[1]+Nerls[2]+2*Nerls[3]+Nerls[4]+1)], axis = 0)    # Ish
    popSum[11] = np.sum(pops[                                                                                (3*Nerls[0]+3*Nerls[1]+Nerls[2]+2*Nerls[3]+Nerls[4]+1):(3*Nerls[0]+3*Nerls[1]+2*Nerls[2]+2*Nerls[3]+Nerls[4]+1)], axis = 0)   # Isp
    popSum[12] = pops[index['F__']]
    popSum[13] = pops[index['B_j']]
    popSum[14] = pops[index['B_f']]
    popSum[15] = pops[index['R__']]
    return popSum

colsA = ["#000000", "#59B3E6", "#009980", "#E69900", "#CC6600", "#0073B3", 'red']
lab = ['0.0', '0.2', '0.4', '0.6', '0.8', '1.0']
index = indexFunction(Nerls)
path = 'C:/Users/helle/PycharmProjects/ebola'

def plotEbola(names, savename, path=path, col=colsA, lab=lab, type='all', Nerls=Nerls):
    index = indexFunction(Nerls)
    popSum0 = popsum2d(pops=np.loadtxt(path +'/ebola_' + names[0] + '.txt'), Nerls=Nerls)
    s = np.shape(popSum0)
    popSum = np.empty(shape = [len(names),s[0], s[1]])
    popSum[0] = popSum0
    for i in range(1,len(names)):
        pops_i = np.loadtxt(path +'/ebola_' + names[i] + '.txt')
        #plt.plot(np.sum(pops_i, axis=0), color=col[i])
        popSum[i] = popsum2d(pops_i, Nerls=Nerls)
    #plt.show()
    # total population (for test)
    #for i in range(0, len(names)):
    #    plt.plot(np.sum(popSum[i], axis =0), color=col[i])
    #plt.show()
    # Susceptible
    if type == 'all' or type == 'S':
        for i in range(0,len(names)):
            plt.plot(popSum[i][0], label = lab[i], color=col[i])
        plt.legend()
        plt.ylabel('Susceptible ind.')
        plt.savefig (path + '/Ebola_S_' + savename + '.pdf', dpi=100)
        plt.show()

    # Latent
    if type == 'all' or type == 'E':
        plt.plot(popSum[0][1], color=col[0], linestyle='-', label='never traced back')
        plt.plot(popSum[0][3], color=col[0], linestyle='--', label='not yet traced back')
        plt.plot(popSum[0][2], color=col[0], linestyle=':', label='traced back')
        for i in range(0, len(names)):
            plt.plot(popSum[i][1], label=lab[i], color=col[i], linestyle='-')
            plt.plot(popSum[i][2], color=col[i], linestyle=':')
            plt.plot(popSum[i][3], color=col[i], linestyle='--')
        plt.legend()
        plt.ylabel('Latent ind.')
        plt.savefig(path + '/Ebola_E_' + savename + '.pdf', dpi=100)
        plt.show()
    # Podromal
    if type == 'all' or type == 'P':
        plt.plot(popSum[0][4], color=col[0], linestyle='-', label='never traced back')
        plt.plot(popSum[0][6], color=col[0], linestyle='--', label='not yet traced back')
        plt.plot(popSum[0][5], color=col[0], linestyle=':', label='traced back')
        for i in range(0, len(names)):
            plt.plot(popSum[i][4], label=lab[i], color=col[i], linestyle='-')
            plt.plot(popSum[i][5], color=col[i], linestyle=':')
            plt.plot(popSum[i][6], color=col[i], linestyle='--')
        plt.legend()
        plt.ylabel('Podromal ind.')
        plt.savefig(path + '/Ebola_P_' + savename + '.pdf', dpi=100)
        plt.show()
    # Fully infected at home
    if type == 'all' or type == 'Ip':
        plt.plot(popSum[0][7], color=col[0], linestyle='-', label='never traced back')
        plt.plot(popSum[0][11], color=col[0], linestyle='--', label='not yet traced back')
        for i in range(0, len(names)):
            plt.plot(popSum[i][7], label=lab[i], color=col[i], linestyle='-')
            plt.plot(popSum[i][11], color=col[i], linestyle=':')
        plt.legend()
        plt.ylabel('Fully inf. ind. at home')
        plt.savefig(path + '/Ebola_Ip_' + savename + '.pdf', dpi=100)
        plt.show()
    # Fully infected in hospital
    if type == 'all' or type == 'Ih':
        plt.plot(popSum[0][8], color=col[0], linestyle='-', label='never traced back')
        plt.plot(popSum[0][10], color=col[0], linestyle='--', label='not yet traced back')
        for i in range(0, len(names)):
            plt.plot(popSum[i][8], label=lab[i], color=col[i], linestyle='-')
            plt.plot(popSum[i][10], color=col[i], linestyle=':')
        plt.legend()
        plt.ylabel('Fully inf. ind. in hospital')
        plt.savefig(path + '/Ebola_Ih_' + savename + '.pdf', dpi=100)
        plt.show()
    # Fully infected in isolation
    if type == 'all' or type == 'Ii':
        for i in range(0, len(names)):
            plt.plot(popSum[i][9], label=lab[i], color=col[i], linestyle=':')
        plt.legend()
        plt.ylabel('Fully inf. ind. in isolation')
        plt.savefig(path + '/Ebola_Ii_' + savename + '.pdf', dpi=100)
        plt.show()
    # Buried
    if type == 'all' or type == 'B':
        plt.plot(popSum[0][14], color=col[0], linestyle='-', label='unsafely')
        plt.plot(popSum[0][13], color=col[0], linestyle='--', label='safely')
        for i in range(0, len(names)):
            plt.plot(popSum[i][13], color=col[i], linestyle='--')
            plt.plot(popSum[i][14], label=lab[i], color=col[i], linestyle='-')
        plt.legend()
        plt.ylabel('Buried ind.')
        plt.savefig(path + '/Ebola_B_' + savename + '.pdf', dpi=100)
        plt.show()
    # Unsafe funerals
    if type == 'all' or type == 'F':
        for i in range(0, len(names)):
            plt.plot(popSum[i][12], label=lab[i], color=col[i], linestyle='-')
        plt.legend()
        plt.ylabel('Unsafe funerals')
        plt.savefig(path + '/Ebola_F_' + savename + '.pdf', dpi=100)
        plt.show()
    # Recovered
    if type == 'all' or type == 'R':
        for i in range(0, len(names)):
            plt.plot(popSum[i][15], label=lab[i], color=col[i], linestyle='-')
        plt.legend()
        plt.ylabel('Recovered ind')
        plt.savefig(path + '/Ebola_R_' + savename + '.pdf', dpi=100)
        plt.show()

    if type == 'all' or type == 'mix':
        plt.plot(popSum[0][0] + popSum[0][15], label='healthy (S+R)', color=col[0], linestyle='-')
        plt.plot(np.sum(popSum[0][1:12], axis=0), label='infected (E+P+I)', color=col[0], linestyle='--')
        plt.plot(np.sum(popSum[0][12:15], axis=0), label='dead (D+B)', color=col[0], linestyle=':')
        for i in range(0, len(names)):
            plt.plot(popSum[i][0] + popSum[i][15], label=lab[i],color=col[i], linestyle='-')
            print('healthy ' + lab[i] + ': '+ str(round((popSum[i][0] + popSum[i][15])[-1])))
            plt.plot(np.sum(popSum[i][1:12], axis=0), color=col[i], linestyle='--')
            print('infected ' + lab[i] + ': ' + str(round(np.sum(popSum[i][1:12], axis=0)[-1])))
            plt.plot(np.sum(popSum[i][12:15], axis=0), color=col[i], linestyle=':')
            print('dead ' + lab[i] + ': ' + str(round(np.sum(popSum[i][12:15], axis=0)[-1])))
        print('---------')
        plt.legend()
        plt.ylabel('Individuals')
        plt.savefig(path + '/Ebola_mix_' + savename + '.pdf', dpi=100)
        plt.show()
    return(savename)


import os
import argparse

#   tests 里的是 configs_XXX.sh 后面的 XXX
#   例: configs_example.sh 中的 XXX 为 example
#   如果要同时做多组就写成 tests=['example1','example2','example3',...]
sim_tests=['example']
reload_tests=['example']

parser=argparse.ArgumentParser(description='A tool for simpoint and reload checkpoints')
parser.add_argument("-n","--name",help="use NAME to replace the original test.")
parser.add_argument("-s","--simpoint",help="use simpoint to generate checkpoint",
                    action="store_true")
parser.add_argument("-r","--reload",help="use new parms to reload checkpoint",
                    action="store_true")
args=parser.parse_args()
#print(args)

def simpoint(tests):
    for item in tests:
        #   第一个是进行 Simpoint 模拟的，第二个是正常跑的，是验证 Simpoint 结果准确性的，
        #   理论上生成 Simpoint 就可以了，第二行命令可以注释掉
        #   ">" 后面的是将运行的输出重定向的地址，改成自己的即可
        os.system('nohup time bash process/process_sim.sh '+item+' > \
                /home/data/ChenRuiyang/time_rec/'+item+'_process.txt &')
        #os.system('nohup time bash process/process_normal.sh '+item+' > \
            # /home/data/ChenRuiyang/time_rec/'+item+'_normal.txt &')

def reload(tests):
    for item in tests:
        #   同上，只改了.sh文件的名字，执行重载操作
        #   ">" 后面的是将运行的输出重定向的地址，改成自己的即可
        os.system('nohup time bash process/process_reload.sh '+item+' > \
            /home/data/ChenRuiyang/time_rec/'+item+'_reload.txt &')

if args.name:
    sim_tests=reload_tests=[args.name]
    print("Using [",args.name,"] as the new test suffix.")
    print()

#   这里不能 simpoint 和 reload 同时做，
#   因为 reload 需要 simpoint 的结果.
if args.simpoint:
    print("Starting simpoint!")
    print()
    simpoint(sim_tests)
elif args.reload:
    print("Starting Reload!")
    print()
    reload(reload_tests)
else:
    print("Please give operations.")
    print("Use -h to get help")
    print()


import re
import requests

start = re.compile('(
Available Rooms
)');
pattern = re.compile('(([^< ]*)[^<]*)')
end = re.compile('(
)');

def get_available_rooms():
    r = requests.get('https://simmons-hall.scripts.mit.edu:444/rooming/text', cert='keystore.pem')
    content = r.text
    content = content[start.search(content).end(1):]
    content = content[:end.search(content).start(1)]
    available = set()
    while True:
        match = pattern.search(content)
        if match is None:
            break
        rm_num = match.group(2)
        content = content[match.end(1):]
        available.add(rm_num)
    return available


from bs4 import BeautifulSoup as bs
from splinter import Browser
import time
import os
import pandas as pd
import requests
from pprint import pprint

### NASA Mars News
url = "https://mars.nasa.gov/news/"
response = requests.get (url)
response
def init_browser():
    executable_path = {"executable_path": "chromedriver.exe"}
    return Browser("chrome", **executable_path, headless=False)
browser = init_browser()
browser.visit(url)
html = browser.html
time.sleep(4)
soup = bs(html, 'html.parser')
time.sleep(2)
results = soup.find_all("div",class_="list_text")
marsinfo={}
for result in results[:1]:
    try:
        news_title=result.find('div',class_="content_title").a.text
        news_description=result.find('div',class_="article_teaser_body").text
        if(news_description and news_title):
            print("Title:")
            print(news_title)
            print("Description:")
            print(news_description)
    except AttributeError as e:
        print(e)

news_date=result.find('div',class_="list_date").text
print("Date of Article:")
print(news_date)
marsinfo["title"]=news_title
marsinfo["paragraph"]=news_description
marsinfo["date"]=news_date
browser.quit()

### JPL Mars Space Images - Featured Image
url="https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path)
browser.visit(url)
time.sleep(3)

soup = bs(browser.html, 'html.parser')
featured_image=soup.find('article', class_="carousel_item")
pprint(featured_image)
forurl = featured_image['style'][23:-3]
featured_image_url = 'https://www.jpl.nasa.gov/' + forurl
marsinfo["featured_image"]=featured_image_url
print(featured_image_url)
browser.quit()

### Mars Facts
url = 'https://space-facts.com/mars/'
tables = pd.read_html(url)
pprint(tables)
df = tables[0]
df.columns = ['Data', 'Information']
df
mars_facts = df.to_html('mars_facts.html')
marsinfo["facts"]=mars_facts
pprint(mars_facts)

### Mars Hemispheres
url= "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path)
browser.visit(url)
time.sleep(3)
hemisphere_urls = []

# Get a List of All the Hemispheres
urls = browser.find_by_css("a.product-item h3")
for item in range(len(urls)):
    hemisphere = {}
    browser.find_by_css("a.product-item h3")[item].click()
    hemisphere["title"] = browser.find_by_css("h2.title").text
    findelement = browser.links.find_by_text("Sample").first
    hemisphere["image_link"] = findelement["href"]
    hemisphere_urls.append(hemisphere)
    browser.back()
browser.quit()


import time, os, json, logging, random, tweepy
from threading import Thread
from dotenv import load_dotenv

load_dotenv()
logging.basicConfig(level = logging.INFO)
logger = logging.getLogger()

from integration import OeAvisaIntegration

class OeAvisaStreamListener(tweepy.StreamListener):
  def __init__(self, api, callback):
    self.api = api
    self.me = api.me()
    self.callback = callback

  def on_status(self, status):
    self.callback(status)

  def on_exception(self, exception):
    logger.error(f'TWITTER_STREAM_ERROR {exception}')
    return True

  def on_error(self, status_code):
    logger.error(f'TWITTER_STREAM_ERROR {status_code}')
    return True # reconnect

class TwitterIntegration(OeAvisaIntegration):
  NAME = 'twitter'

  TW_SCREEN_NAME = 'OeAvisa'
  TW_HANDLE = '@oeavisa'

  FOLLOW_ME_MESSAGE = " Sígueme pa' poder notificarte! ;)" # TODO: use emoji :D
  FOLLOW_ME_ON_HIT  = "Tengo pa' tí. Sígueme pa' poder notificarte! ;)"

  def __init__(self):
    logger.info('TWITTER Initializing...')
    super().__init__()
    self.api = self.create_api()
    tweepy.Stream(
      auth     = self.api.auth,
      listener = OeAvisaStreamListener(self.api, self.process_mention)
    ).filter(track = [self.TW_SCREEN_NAME], is_async = True)

    Thread(target = self.consume_direct_messages).start()

  def cleanup_message(self, message):
      message = message.lower().replace(self.TW_HANDLE, '')
      return super().cleanup_str(message)

  def process_tweet(self, tweet):
    clean_message = self.cleanup_message(tweet.text)
    logger.info(f'TWITTER_PROCESS_TWEET (clean): {clean_message}')

    response = self.CANNED_RESPONSES.get(clean_message)
    if response is None:
      self.register_query(clean_message, f'{tweet.id}:{tweet.user.id}:{tweet.user.screen_name}')
      response = random.sample(self.RESPONSES, 1)[0]

    return response

  def process_direct_message(self, message):
    clean_message = self.cleanup_message(message.message_create['message_data']['text'])

    sender_id = message.message_create['sender_id']
    response = self.CANNED_RESPONSES.get(clean_message)

    if response:
      try: self.api.send_direct_message(sender_id, response)
      except Exception as error: logger.error(f'TWITTER_PROCESS_DIRECT_MESSAGE_SEND {error}')
      return
    elif str(self.api.me().id) != str(sender_id):
      logger.info(f'TWITTER_PROCESS_DIRECT_MESSAGE (clean): {clean_message}')
      user = self.api.get_user(sender_id)

      if response is None: response = random.sample(self.RESPONSES, 1)[0]

      try: self.api.send_direct_message(sender_id, response)
      except Exception as error: logger.error(f'TWITTER_PROCESS_DIRECT_MESSAGE_SEND {error}')

      self.register_query(clean_message, f'{message.id}:{user.id}:{user.screen_name}')

  def process_mention(self, tweet):
    mentions = list( map(lambda x: x['screen_name'], tweet.entities['user_mentions']) )

    if len(mentions) == 1 and self.TW_SCREEN_NAME in mentions:
      logger.info(f'TWITTER_PROCESS_MENTION Found mention: {tweet.text}')

      if not tweet.user.following: tweet.user.follow() # follow user

      try:
        response = self.process_tweet(tweet)
        self.api.update_status( # respond to user
          status = f'@{tweet.user.screen_name} {response}',
          in_reply_to_status_id = tweet.id,
        )
      except tweepy.TweepError as error:
        if error.api_code == 187:
          logger.warning('TWITTER_PROCESS_MENTION Duplicate message.')
        else:
          raise error

  def process_hit(self, hit, subscriber):
    tweet_id, user_id, screen_name = subscriber.split(':')
    logger.info('TWITTER_PROCESS_HIT Handle: ' + subscriber)
    logger.info(f'TWITTER_PROCESS_HIT Response: {self.hit_response(hit)}')
    try:
      self.api.send_direct_message(user_id, self.hit_response(hit)) # respond to user with hit
      return True
    except tweepy.TweepError as error:
      if error.api_code != 226: # This request looks like it might be automated. To protect our users from spam...
        try:
          if not self.api.exists_friendship(user_id, self.api.me().id): # not follower?
            self.api.update_status( # respond to user asking to follow
              status = f"@{screen_name} {self.FOLLOW_ME_ON_HIT}",
              in_reply_to_status_id = tweet_id,
            )
        except Exception as error:
          logger.warning(f'TWITTER_PROCESS_HIT {error}')
      return False

  def consume_direct_messages(self):
    while True:
      logger.info('TWITTER_CONSUME_DIRECT_MESSAGE Starting...')

      messages = self.api.list_direct_messages()
      for message in messages:
        time.sleep(10)
        try: self.api.destroy_direct_message(message.id)
        except Exception as error: logger.error(f'TWITTER_PROCESS_DIRECT_MESSAGE_DESTROY {error}')
        time.sleep(10)
        try: self.process_direct_message(message)
        except Exception as error: logger.error(f'TWITTER_CONSUME_DIRECT_MESSAGE {error}')

      time.sleep(120)

  def create_api(self):
    consumer_key = os.getenv("CONSUMER_KEY")
    consumer_secret = os.getenv("CONSUMER_SECRET")
    access_token = os.getenv("ACCESS_TOKEN")
    access_token_secret = os.getenv("ACCESS_TOKEN_SECRET")

    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)

    api = tweepy.API(auth,
      wait_on_rate_limit        = True,
      wait_on_rate_limit_notify = True
    )

    try:
      api.verify_credentials()
    except Exception as e:
      logger.error("TWITTER_CREATE_API Error creating API", exc_info=True)
      raise e

    logger.info("TWITTER_CREATE_API Created!")

    return api


# -*- coding: utf_8 -*-
"""Module holding the functions for code analysis."""

import binascii
import hashlib
import logging
import os
import re

from androguard.core.bytecodes.apk import APK
from androguard.util import get_certificate_name_string

from asn1crypto import x509

from oscrypto import asymmetric

from django.utils.html import escape

logger = logging.getLogger(__name__)

def get_hardcoded_cert_keystore(files):
    """Returns the hardcoded certificate keystore."""
    try:
        logger.info('Getting Hardcoded Certificates/Keystores')
        findings = []
        certz = []
        key_store = []
        for file_name in files:
            if '.' not in file_name:
                continue
            ext = file_name.split('.')[-1]
            if re.search('cer|pem|cert|crt|pub|key|pfx|p12|der', ext):
                certz.append(escape(file_name))
            if re.search('jks|bks', ext):
                key_store.append(escape(file_name))
        if certz:
            desc = 'Certificate/Key files hardcoded inside the app.'
            findings.append({'finding': desc, 'files': certz})
        if key_store:
            desc = 'Hardcoded Keystore found.'
            findings.append({'finding': desc, 'files': key_store})
        return findings
    except Exception:
        logger.exception('Getting Hardcoded Certificates/Keystores')

def cert_info(app_dir, app_file):
    """Return certificate information."""
    try:
        logger.info('Reading Code Signing Certificate')
        manifestfile = None
        manidat = ''
        cert_info = ''
        certlist = []
        cert_path = os.path.join(app_dir, 'META-INF/')

        apk_file = os.path.join(app_dir, app_file)
        hashfunctions = {
            'md5': hashlib.md5,
            'sha1': hashlib.sha1,
            'sha256': hashlib.sha256,
            'sha512': hashlib.sha512,
        }
        files = [f for f in os.listdir(
            cert_path) if os.path.isfile(os.path.join(cert_path, f))]
        a = APK(apk_file)
        if a.is_signed():
            certlist.append('APK is signed')
        else:
            certlist.append('Missing certificate')
        certlist.append('v1 signature: {}'.format(a.is_signed_v1()))
        certlist.append('v2 signature: {}'.format(a.is_signed_v2()))
        certlist.append('v3 signature: {}'.format(a.is_signed_v3()))

        certs = set(a.get_certificates_der_v3() + a.get_certificates_der_v2()
                    + [a.get_certificate_der(x)
                       for x in a.get_signature_names()])
        pkeys = set(a.get_public_keys_der_v3() + a.get_public_keys_der_v2())

        if len(certs) > 0:
            certlist.append('Found {} unique certificates'.format(len(certs)))

        for cert in certs:
            x509_cert = x509.Certificate.load(cert)
            certlist.append('Subject: {}'.format(
                get_certificate_name_string(x509_cert.subject, short=True)))
            certlist.append('Signature Algorithm: {}'.format(
                x509_cert.signature_algo))
            certlist.append('Valid From: {}'.format(
                x509_cert['tbs_certificate']['validity']['not_before'].native))
            certlist.append('Valid To: {}'.format(
                x509_cert['tbs_certificate']['validity']['not_after'].native))
            certlist.append('Issuer: {}'.format(
                get_certificate_name_string(x509_cert.issuer, short=True)))
            certlist.append('Serial Number: {}'.format(
                hex(x509_cert.serial_number)))
            certlist.append('Hash Algorithm: {}'.format(x509_cert.hash_algo))
            for k, v in hashfunctions.items():
                certlist.append('{}: {}'.format(k, v(cert).hexdigest()))

        for public_key in pkeys:
            x509_public_key = asymmetric.load_public_key(public_key)
            certlist.append('PublicKey Algorithm: {}'.format(
                x509_public_key.algorithm))
            certlist.append('Bit Size: {}'.format(x509_public_key.bit_size))
            certlist.append('Fingerprint: {}'.format(
                binascii.hexlify(x509_public_key.fingerprint).decode('utf-8')))
        cert_info = '\n'.join(certlist)
        if 'MANIFEST.MF' in files:
            manifestfile = os.path.join(cert_path, 'MANIFEST.MF')
        if manifestfile:
            with open(manifestfile, 'r', encoding='utf-8') as manifile:
                manidat = manifile.read()
        sha256_digest = bool(re.findall(r'SHA-256-Digest', manidat))
        findings = []
        if a.is_signed():
            findings.append((
                'good',
                'Application is signed with a code '
                'signing certificate'))
        else:
            findings.append((
                'bad',
                'Code signing certificate not found'))
        if a.is_signed_v1():
            status = 'bad'
            if a.is_signed_v2() or a.is_signed_v3():
                status = 'warning'
            findings.append((
                status,
                'Application is signed with v1 signature scheme, '
                'making it vulnerable to Janus vulnerability on '
                'Android <7.0'))
        if re.findall(r'CN=Android Debug', cert_info):
            findings.append((
                'bad',
                'Application signed with a debug certificate. '
                'Production application must not be shipped '
                'with a debug certificate.'))
        if re.findall(r'Hash Algorithm: sha1', cert_info):
            status = 'bad'
            desc = (
                'Application is signed with SHA1withRSA. '
                'SHA1 hash algorithm is known to have '
                'collision issues.')
            if sha256_digest:
                status = 'warning'
                desc += (
                    ' The manifest file indicates SHA256withRSA'
                    ' is in use.')
            findings.append((status, desc))
        cert_dic = {
            'certificate_info': cert_info,
            'certificate_findings': findings,
        }
        return cert_dic
    except Exception:
        logger.exception('Reading Code Signing Certificate')
        return {}


'''
use pnn to predict the real values of states
using uncetainty
the input length of Bayes is 10 (6 is worse than 10 and 8)
For planning, the node pruning method is employed in the decision-making tree

'''
import argparse
import numpy as np
import fixed_env as env
import load_trace
import matplotlib.pyplot as plt
import itertools
# import bandwidth_pred as bp
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torch.utils.data as Data
from torch.autograd import Variable
# import test as bw_pre
# import mcts_abr_pnn as mcts
import pandas
from collections import Counter
# import lstm_net as net

S_INFO = 5  # bit_rate, buffer_size, rebuffering_time, bandwidth_measurement, chunk_til_video_end
S_LEN = 10  # take how many frames in the past
A_DIM = 6
MPC_FUTURE_CHUNK_COUNT = 3
SIMULATION_NUM = 8
ACTOR_LR_RATE = 0.0001
CRITIC_LR_RATE = 0.001
VIDEO_BIT_RATE = [300,750,1200,1850,2850,4300]  # Kbps
BITRATE_REWARD = [1, 2, 3, 12, 15, 20]
BUFFER_NORM_FACTOR = 10.0
CHUNK_TIL_VIDEO_END_CAP = 48.0
TOTAL_VIDEO_CHUNKS = 48
M_IN_K = 1000.0
# REBUF_PENALTY = 4.3  # 1 sec rebuffering -> 3 Mbps
SMOOTH_PENALTY = 1
DEFAULT_QUALITY = 1  # default video quality without agent
RANDOM_SEED = 42
RAND_RANGE = 1000000

VIDEO_SIZE_FILE = './video_size/ori/video_size_'
# log in format of time_stamp bit_rate buffer_size rebuffer_time chunk_size download_time reward
# NN_MODEL = './models/nn_model_ep_5900.ckpt'

TIME_STEP = 8
# TARGET_SIZE = 5
INPUT_SIZE = 1
HIDDEN_SIZE = 128

BATCH_SIZE = 1
LR = 0.001
EPOCH = 20

CHUNK_COMBO_OPTIONS = []

parser = argparse.ArgumentParser(description='BayesMPC')
parser.add_argument('--lin', action='store_true', help='QoE_lin metric')
parser.add_argument('--log', action='store_true', help='QoE_log metric')
parser.add_argument('--FCC', action='store_true', help='Test in FCC dataset')
parser.add_argument('--HSDPA', action='store_true', help='Test in HSDPA dataset')
parser.add_argument('--Oboe', action='store_true', help='Test in Oboe dataset')
parser.add_argument('--cb', action='store_true', help='Compare the lower bound')

# past errors in bandwidth
past_errors = []
past_bandwidth_ests = []

#size_video1 = [3155849, 2641256, 2410258, 2956927, 2593984, 2387850, 2554662, 2964172, 2541127, 2553367, 2641109, 2876576, 2493400, 2872793, 2304791, 2855882, 2887892, 2474922, 2828949, 2510656, 2544304, 2640123, 2737436, 2559198, 2628069, 2626736, 2809466, 2334075, 2775360, 2910246, 2486226, 2721821, 2481034, 3049381, 2589002, 2551718, 2396078, 2869088, 2589488, 2596763, 2462482, 2755802, 2673179, 2846248, 2644274, 2760316, 2310848, 2647013, 1653424]
# size_video1 = [2354772, 2123065, 2177073, 2160877, 2233056, 1941625, 2157535, 2290172, 2055469, 2169201, 2173522, 2102452, 2209463, 2275376, 2005399, 2152483, 2289689, 2059512, 2220726, 2156729, 2039773, 2176469, 2221506, 2044075, 2186790, 2105231, 2395588, 1972048, 2134614, 2164140, 2113193, 2147852, 2191074, 2286761, 2307787, 2143948, 1919781, 2147467, 2133870, 2146120, 2108491, 2184571, 2121928, 2219102, 2124950, 2246506, 1961140, 2155012, 1433658]
# size_video2 = [1728879, 1431809, 1300868, 1520281, 1472558, 1224260, 1388403, 1638769, 1348011, 1429765, 1354548, 1519951, 1422919, 1578343, 1231445, 1471065, 1491626, 1358801, 1537156, 1336050, 1415116, 1468126, 1505760, 1323990, 1383735, 1480464, 1547572, 1141971, 1498470, 1561263, 1341201, 1497683, 1358081, 1587293, 1492672, 1439896, 1139291, 1499009, 1427478, 1402287, 1339500, 1527299, 1343002, 1587250, 1464921, 1483527, 1231456, 1364537, 889412]
# size_video3 = [1034108, 957685, 877771, 933276, 996749, 801058, 905515, 1060487, 852833, 913888, 939819, 917428, 946851, 1036454, 821631, 923170, 966699, 885714, 987708, 923755, 891604, 955231, 968026, 874175, 897976, 905935, 1076599, 758197, 972798, 975811, 873429, 954453, 885062, 1035329, 1026056, 943942, 728962, 938587, 908665, 930577, 858450, 1025005, 886255, 973972, 958994, 982064, 830730, 846370, 598850]
# size_video4 = [668286, 611087, 571051, 617681, 652874, 520315, 561791, 709534, 584846, 560821, 607410, 594078, 624282, 687371, 526950, 587876, 617242, 581493, 639204, 586839, 601738, 616206, 656471, 536667, 587236, 590335, 696376, 487160, 622896, 641447, 570392, 620283, 584349, 670129, 690253, 598727, 487812, 575591, 605884, 587506, 566904, 641452, 599477, 634861, 630203, 638661, 538612, 550906, 391450]
# size_video5 = [450283, 398865, 350812, 382355, 411561, 318564, 352642, 437162, 374758, 362795, 353220, 405134, 386351, 434409, 337059, 366214, 360831, 372963, 405596, 350713, 386472, 399894, 401853, 343800, 359903, 379700, 425781, 277716, 400396, 400508, 358218, 400322, 369834, 412837, 401088, 365161, 321064, 361565, 378327, 390680, 345516, 384505, 372093, 438281, 398987, 393804, 331053, 314107, 255954]
# size_video6 = [181801, 155580, 139857, 155432, 163442, 126289, 153295, 173849, 150710, 139105, 141840, 156148, 160746, 179801, 140051, 138313, 143509, 150616, 165384, 140881, 157671, 157812, 163927, 137654, 146754, 153938, 181901, 111155, 153605, 149029, 157421, 157488, 143881, 163444, 179328, 159914, 131610, 124011, 144254, 149991, 147968, 161857, 145210, 172312, 167025, 160064, 137507, 118421, 112270]

# def create_DataTensor(dataset, window):
#     # Input: dataset, flattened numpy array
#     # Output: Tensor_x [batch, time_step, features]
#     #         Tensor_y [batch, target_size, features]
#     # Each dataset has one output
#     data_x, data_y = [], []
#     for i in range(window, len(dataset)):
#         batch_x = dataset[i - window: i]
#         batch_y = dataset[i]
#         data_x.append(batch_x[:, np.newaxis])
#         data_y.append(np.array([[batch_y]]))

#     data_x, data_y = np.asarray(data_x), np.asarray(data_y)

#     Tensor_x = torch.from_numpy(data_x)
#     Tensor_y = torch.from_numpy(data_y)
#     return Tensor_x, Tensor_y

# torch.cuda.device(0)
# torch.cuda.get_device_name(torch.cuda.current_device())

def to_variable(var=(), cuda=True, volatile=False):
    out = []
    for v in var:

        if isinstance(v, np.ndarray):
            v = torch.from_numpy(v).type(torch.FloatTensor)

        if not v.is_cuda and cuda:
            v = v.cuda()

        if not isinstance(v, Variable):
            v = Variable(v, volatile=volatile)

        out.append(v)
    return out

def log_gaussian_loss(output, target, sigma, no_dim, sum_reduce=True):
    exponent = -0.5*(target - output)**2/sigma**2
    log_coeff = -no_dim*torch.log(sigma) - 0.5*no_dim*np.log(2*np.pi)

    if sum_reduce:
        return -(log_coeff + exponent).sum()
    else:
        return -(log_coeff + exponent)

def get_kl_divergence(weights, prior, varpost):
    prior_loglik = prior.loglik(weights)

    varpost_loglik = varpost.loglik(weights)
    varpost_lik = varpost_loglik.exp()

    return (varpost_lik*(varpost_loglik - prior_loglik)).sum()

class gaussian:
    def __init__(self, mu, sigma):
        self.mu = mu
        self.sigma = sigma

    def loglik(self, weights): # log Guassian value
        exponent = -0.5*(weights - self.mu)**2/self.sigma**2
        log_coeff = -0.5*(np.log(2*np.pi) + 2*np.log(self.sigma))

        return (exponent + log_coeff).sum()

class BayesLinear_Normalq(nn.Module):
    def __init__(self, input_dim, output_dim, prior):
        super(BayesLinear_Normalq, self).__init__()
        self.input_dim = input_dim
        self.output_dim = output_dim
        self.prior = prior

        # scale = (2/self.input_dim)**0.5
        # rho_init = np.log(np.exp((2/self.input_dim)**0.5) - 1)
        self.weight_mus = nn.Parameter(torch.Tensor(self.input_dim, self.output_dim).uniform_(-0.01, 0.01))
        self.weight_rhos = nn.Parameter(torch.Tensor(self.input_dim, self.output_dim).uniform_(-3, -3))

        self.bias_mus = nn.Parameter(torch.Tensor(self.output_dim).uniform_(-0.01, 0.01))
        self.bias_rhos = nn.Parameter(torch.Tensor(self.output_dim).uniform_(-4, -3))

    def forward(self, x, sample = True):

        if sample:
            # sample gaussian noise for each weight and each bias
            weight_epsilons = Variable(self.weight_mus.data.new(self.weight_mus.size()).normal_())
            bias_epsilons =  Variable(self.bias_mus.data.new(self.bias_mus.size()).normal_())

            # calculate the weight and bias stds from the rho parameters
            weight_stds = torch.log(1 + torch.exp(self.weight_rhos))
            bias_stds = torch.log(1 + torch.exp(self.bias_rhos))

            # calculate samples from the posterior from the sampled noise and mus/stds
            weight_sample = self.weight_mus + weight_epsilons*weight_stds
            bias_sample = self.bias_mus + bias_epsilons*bias_stds

            output = torch.mm(x, weight_sample) + bias_sample ## Performs a matrix multiplication of the matrices input and mat2.

            # computing the KL loss term
            prior_cov, varpost_cov = self.prior.sigma**2, weight_stds**2
            KL_loss = 0.5*(torch.log(prior_cov/varpost_cov)).sum() - 0.5*weight_stds.numel()
            KL_loss = KL_loss + 0.5*(varpost_cov/prior_cov).sum()
            KL_loss = KL_loss + 0.5*((self.weight_mus - self.prior.mu)**2/prior_cov).sum()

            prior_cov, varpost_cov = self.prior.sigma**2, bias_stds**2
            KL_loss = KL_loss + 0.5*(torch.log(prior_cov/varpost_cov)).sum() - 0.5*bias_stds.numel()
            KL_loss = KL_loss + 0.5*(varpost_cov/prior_cov).sum()
            KL_loss = KL_loss + 0.5*((self.bias_mus - self.prior.mu)**2/prior_cov).sum()

            return output, KL_loss

        else:
            output = torch.mm(x, self.weight_mus) + self.bias_mus
            return output, KL_loss

    def sample_layer(self, no_samples):
        all_samples = []
        for i in range(no_samples):
            # sample gaussian noise for each weight and each bias
            weight_epsilons = Variable(self.weight_mus.data.new(self.weight_mus.size()).normal_())

            # calculate the weight and bias stds from the rho parameters
            weight_stds = torch.log(1 + torch.exp(self.weight_rhos))

            # calculate samples from the posterior from the sampled noise and mus/stds
            weight_sample = self.weight_mus + weight_epsilons*weight_stds

            all_samples += weight_sample.view(-1).cpu().data.numpy().tolist()

        return all_samples

class BBP_Heteroscedastic_Model_Wrapper:
    def __init__(self, network, learn_rate, batch_size, no_batches):

        self.learn_rate = learn_rate
        self.batch_size = batch_size
        self.no_batches = no_batches

        self.network = network
        self.network.cuda()

        self.optimizer = torch.optim.Adam(self.network.parameters(), lr = self.learn_rate)
        self.loss_func = log_gaussian_loss

    def fit(self, x, y, no_samples):
        x, y = to_variable(var=(x, y), cuda=True)

        # reset gradient and total loss
        self.optimizer.zero_grad()
        fit_loss_total = 0

        for i in range(no_samples):
            output, KL_loss_total = self.network(x)

            # calculate fit loss based on mean and standard deviation of output
            fit_loss = self.loss_func(output[:, :1], y, output[:, 1:].exp(), 1) ## sigma.exp() make sure the sigma > 0
            fit_loss_total = fit_loss_total + fit_loss

        KL_loss_total = KL_loss_total/self.no_batches
        total_loss = (fit_loss_total + KL_loss_total)/(no_samples*x.shape[0])
        total_loss.backward()
        self.optimizer.step()

        return fit_loss_total/no_samples, KL_loss_total

    def get_loss_and_rmse(self, x, y, no_samples):
        x, y = to_variable(var=(x, y), cuda=True)

        means, stds = [], []
        for i in range(no_samples):
            output, KL_loss_total = self.network(x)
            means.append(output[:, :1, None])
            stds.append(output[:, 1:, None].exp())

        means, stds = torch.cat(means, 2), torch.cat(stds, 2)
        mean = means.mean(dim=2)
        std = (means.var(dim=2) + stds.mean(dim=2)**2)**0.5

        # calculate fit loss based on mean and standard deviation of output
        logliks = self.loss_func(output[:, :1], y, output[:, 1:].exp(), 1, sum_reduce=False)
        rmse = float((((mean - y)**2).mean()**0.5).cpu().data)

        return logliks, rmse

class BBP_Heteroscedastic_Model_UCI(nn.Module):
    def __init__(self, input_dim, output_dim, num_units):
        super(BBP_Heteroscedastic_Model_UCI, self).__init__()

        self.input_dim = input_dim
        self.output_dim = output_dim

        # network with two hidden and one output layer
        self.layer1 = BayesLinear_Normalq(input_dim, num_units, gaussian(0, 1))
        self.layer2 = BayesLinear_Normalq(num_units, num_units, gaussian(0, 1))
        self.layer3 = BayesLinear_Normalq(num_units, 2*output_dim, gaussian(0, 1))

        # activation to be used between hidden layers
        self.activation = nn.ReLU(inplace = True)

    def forward(self, x):

        KL_loss_total = 0
        x = x.view(-1, self.input_dim)

        x, KL_loss = self.layer1(x)
        KL_loss_total = KL_loss_total + KL_loss
        x = self.activation(x)

        x, KL_loss = self.layer2(x)
        KL_loss_total = KL_loss_total + KL_loss
        x = self.activation(x)

        x, KL_loss = self.layer3(x)
        KL_loss_total = KL_loss_total + KL_loss

        return x, KL_loss_total

def throughput_pre(data, model, horizon):
    # best_net = torch.load("./throughput_v0.pkl")
    best_net = model
    H_step = horizon
    in_dim = 10
    x_test = data
    throughput_mean = []
    throughput_std = []
    x, y = to_variable(var=(x_test, x_test), cuda=True)

    for step in range(H_step):
        means, stds = [], []
        no_samples = SIMULATION_NUM
        for i in range(no_samples):
            output, KL_loss_total = best_net(x)
            means.append(output[:, :1, None])
            stds.append(output[:, 1:, None].exp())

        means, stds = torch.cat(means, 2), torch.cat(stds, 2)
        mean = means.mean(dim=2)
        std = (means.var(dim=2) + stds.mean(dim=2)**2)**0.5
        # std = stds.mean(dim=2)
        throughput_mean.append(mean.cpu().detach().numpy()[0].tolist()[0])
        throughput_std.append(std.cpu().detach().numpy()[0].tolist()[0])

        x_test = np.roll(x_test, -1, axis=0)
        x_test[-1] = mean.cpu().detach().numpy()
        x, y = to_variable(var=(x_test, x_test), cuda=True)

    return throughput_mean, throughput_std

# def get_chunk_size(quality, index):
#     if ( index < 0 or index > 48 ):
#         return 0
#     # note that the quality and video labels are inverted (i.e., quality 4 is highest and this pertains to video1)
#     sizes = {5: size_video1[index], 4: size_video2[index], 3: size_video3[index], 2: size_video4[index], 1: size_video5[index], 0:size_video6[index]}
#     return sizes[quality]
class video_size(object):
    def __init__(self):
        self.video_sizes = {}

    def store_size(self):
        for bitrate in range(A_DIM):
            self.video_sizes[bitrate] = []
            with open(VIDEO_SIZE_FILE + str(bitrate)) as f:
                for line in f:
                    self.video_sizes[bitrate].append(int(line.split()[0]))

    def get_chunk_size(self, quality, index):
        if ( index < 0 or index > 47 ):
            return 0
        # note that the quality and video labels are inverted (i.e., quality 4 is highest and this pertains to video1)
        # sizes = {5: size_video1[index], 4: size_video2[index], 3: size_video3[index], 2: size_video4[index], 1: size_video5[index], 0:size_video6[index]}
        return self.video_sizes[quality][index]

def main():
    args = parser.parse_args()
    if args.cb or args.lin:
        qoe_metric = 'results_lin'
    elif args.log:
        qoe_metric = 'results_log'
    else:
        print('Please select the QoE Metric!')

    if args.FCC:
        dataset = 'fcc'
    elif args.HSDPA:
        dataset = 'HSDPA'
    elif args.Oboe:
        dataset = 'Oboe'
    else:
        print('Please select the dataset!')

    dataset_path = './traces_' + dataset + '/'
    if args.cb:
        Log_file_path = './' + qoe_metric + '/cb_' + dataset + '/log_sim_bayesmpc'
    else:
        Log_file_path = './' + qoe_metric + '/' + dataset + '/log_sim_bayesmpc'

    np.random.seed(RANDOM_SEED)

    assert len(VIDEO_BIT_RATE) == A_DIM

    all_cooked_time, all_cooked_bw, all_file_names = load_trace.load_trace(dataset_path)

    net_env = env.Environment(all_cooked_time=all_cooked_time,
                              all_cooked_bw=all_cooked_bw)

    log_path = Log_file_path + '_' + all_file_names[net_env.trace_idx]
    log_file = open(log_path, 'w')

    model = torch.load("./throughput_v0.pkl")

    chunk_size_info = video_size()
    chunk_size_info.store_size()

    time_stamp = 0

    last_bit_rate = DEFAULT_QUALITY
    bit_rate = DEFAULT_QUALITY

    action_vec = np.zeros(A_DIM)
    action_vec[bit_rate] = 1

    s_batch = [np.zeros((S_INFO, S_LEN))]
    a_batch = [action_vec]
    r_batch = []
    future_bandwidth = [0]
    future_bandwidth_r = [0]
    predicted_mean = [0]
    predicted_unc = [0]
    # entropy_record = []

    video_count = 0

    # make chunk combination options
    # for combo in itertools.product([0,1,2,3,4,5], repeat=MPC_FUTURE_CHUNK_COUNT):
    #     CHUNK_COMBO_OPTIONS.append(combo)

    while True:  # serve video forever
        # the action is from the last decision
        # this is to make the framework similar to the real
        delay, sleep_time, buffer_size, rebuf, \
        video_chunk_size, _,\
        end_of_video, video_chunk_remain = \
            net_env.get_video_chunk(bit_rate)

        time_stamp += delay  # in ms
        time_stamp += sleep_time  # in ms

        # reward is video quality - rebuffer penalty
        if qoe_metric == 'results_lin':
            REBUF_PENALTY = 4.3
            reward = VIDEO_BIT_RATE[bit_rate] / M_IN_K \
                    - REBUF_PENALTY * rebuf \
                    - SMOOTH_PENALTY * np.abs(VIDEO_BIT_RATE[bit_rate] -
                                            VIDEO_BIT_RATE[last_bit_rate]) / M_IN_K
        else:# log scale reward
            REBUF_PENALTY = 2.66
            log_bit_rate = np.log(VIDEO_BIT_RATE[bit_rate] / float(VIDEO_BIT_RATE[0]))
            log_last_bit_rate = np.log(VIDEO_BIT_RATE[last_bit_rate] / float(VIDEO_BIT_RATE[0]))

            reward = log_bit_rate \
                    - REBUF_PENALTY * rebuf \
                    - SMOOTH_PENALTY * np.abs(log_bit_rate - log_last_bit_rate)

        # reward = BITRATE_REWARD[bit_rate] \
        #          - 8 * rebuf - np.abs(BITRATE_REWARD[bit_rate] - BITRATE_REWARD[last_bit_rate])

        r_batch.append(reward)

        last_bit_rate = bit_rate

        # log time_stamp, bit_rate, buffer_size, reward
        log_file.write(str(time_stamp / M_IN_K) + '\t' +
                       str(VIDEO_BIT_RATE[bit_rate]) + '\t' +
                       str(buffer_size) + '\t' +
                       str(rebuf) + '\t' +
                       str(video_chunk_size) + '\t' +
                       str(delay) + '\t' +
                       str(reward) + '\t' +
                       str(predicted_mean[0]/8) + '\t' + str(predicted_unc[0]/8) + '\t' +
                       str(future_bandwidth_r[0]/8) + '\t' +
                       str(float(video_chunk_size) / float(delay) / M_IN_K) + '\n')
        log_file.flush()

        # retrieve previous state
        if len(s_batch) == 0:
            state = [np.zeros((S_INFO, S_LEN))]
        else:
            state = np.array(s_batch[-1], copy=True)

        # dequeue history record
        state = np.roll(state, -1, axis=1)

        # this should be S_INFO number of terms
        state[0, -1] = VIDEO_BIT_RATE[bit_rate] / float(np.max(VIDEO_BIT_RATE))  # last quality
        state[1, -1] = buffer_size / BUFFER_NORM_FACTOR
        state[2, -1] = rebuf
        state[3, -1] = float(video_chunk_size) / float(delay) / M_IN_K * 8# kilo bits / ms
        state[4, -1] = np.minimum(video_chunk_remain, CHUNK_TIL_VIDEO_END_CAP) / float(CHUNK_TIL_VIDEO_END_CAP)
        # state[5: 10, :] = future_chunk_sizes / M_IN_K / M_IN_K

        # ================== MPC =========================
        curr_error = 0 # defualt assumes that this is the first request so error is 0 since we have never predicted bandwidth
        # curr_error_for_net = 0
        if ( len(past_bandwidth_ests) > 0 ):
            curr_error  = abs(past_bandwidth_ests[-1]-state[3,-1])/float(state[3,-1])
        # curr_error_for_net = past_bandwidth_ests[-1]-state[3,-1]
        past_errors.append(curr_error)

        # pick bitrate according to MPC
        # first get harmonic mean of last 5 bandwidths
        # past_bandwidths = [0 for i in range(10)]
        past_bandwidths = state[3,-S_LEN:]

        # simulation for many times
        est_current = []
        predicted_mean, predicted_unc = throughput_pre(past_bandwidths, model, MPC_FUTURE_CHUNK_COUNT)
        future_bandwidth = predicted_mean
        # last_bandwidth = state[3, -1]
        # for i in range(MPC_FUTURE_CHUNK_COUNT):
        #     future_bandwidth.append(last_bandwidth + prediction_output[i])
        #     last_bandwidth = last_bandwidth + prediction_output[i]
        # while past_bandwidths[0] == 0.0:
        #     past_bandwidths = past_bandwidths[1:]
        # #if ( len(state) < 5 ):
        # #    past_bandwidths = state[3,-len(state):]
        # #else:
        # #    past_bandwidths = state[3,-5:]
        # bandwidth_sum = 0
        # for past_val in past_bandwidths:
        #     bandwidth_sum += (1/float(past_val))
        # harmonic_bandwidth = 1.0/(bandwidth_sum/len(past_bandwidths))

        # future bandwidth prediction
        # divide by 1 + max of last 5 (or up to 5) errors
        max_error = 0
        error_pos = -5
        if ( len(past_errors) < 5 ):
            error_pos = -len(past_errors)
        max_error = float(np.max(past_errors[error_pos:]))
        future_bandwidth_r = []
        for i in range(MPC_FUTURE_CHUNK_COUNT): #MPC_FUTURE_CHUNK_COUNT
            # future_bandwidth_r.append(future_bandwidth[i]/(1+max_error))  # robustMPC here
            future_bandwidth_r.append(max(future_bandwidth[i] - 1.2*predicted_unc[i], 0.01))  # robustMPC hered
            # future_bandwidth_r.append(max(future_bandwidth[i] - (2.2-np.log10(buffer_size))*predicted_unc[i], 0.001))
        est_current.append(future_bandwidth[0])
        past_bandwidth_ests.append(future_bandwidth[0])

        # future chunks length (try 4 if that many remaining)
        last_index = int(CHUNK_TIL_VIDEO_END_CAP - video_chunk_remain - 1)
        future_chunk_length = MPC_FUTURE_CHUNK_COUNT
        if ( TOTAL_VIDEO_CHUNKS - last_index < MPC_FUTURE_CHUNK_COUNT ):
            future_chunk_length = TOTAL_VIDEO_CHUNKS - last_index

        # all possible combinations of 5 chunk bitrates (9^5 options)
        # iterate over list and for each, compute reward and store max reward combination
        max_reward = -100000000
        # best_combo = ()
        start_buffer = buffer_size
        #start = time.time()
        download_time_every_step = []
        for position in range(future_chunk_length):
            download_time_current = []
            for action in range(0, A_DIM):
                index = last_index + position + 1 # e.g., if last chunk is 3, then first iter is 3+0+1=4
                download_time = (chunk_size_info.get_chunk_size(action, index)/1000000.)/future_bandwidth_r[0] * 8 # this is MB/MB/s --> seconds
                download_time_current.append(download_time)
            download_time_every_step.append(download_time_current)

        reward_comparison = False
        send_data = 0
        parents_pool = [[0.0, start_buffer, int(bit_rate)]]
        for position in range(future_chunk_length):
            if position == future_chunk_length-1:
                reward_comparison = True
            children_pool = []
            for parent in parents_pool:
                action = 0
                curr_buffer = parent[1]
                last_quality = parent[-1]
                curr_rebuffer_time = 0
                chunk_quality = action
                download_time = download_time_every_step[position][chunk_quality]
                if ( curr_buffer < download_time ):
                    curr_rebuffer_time += (download_time - curr_buffer)
                    curr_buffer = 0.0
                else:
                    curr_buffer -= download_time
                curr_buffer += 4

                # reward
                bitrate_sum = VIDEO_BIT_RATE[chunk_quality]
                smoothness_diffs = abs(VIDEO_BIT_RATE[chunk_quality] - VIDEO_BIT_RATE[last_quality])
                reward = (bitrate_sum/1000.) - (REBUF_PENALTY*curr_rebuffer_time) - (SMOOTH_PENALTY*smoothness_diffs/1000.)
                reward += parent[0]

                children = parent[:]
                children[0] = reward
                children[1] = curr_buffer
                children.append(action)
                children_pool.append(children)
                if (reward >= max_reward) and reward_comparison:
                    if send_data > children[3] and reward == max_reward:
                        send_data = send_data
                    else:
                        send_data = children[3]
                    max_reward = reward

                # criterion terms
                # theta = SMOOTH_PENALTY * (VIDEO_BIT_RATE[action+1]/1000. - VIDEO_BIT_RATE[action]/1000.)
                rebuffer_term = REBUF_PENALTY * (max(download_time_every_step[position][action+1] - parent[1], 0) - max(download_time_every_step[position][action] - parent[1], 0))
                if (action + 1 <= parent[-1]):
                    High_Maybe_Superior = ((1.0 + 2 * SMOOTH_PENALTY)*(VIDEO_BIT_RATE[action]/1000. - VIDEO_BIT_RATE[action+1]/1000.) + rebuffer_term < 0.0)
                else:
                    High_Maybe_Superior = ((VIDEO_BIT_RATE[action]/1000. - VIDEO_BIT_RATE[action+1]/1000.) + rebuffer_term < 0.0)

                # while REBUF_PENALTY*(download_time_every_step[position][action+1] - parent[1]) <= ((VIDEO_BIT_RATE[action+1]/1000. - VIDEO_BIT_RATE[action]/1000.)-(abs(VIDEO_BIT_RATE[action+1] - VIDEO_BIT_RATE[parent[-1]]) - abs(VIDEO_BIT_RATE[action] - VIDEO_BIT_RATE[parent[-1]]))/1000.):
                while High_Maybe_Superior:
                    curr_buffer = parent[1]
                    last_quality = parent[-1]
                    curr_rebuffer_time = 0
                    chunk_quality = action + 1
                    download_time = download_time_every_step[position][chunk_quality]
                    if ( curr_buffer < download_time ):
                        curr_rebuffer_time += (download_time - curr_buffer)
                        curr_buffer = 0
                    else:
                        curr_buffer -= download_time
                    curr_buffer += 4

                    # reward
                    bitrate_sum = VIDEO_BIT_RATE[chunk_quality]
                    smoothness_diffs = abs(VIDEO_BIT_RATE[chunk_quality] - VIDEO_BIT_RATE[last_quality])
                    reward = (bitrate_sum/1000.) - (REBUF_PENALTY*curr_rebuffer_time) - (SMOOTH_PENALTY*smoothness_diffs/1000.)
                    reward += parent[0]

                    children = parent[:]
                    children[0] = reward
                    children[1] = curr_buffer
                    children.append(chunk_quality)
                    children_pool.append(children)
                    if (reward >= max_reward) and reward_comparison:
                        if send_data > children[3] and reward == max_reward:
                            send_data = send_data
                        else:
                            send_data = children[3]
                        max_reward = reward

                    action += 1
                    if action + 1 == A_DIM:
                        break
                    # criterion terms
                    # theta = SMOOTH_PENALTY * (VIDEO_BIT_RATE[action+1]/1000. - VIDEO_BIT_RATE[action]/1000.)
                    rebuffer_term = REBUF_PENALTY * (max(download_time_every_step[position][action+1] - parent[1], 0) - max(download_time_every_step[position][action] - parent[1], 0))
                    if (action + 1 <= parent[-1]):
                        High_Maybe_Superior = ((1.0 + 2 * SMOOTH_PENALTY)*(VIDEO_BIT_RATE[action]/1000. - VIDEO_BIT_RATE[action+1]/1000.) + rebuffer_term < 0)
                    else:
                        High_Maybe_Superior = ((VIDEO_BIT_RATE[action]/1000. - VIDEO_BIT_RATE[action+1]/1000.) + rebuffer_term < 0)

            parents_pool = children_pool
        bit_rate = send_data
        # for item in L:
        #     if L[item] > max_time:
        #         max_time = L[item]
        #         bit_rate = item
        #     elif L[item] == max_time:
        #         if bit_rate < item:
        #             bit_rate = item
        # bit_rate = send_data
        # hack
        # if bit_rate == 1 or bit_rate == 2:s
        #    bit_rate = 0

        # ================================================

        # Note: we need to discretize the probability into 1/RAND_RANGE steps,
        # because there is an intrinsic discrepancy in passing single state and batch states

        s_batch.append(state)
        past_bandwidth_ests.append(np.mean(est_current))

        if end_of_video:
            log_file.write('\n')
            log_file.close()

            last_bit_rate = DEFAULT_QUALITY
            bit_rate = DEFAULT_QUALITY  # use the default action here

            del s_batch[:]
            del a_batch[:]
            del r_batch[:]
            del past_bandwidth_ests[:]

            time_stamp = 0

            action_vec = np.zeros(A_DIM)
            action_vec[bit_rate] = 1

            s_batch.append(np.zeros((S_INFO, S_LEN)))
            a_batch.append(action_vec)
            entropy_record = []
            # past_errors_for_net = []

            print("video count", video_count)
            video_count += 1

            if video_count >= len(all_file_names):
                break

            log_path = Log_file_path + '_' + all_file_names[net_env.trace_idx]
            log_file = open(log_path, 'w')

if __name__ == '__main__':
    main()


from typing import List
import os
import yaml
import h5py
import shutil
import numpy as np
import zarr
from scipy.io import wavfile
from matplotlib.pyplot import specgram
import pickle
from .init import _find_singular_file_in_dir
from ._project_config import _set_session_config_value

def create_spectrograms(session: str):
    dirname = f'./{session}'
    with open(f'{dirname}/isa-session.yaml', 'r') as f:
        config = yaml.safe_load(f)
    print('USING CONFIG')
    print(config)

    spectrograms_pkl_fname = f'{dirname}/spectrograms.pkl'
    spectrogram_for_gui_zarr_fname = f'{dirname}/spectrogram_for_gui.zarr'

    duration_sec = config['audio_duration_sec']
    audio_sr_hz = config['audio_sr_hz']

    # freq_range=[130, 230]

    # # between 0 and 100... percentile cutoff in the target freq band
    # # in spectrogram_for_gui
    # # everything below the value is set to zero
    # # affects detection and compressed file size
    # threshold_pct = 99.8

    audio_fname = _find_singular_file_in_dir(dirname, ['.h5', '.wav'])
    if audio_fname is None:
        raise Exception(f'No .h5 or .wav file found in directory: {dirname}')
    print(f'USING AUDIO: {audio_fname}')
    audio_path = f'{dirname}/{audio_fname}'

    print('Extracting audio signals')
    if audio_path.endswith('.h5'):
        with h5py.File(audio_path, 'r') as f:
            ch1 = np.array(f['ai_channels/ai0'])
            ch2 = np.array(f['ai_channels/ai1'])
            ch3 = np.array(f['ai_channels/ai2'])
            ch4 = np.array(f['ai_channels/ai3'])
            X = np.stack([ch1, ch2, ch3, ch4]).T

            # crop to duration
            X = X[0:int(duration_sec * audio_sr_hz)]
    elif audio_path.endswith('.wav'):
        sr, audio = wavfile.read(audio_path)
        # n_samples = audio.shape[0]
        # n_channels = audio.shape[1]
        # crop to duration
        X = audio[0:int(duration_sec * audio_sr_hz)]
    else:
        raise Exception(f'Unknown audio file type: {audio_path}')

    num_channels = X.shape[1]

    print('Computing spectrograms')
    spectrograms = []
    for channel_ind in range(num_channels):
        s, spectrogram_frequencies, spectrogram_times, im = specgram(X[:, channel_ind], NFFT=512, noverlap=256, Fs=audio_sr_hz)
        sr_spectrogram = float(1 / (spectrogram_times[1] - spectrogram_times[0]))
        spectrograms.append(s.T) # Use transpose so we have Nt x Nf
    print(f'Spectrogram sampling rate (Hz): {sr_spectrogram}')
    _set_session_config_value(session, 'spectrogram_sr_hz', sr_spectrogram)
    _set_session_config_value(session, 'spectrogram_num_frequencies', len(spectrogram_frequencies))
    _set_session_config_value(session, 'spectrogram_df', float(spectrogram_frequencies[1] - spectrogram_frequencies[0]))
    spectrogram_for_gui = sum(spectrograms)

    print('Auto detecting maxval')
    maxval = _auto_detect_spectrogram_maxval(spectrogram_for_gui, sr_spectrogram=sr_spectrogram)
    minval = 0
    print(f'Absolute spectrogram max: {np.max(spectrogram_for_gui)}')
    print(f'Auto detected spectrogram max: {maxval}')

    print('Scaling spectogram data')
    # Nf x Nt
    spectrogram_for_gui: np.ndarray = np.floor((spectrogram_for_gui - minval) / (maxval - minval) * 255).astype(np.uint8)

    # threshold = np.percentile(spectrogram_for_gui[freq_range[0]:freq_range[1]], threshold_pct)
    # print(f'Using threshold: {threshold} ({threshold_pct} pct)')
    # spectrogram_for_gui[spectrogram_for_gui <= threshold] = 0

    if os.path.exists(spectrogram_for_gui_zarr_fname):
        shutil.rmtree(spectrogram_for_gui_zarr_fname)

    if os.path.exists(spectrograms_pkl_fname):
        os.remove(spectrograms_pkl_fname)

    print(f'Writing {spectrograms_pkl_fname}')
    with open(spectrograms_pkl_fname, 'wb') as fb:
        pickle.dump({
            'spectrograms': spectrograms,
            'frequencies': spectrogram_frequencies,
            'times': spectrogram_times,
            'spectrogram_for_gui': spectrogram_for_gui
        }, fb)

    print(f'Writing {spectrogram_for_gui_zarr_fname}')
    root_group = zarr.open(spectrogram_for_gui_zarr_fname, mode="w")
    root_group.create_dataset("spectrogram", data=spectrogram_for_gui, chunks=(10000, len(spectrogram_frequencies)))

    ds_factor = 3
    previous_spectrogram = spectrogram_for_gui
    while ds_factor < len(spectrogram_times):
        spectrogram_for_gui_downsampled = downsample_spectrogram_using_max(previous_spectrogram, ds_factor=3)
        root_group.create_dataset(f"spectrogram_ds{ds_factor}", data=spectrogram_for_gui_downsampled, chunks=(10000, len(spectrogram_frequencies)))
        previous_spectrogram = spectrogram_for_gui_downsampled
        ds_factor *= 3

    root_group.attrs['spectrogram_sr_hz'] = sr_spectrogram
    root_group.create_dataset("frequencies", data=spectrogram_frequencies)
    root_group.create_dataset("times", data=spectrogram_times)

def _auto_detect_spectrogram_maxval(spectrogram: np.array, *, sr_spectrogram: float):
    Nt = spectrogram.shape[0]
    Nf = spectrogram.shape[1]
    chunk_num_samples = int(15 * sr_spectrogram)
    chunk_maxvals: List[float] = []
    i = 0
    while i + chunk_num_samples < Nt:
        chunk = spectrogram[i:i + chunk_num_samples]
        chunk_maxvals.append(np.max(chunk))
        i += chunk_num_samples
    v = np.median(chunk_maxvals)
    return v

def downsample_spectrogram_using_max(spectrogram: np.ndarray, *, ds_factor: int):
    Nt = spectrogram.shape[0]
    Nf = spectrogram.shape[1]
    Nt_ds = Nt // ds_factor
    spectrogram_ds = np.zeros((Nt_ds, Nf), dtype=spectrogram.dtype)
    spectrogram = spectrogram[:Nt_ds * ds_factor, :]
    spectrogram_reshaped = spectrogram.reshape((Nt_ds, ds_factor, Nf))
    spectrogram_ds = np.max(spectrogram_reshaped, axis=1)
    return spectrogram_ds


import requests

from bs4 import BeautifulSoup

response = requests.get(input())

if response.status_code == 200:
    soup = BeautifulSoup(response.content, "html.parser")
    links = soup.find_all("a")
    filtered_links = list(filter(lambda x: len(x.text) > 1 and x.text.startswith("S") and (("entity" in x.get("href") or "topic" in x.get("href")) if x.get("href") is not None else True), links))
    print(list(map(lambda x: x.text, filtered_links)))
else:
    print("Error")

Topics/BeautifulSoup  working with HTML (3)/Health issues/main.py