#!/usr/bin/python3
# -*- coding: utf-8 -*-

"""
Управление классами ядра системы
"""

from os import path
import json
import logging

from config import SETTINGS
from config import TEST_PATH_RESULTS, WRONG_AUTO_MINFIN_TESTS_FILE
from constants import ERROR_NO_DOCS_FOUND, ERROR_REQUEST_CONTAINS_BAD_WORD
from core.answer_object import CoreAnswer
from core.cube_classifier import CubeClassifier
from core.cube_docs_processing import CubeAnswer
from core.cube_docs_processing import CubeProcessor
from core.cube_or_minfin_classifier import CubeOrMinfinClassifier
from core.minfin_docs_processing import MinfinProcessor
from core.solr import Solr
from core.support_library import group_documents
from core.support_library import process_cube_answer
from core.support_library import process_server_response
from core.support_library import send_request_to_server
from model_manager import MODEL_CONFIG
from text_preprocessing import TextPreprocessing
import logs_helper  # pylint: disable=unused-import

class DataRetrieving:
    """
    Модуль связывающие в себе результаты работы Apache Solr,
    и классов обработки его выдачи - CubeProcessor и MinfinProcessor
    """

    TPP = TextPreprocessing(label='DATRET', delete_question_words=False)

    # хранение данных в памяти
    _minfin_auto_wrong_data = None

    @staticmethod
    def get_data(user_request: str, request_id: str):
        """API метод к ядру системы"""

        core_answer = CoreAnswer()
        core_answer.user_request = user_request

        if user_request.lower().startswith("кто будет"):
            return core_answer

        norm_user_request = DataRetrieving._preprocess_user_request(
            core_answer.user_request,
            request_id
        )

        correct_answer_num = DataRetrieving._process_exact_minfin_answers(
            user_request
        )

        # обработка плохих слов
        badcount = norm_user_request.count('<censored>')
        if badcount > 0:
            core_answer.bad_content = True
            core_answer.message = ERROR_REQUEST_CONTAINS_BAD_WORD

            logging.info(
                'Query_ID: {}\tMessage: Запрос содержал {}'.format(
                    request_id,
                    'плохое слово' if badcount == 1 else 'плохие слова'
                )
            )

            return core_answer

        # получение результатов поиска от Apache Solr в JSON-строке
        solr_response = Solr.get_data(
            norm_user_request,
            request_id,
            SETTINGS.SOLR_MAIN_CORE
        )['response']

        # Если хотя бы 1 документ найден:
        if solr_response['numFound']:
            minfin_docs, cube_data = group_documents(
                solr_response['docs'],
                core_answer.user_request,
                norm_user_request,
                request_id
            )

            minfin_answers = MinfinProcessor.get_data(minfin_docs)

            clf = CubeClassifier.inst()
            best_prediction = tuple(
                clf.predict_proba(core_answer.user_request)
            )[0]

            ans_type = None

            # ручная проверка, актуальная для вопросов по pretty_feedback
            clf_status, cube = DataRetrieving._manual_cube_classification(
                norm_user_request
            )

            if clf_status:
                best_prediction = (cube, 0.99)
                ans_type = 'cube'

            cube_answers, cube_confidence = CubeProcessor.get_data(
                cube_data, best_prediction)

            logging.info(
                "Query_ID: {}\tMessage: Найдено {} докумета(ов) по кубам и {} по Минфину".format(
                    request_id,
                    len(cube_answers),
                    len(minfin_answers)
                )
            )

            answers = DataRetrieving._sort_answers(
                minfin_answers,
                cube_answers,
                correct_answer_num,
                ans_type
            )

            DataRetrieving._format_core_answer(
                answers,
                request_id,
                core_answer,
                cube_confidence
            )
        else:
            # Обработка случая, когда документы не найдены
            core_answer.message = ERROR_NO_DOCS_FOUND
            logging.info(
                'Query_ID: {}\tMessage: Документа не найдены'.format(
                    request_id)
            )

        return core_answer

    @staticmethod
    def _process_exact_minfin_answers(user_request: str):
        if MODEL_CONFIG["use_local_file_processing_for_minfin"]:
            minfin_auto_wrong_tests = DataRetrieving._minfin_auto_wrong_questions()
            user_request = user_request.lower()

            user_request = user_request.replace('?', '')

            # сработает для формулировки с любым порядком слов
            for key in minfin_auto_wrong_tests.keys():
                if set(user_request.split()) == set(key.split()):
                    return minfin_auto_wrong_tests.get(key, 0)

    @staticmethod
    def _minfin_auto_wrong_questions():
        if not DataRetrieving._minfin_auto_wrong_data:
            minfin_wrong_auto_tests_file = path.join(
                TEST_PATH_RESULTS, WRONG_AUTO_MINFIN_TESTS_FILE
            )

            with open(minfin_wrong_auto_tests_file, 'r', encoding='utf-8') as file:
                DataRetrieving._minfin_auto_wrong_data = json.loads(
                    file.read())

        return DataRetrieving._minfin_auto_wrong_data

    @staticmethod
    def _preprocess_user_request(user_request: str, request_id: str):
        """Предобработка запроса пользователя"""

        # нормализация запроса пользователя
        norm_user_request = DataRetrieving.TPP(user_request, request_id)

        if MODEL_CONFIG["delete_repeating_words_in_request"]:
            norm_user_request = DataRetrieving._set_user_request(
                norm_user_request, request_id
            )

        if MODEL_CONFIG["enable_repetition_for_short_request"]:
            norm_user_request = DataRetrieving._multiple_user_request(
                norm_user_request, request_id
            )

        return norm_user_request

    @staticmethod
    def _set_user_request(norm_user_request: str, request_id: str):
        """
        Удаление повторяющихся слов и увеличение длины запроса
        при необходимости
        """

        request_words = norm_user_request.split()
        unique_of_request_words = set(request_words)

        # удаление повторяющихся слов, чтобы пользователи не читерили
        if len(unique_of_request_words) != len(request_words):
            norm_user_request = ' '.join(unique_of_request_words)

            logging.info(
                "Query_ID: {}\tMessage: Удалено {} повторяющихся слов".format(
                    request_id,
                    len(request_words) - len(unique_of_request_words)
                )
            )

        return norm_user_request

    @staticmethod
    def _multiple_user_request(norm_user_request: str, request_id: str):
        """
        Дублирование коротких запросов
        """

        short_question_threshold = MODEL_CONFIG["short_request_threshold"]
        multiplier = MODEL_CONFIG["repetition_num_for_short_request"]
        request_len = len(norm_user_request.split())

        if request_len <= short_question_threshold:
            norm_user_request = ' '.join(
                [norm_user_request.strip()] * multiplier
            )

            logging.info(
                "Query_ID: {}\tMessage: Запрос из {} слов был "
                "удлинен в {} раза".format(
                    request_id,
                    request_len,
                    multiplier
                )
            )

        return norm_user_request

    @staticmethod
    def _sort_answers(
            minfin_answers: list,
            cube_answers: list,
            correct_answer_num: str,
            ans_type: str
    ):
        """Совокупное ранжирование ответов по кубам и минфину"""

        # Если по минфину найден только 1 ответ
        if not isinstance(minfin_answers, list):
            minfin_answers = [minfin_answers]

        # Если по куба найден только 1 ответ
        if not isinstance(cube_answers, list):
            cube_answers = [cube_answers]

        # Фильтрация выборки в порядке убывания по score
        all_answers = sorted(
            cube_answers + minfin_answers,
            key=lambda ans: ans.get_score(),
            reverse=True
        )

        if all_answers:
            DataRetrieving._first_place_right_type(
                all_answers,
                ans_type
            )

            if MODEL_CONFIG["use_local_file_processing_for_minfin"]:
                DataRetrieving._first_place_exact_minfin_answer(
                    all_answers,
                    correct_answer_num
                )

        return all_answers

    @staticmethod
    def _first_place_right_type(all_answers: list, ans_type: str):

        if not ans_type:
            user_request = all_answers[0].user_request

            clf = CubeOrMinfinClassifier.inst()
            prediction = tuple(clf.predict_proba(user_request))[0]
            ans_type = prediction[0].lower()

        if all_answers[0].type != ans_type:
            for elem in list(all_answers):
                if elem.type == ans_type:
                    all_answers.remove(elem)
                    all_answers.insert(0, elem)

                    logging.info(
                        "Query_ID: {}\tMessage: Главный ответ был "
                        "сменен на тип: {}".format(
                            elem.request_id,
                            ans_type
                        )
                    )

                    break
        else:
            logging.info(
                "Query_ID: {}\tMessage: Тип главного ответа совпадает с "
                "типом из классификатора".format(all_answers[0].request_id)
            )

    @staticmethod
    def _first_place_exact_minfin_answer(all_answers: list, correct_number: str):
        if correct_number:
            for answer in list(all_answers):
                if (answer.type == 'minfin' and
                        answer.number == correct_number):
                    all_answers.remove(answer)
                    all_answers.insert(0, answer)

                    if answer.get_score() < MODEL_CONFIG["relevant_minfin_main_answer_threshold"]:
                        score = MODEL_CONFIG["relevant_minfin_main_answer_threshold"]

                        logging.info(
                            "Query_ID: {}\tMessage: Score был увеличен с {} до {}".format(
                                all_answers[0].request_id,
                                answer.get_score(),
                                score
                            )
                        )

                        answer.score = score

                    logging.info(
                        "Query_ID: {}\tMessage: Главный ответ был сменен на основе"
                        " логов некорректных автоматических тестов".format(
                            all_answers[0].request_id
                        )
                    )

                    break

    @staticmethod
    def _format_core_answer(
            answers: list,
            request_id: str,
            core_answer: CoreAnswer,
            cube_confidence: str
    ):
        """Формирование структуры финального ответа"""

        # Предельное количество "смотри также"
        THRESHOLD = 5

        if answers:
            DataRetrieving._process_main_answer(
                core_answer,
                answers,
                request_id,
                cube_confidence
            )

            starting_from = 1
            if not core_answer.answer:
                starting_from = 0
                THRESHOLD -= 1

            DataRetrieving._process_more_answers(
                core_answer,
                answers[starting_from:THRESHOLD + 1],
                request_id
            )

        return core_answer

    @staticmethod
    def _process_main_answer(
            core_answer: CoreAnswer,
            answers: list,
            request_id: str,
            cube_confidence: str
    ):
        """Форматирование главного ответа"""

        # Выбор главного ответа
        core_answer.answer = answers[0]

        # Если главный ответ по кубам
        if isinstance(core_answer.answer, CubeAnswer):
            core_answer.status = True
            core_answer.confidence = cube_confidence

            logging.info(
                "Query_ID: {}\tMessage: Главный ответ - "
                "ответ по кубу - {} - {}".format(
                    request_id,
                    core_answer.answer.get_score(),
                    core_answer.answer.mdx_query
                ))

            response = send_request_to_server(
                core_answer.answer.mdx_query,
                core_answer.answer.cube
            )

            # ответ с сервера
            value = process_server_response(core_answer.answer, response)

            # форматирование ответа при его наличии
            if value is not None:
                process_cube_answer(core_answer.answer, value)
        # Если главный ответ по минфину
        else:
            # фильтр по релевантности на минфин
            if (core_answer.answer.get_score() >=
                    MODEL_CONFIG["relevant_minfin_main_answer_threshold"]):

                core_answer.status = True

                logging.info(
                    "Query_ID: {}\tMessage: Главный точный ответ - "
                    "ответ по Минфину - {} - {}".format(
                        request_id,
                        core_answer.answer.get_score(),
                        core_answer.answer.number
                    ))
            elif (core_answer.answer.get_score() >=
                  MODEL_CONFIG["minfin_main_answer_confidence_threshold"]):

                core_answer.status = True
                core_answer.confidence = False

                logging.info(
                    "Query_ID: {}\tMessage: Главный возможный ответ - "
                    "ответ по Минфину - {} - {}".format(
                        request_id,
                        core_answer.answer.get_score(),
                        core_answer.answer.number
                    ))
            else:
                # Обнуление найденого ответа
                core_answer.answer = None

                logging.info(
                    "Query_ID: {}\tMessage: Главный ответ по Минфину "
                    "не прошел порог ({} vs {})".format(
                        request_id,
                        answers[0].get_score(),
                        MODEL_CONFIG["relevant_minfin_main_answer_threshold"]
                    ))

    @staticmethod
    def _process_more_answers(
            core_answer: CoreAnswer,
            more_answers: list,
            request_id: str
    ):
        """Обработка дополнительных ответов"""

        more_cube_answers = []
        more_minfin_answers = []

        more_answers_order = ''

        for ind, answer in enumerate(more_answers):
            answer.order = ind

        for answer in more_answers:
            if isinstance(answer, CubeAnswer):
                more_cube_answers.append(answer)
                more_answers_order += '0'
            else:
                more_minfin_answers.append(answer)
                more_answers_order += '1'

        if more_cube_answers:
            core_answer.more_cube_answers = more_cube_answers

        if more_minfin_answers:
            core_answer.more_minfin_answers = more_minfin_answers

        if more_answers_order:
            core_answer.more_answers_order = more_answers_order

        logging.info(
            "Query_ID: {}\tMessage: В смотри также {} "
            "ответ(а, ов) по кубам и {} по Минфину".format(
                request_id,
                len(more_cube_answers),
                len(more_minfin_answers)
            ))

    @staticmethod
    def _manual_cube_classification(user_request: str):
        key_words_to_cube = {
            "основной показатель субъект РФ": "CLDO02",
            "источник финансирование": "FSYR01",
            "оперативный расход": "EXDO01",
            "годовой расход": "EXYR03",
            "оперативный доход": "INDO01",
            "основной характеристика федеральный бюджет": "CLDO01",
            "годовой доход": "INYR03",
            "госдолг РФ": "CLMR02"
        }

        for key, value in key_words_to_cube.items():
            if user_request.startswith(key):
                return True, value

        return False, None


import subprocess
import os
from six import string_types

# SLHA parser
# by Florian Staub (florian.staub@gmail.com)

# ----------------------------------------------------------
# SLHA Class
# ----------------------------------------------------------

class SLHA():
    def __init__(self):
        self.blocks = {}
        self.br = {}
        self.widths = {}
        self.br1L = {}
        self.widths1L = {}
        self.xsections = {}

        self.block_name = None
        self.entries = {}
        self.reading_block = False
        self.reading_decay = False
        self.reading_xsection = False
        self.reading_hb_fermion = False
        self.reading_hb_boson = False
        self.decay1L = False
        self.decay_part = 0

    # return wdith and BR
    def BR(self, init, final):
        # frozenset: make sure that the final states are order-less
        return self.br[init][tuple(sorted(final))]

    def Width(self, pdg):
        return self.widths[pdg]

    def Value(self, block, number):
        '''return value of a parameter defined by block and entry
            or the width or an BR'''
        if block == 'WIDTH':
            return self.widths[number]
        elif block == 'BR':
            return self.br[number[0]][tuple(sorted(number[1]))]
        elif block == 'WIDTH1L':
            return self.widths1L[number]
        elif block == 'BR1L':
            return self.br1L[number[0]][tuple(sorted(number[1]))]
        elif block == 'XSECTION':
            xs = self.xsections[tuple(number)]
            return [[x, xs[x]] for x in xs.keys()]
        else:
            return self.blocks[block.upper()][
                str(number)[1:-1].replace(" ", "")]

    def start_decay(self, li):
        parsed = list(filter(None, li.split(' ')))
        self.decay1L = li.upper().startswith("DECAY1L")
        self.decay_part = int(parsed[1])
        if self.decay1L:
            self.widths1L[self.decay_part] = float(parsed[2])
        else:
            self.widths[self.decay_part] = float(parsed[2])
        self.entries = {}
        self.reading_block, self.reading_decay, self.reading_xsection \
            = False, True, False

    def start_block(self, li):
        self.block_name = (list(filter(None, li.split(' ')))[1]).upper()
        self.entries = {}
        self.reading_block, self.reading_decay, self.reading_xsection \
            = True, False, False
        self.reading_hb_boson = \
            self.block_name in ["HIGGSBOUNDSINPUTHIGGSCOUPLINGSBOSONS",
                                "HIGGSCOUPLINGSBOSONS"]
        self.reading_hb_fermion = \
            self.block_name in ["HIGGSBOUNDSINPUTHIGGSCOUPLINGSFERMIONS",
                                "HIGGSCOUPLINGSFERMIONS"]

    def start_xsection(self, li):
        parsed = list(filter(None, li.split(' ')))
        if "#" in parsed:
            parsed = parsed[:parsed.index("#")]  # remove comments
        self.xs_head = tuple(
            [float(parsed[1]),
             tuple([int(parsed[2]), int(parsed[3])]),
             tuple([int(parsed[-2]), int(parsed[-1])])
             ])
        self.entries = {}
        self.reading_block, self.reading_decay, self.reading_xsection \
            = False, False, True

    def flush(self):
        '''store the information once a block is completely parsed'''
        if len(self.entries) > 0:
            if self.reading_block:
                self.blocks[self.block_name] = self.entries
            if self.reading_decay:
                if self.decay1L:
                    self.br1L[self.decay_part] = self.entries
                else:
                    self.br[self.decay_part] = self.entries
            if self.reading_xsection:
                self.xsections[self.xs_head] = self.entries

# ----------------------------------------------------------
# Reading
# ----------------------------------------------------------

# now the main function to read the SLHA file
def read(file, separator=None, verbose=False):
    spc = SLHA()
    if separator is not None:
        all_files = []
        count = 1

    with open(file) as infile:
        for line in infile:
            li = line.strip().upper()

            if li.startswith("#") or len(li) < 1:
                continue

            if separator is not None:
                if li.startswith(separator):
                    spc.flush()
                    if max(len(spc.blocks.keys()),len(spc.widths.keys())) > 0:
                        all_files.append(spc)
                        # start next point
                        spc = SLHA()
                        count = count + 1
                        if verbose:
                            print("Read spc file:", count)
                        continue

            # New block started
            if li.startswith("BLOCK"):
                spc.flush()  # store information which was read
                spc.start_block(li)
            elif li.startswith("DECAY"):
                spc.flush()  # store information which was read
                spc.start_decay(li)
            elif li.startswith("XSECTION"):
                spc.flush()  # store information which was read
                spc.start_xsection(li)

            # Reading and parsing values
            else:
                parsed = list(filter(None, li.split(' ')))
                if "#" in parsed:
                    parsed = parsed[:parsed.index("#")]  # remove comments
                if spc.reading_block:
                    if spc.reading_hb_fermion:
                        spc.entries[",".join(parsed[3:])] = \
                            [float(parsed[0]), float(parsed[1])]
                    elif spc.reading_hb_boson:
                        spc.entries[",".join(parsed[2:])] = \
                            float(parsed[0])
                    else:
                        # Value might be a string like in SPINFO block
                        try:
                            value = float(parsed[-1])
                        except:
                            value = parsed[-2]
                        spc.entries[",".join(parsed[0:-1])] = value

                if spc.reading_decay:
                    spc.entries[
                        tuple(sorted(eval("[" + ",".join(parsed[2:]) + "]")))
                    ] = float(parsed[0])

                if spc.reading_xsection:
                    spc.entries[
                        tuple(eval("[" + ",".join(parsed[0:-2]) + "]"))
                    ] = float(parsed[-2])

    spc.flush()  # save the very last block in the file

    if verbose:
        print("Read %i blocks and %i decays" % (len(spc.blocks), len(spc.br)))
    if separator is None:
        return spc
    else:
        if len(spc.entries) > 0:
            all_files.append(spc)
        return all_files

# wrapper for faster read-in of multiple files
# squeeze the file (just keeping the necessary entries) to make the reading more efficient
# example: read_small_spc(filename,["# m0","# m12","# relic"],separator="ENDOF")
def read_small(file, entries, sep):
    if entries is None:
        out = read(file, separator=sep)
    else:
        string = "--regexp=\"" + sep + "\" --regexp=\"Block\" "
        for i in entries:
            string = string + "--regexp=\"" + i + "\" "
        if os.path.isfile("temp.spc"):
            subprocess.call("rm temp.spc", shell=True)
        subprocess.call("cat " + file + " | grep -i " + string
                        + " > temp_read_small.spc", shell=True)
        out = read("temp_read_small.spc", separator=sep)
        subprocess.call("rm temp_read_small.spc", shell=True)

    return out

def read_dir(dir, entries=None):
    if os.path.isfile("temp_read_dir.spc"):
        subprocess.call("rm temp_read_dir.spc", shell=True)
#    subprocess.check_call("cat "+dir+"/* > temp_read_dir.spc",shell=True)
    subprocess.check_call("tail -n+1 " + dir + "/* > temp_read_dir.spc",
                          shell=True)
    out = read_small("temp_read_dir.spc", entries, "==>")
    subprocess.call("rm temp_read_dir.spc", shell=True)

    return out

#def read_dir(dir,entries=None):
    #subprocess.call("rm temp_read_dir.spc",shell=True)
    #subprocess.check_call("cat "+dir+"/* > temp_read_dir.spc",shell=True)
    #with open("temp_read_dir.spc") as infile:
     #for line in infile:
       #li=line.strip().upper()
       #if li.startswith("#") or len(li)<1:
          #continue
       #else:
          #file_sep=li[:li.index("#")]
          #break
    #out=read_small("temp_read_dir.spc",entries,file_sep)
    #subprocess.call("rm temp_read_dir.spc",shell=True)

    #return out

# ----------------------------------------------------------
# Writing
# ----------------------------------------------------------

def write(blocks, file):
    with open(file, 'w+') as f:
        for b in blocks:
            write_block_head(b, f)
            write_block_entries(blocks[b], f)

def write_block_entries(values, file):
    for v in values.keys():
        file.write(' %s %10.4e # \n' % (v, float(values[v])))

def write_les_houches(block, values, point, file):
    write_block_head(block, file)
    write_block_numbers(block, values, point, file)

def write_block_head(name, file):
    file.write("Block " + name.upper() + " # \n")

def write_block_numbers(name, values, Variable, file):
    for v in values.keys():
        # if type(values[v]) is string_types:
        if isinstance(values[v], string_types):  # to be 2 and 3 compatible
            if str(eval(values[v]))==values[v]:
                file.write(' %s %s # %s \n'
                           % (v, values[v],
                              name.upper() + "[" + str(v) + "]"))
            else:
                file.write(' %s %10.4e # %s \n'
                           % (v, float(eval(values[v])),
                              name.upper() + "[" + str(v) + "]"))
        elif isinstance(values[v], int):
            file.write(' %s %i # %s \n'
                       % (v, (values[v]), name.upper() + "[" + str(v) + "]"))
        else:
            file.write(' %s %10.4e # %s \n'
                       % (v, float(values[v]),
                          name.upper() + "[" + str(v) + "]"))


import math
import time

from matplotlib import cm, pyplot as plt
from numpy import array, convolve, linalg, matrix, trace, transpose
from PIL import Image

from gaussian_filters import gauss, gauss_1der, gauss_2der
from mark_show import mark_show_hess

def hessian(img,  # the input image name
            scale,  # standard Deviation value
            threshold  # threshold of eigen value to be considered as edge
            ):
    begin = time.time()
    size = 2  # kernel size = 2*(size) + 1
    I = array(Image.open(img).convert("L"))  # reads the input image into I

    # Gaussian along x
    Gx = []
    for i in range(- size, size + 1):
        Gx.append(gauss_1der(i, 0, scale, "x"))

    # Gaussian along y
    Gy = []
    for i in range(- size, size + 1):
        Gy.append([gauss_1der(0, i, scale, "y")])

    Gx2 = []
    for i in range(- size, size + 1):
        Gx2.append(gauss_2der(i, 0, scale, "x"))

    Gy2 = []
    for i in range(- size, size + 1):
        Gy2.append([gauss_2der(0, i, scale, "y")])

    # I*G along x
    Ix = []
    for i in range(len(I[:, 0])):
        Ix.extend([convolve(I[i, :], Gx)])
    Ix = array(matrix(Ix))

    # I*G along y
    Iy = []
    for i in range(len(I[0, :])):
        Iy.extend([convolve(I[:, i], Gx)])
    Iy = array(matrix(transpose(Iy)))

    # Ix*Gx along x
    Ixx = []
    for i in range(len(Ix[:, 0])):
        Ixx.extend([convolve(Ix[i, :], Gx2)])
    Ixx = array(matrix(Ixx))

    # Iy*Gy in y direction
    Iyy = []
    for i in range(len(Iy[0, :])):
        Iyy.extend([convolve(Ix[:, i], Gx2)])
    Iyy = array(matrix(transpose(Iyy)))

    Ixy = []
    for i in range(len(Iy[0, :])):
        Ixy.extend([convolve(Ix[:, i], Gx2)])
    Ixy = array(matrix(transpose(Ixy)))

    x = []  # x vertices of the corners
    y = []  # y vertices of the corners
    for i in range(len(I[:, 0])):
        for j in range(len(I[0, :])):
            H1 = linalg.eigvals(
                ([Ixx[i, j], Ixy[i, j]], [Ixy[i, j], Iyy[i, j]]))
            if((abs(H1[0]) > threshold) & (abs(H1[1]) > threshold)):  # check corner condition
                y.append(i - 2)
                x.append(j - 2)

    mark_show_hess(I, x, y)
    return time.time() - begin

img1 = hessian("../input_images/bicycle.bmp", scale=1, threshold=21)
img2 = hessian("../input_images/bird.bmp", scale=1, threshold=12)
img3 = hessian("../input_images/dog.bmp", scale=1, threshold=6)
img4 = hessian("../input_images/einstein.bmp", scale=1, threshold=15)
img5 = hessian("../input_images/plane.bmp", scale=1, threshold=10)
img6 = hessian("../input_images/toy_image.jpg", scale=1, threshold=38)

print("Execution time:\nImage 1: %.2fseconds\nImage 2: %.2fseconds\nImage 3: %.2fseconds\nImage 4: %.2fseconds\nImage 5: %.2fseconds\nImage 6: %.2fseconds)" %
      (img1, img2, img3, img4, img5, img6))


from collections import deque

from Entidades.Term import TermUnit
from Entidades.Term import Term
from Entidades.Grammar.TextGrammar import TextGrammar
from Entidades.Grammar.Grammar import Grammar
from Services.LexicAnalyzer import LexicAnalyzer

class TableService:
    ErrorString = "Error"

    def __init__(self, grammar):
        self.grammar = grammar
        self.table = None

    def makeTable(self):
        self.table = {}

        for non_terminal in self.grammar.NonTerminals:
            self.table[non_terminal.text] = {Grammar.STREAM_END_UNIT.text: self.ErrorString}

            for terminal in self.grammar.Alphabet:
                self.table[non_terminal.text][terminal.text] = self.ErrorString

    def passToTable(self, i, j, value):
        actualValue = self.table[i][j]

        if(actualValue == self.ErrorString):
            self.table[i][j] = value
        else:
            raise Exception('Ambiguos grammar on '+ i + ' -> ' + j)

    def build_table(self):
        self.makeTable()

        for term in self.grammar.Terms:
            for stream in term.right:
                termToStreamTuple = (term, stream)

                streamFirst = self.first(stream)

                for item in streamFirst:
                    if(item.type is TermUnit.TERMINAL):
                        self.passToTable(term.left, item.text, termToStreamTuple)

                if(TextGrammar.EMPTY_UNIT in streamFirst):
                    if(Grammar.STREAM_END_UNIT in term.follow):
                        self.passToTable(term.left, Grammar.STREAM_END_UNIT.text, termToStreamTuple)

                    for item in term.follow:
                        if(item.type is TermUnit.TERMINAL):
                            self.passToTable(term.left, item.text, termToStreamTuple)

    def apply_follow_third_rule(self):
        """
        A -> Alfa B Beta, B.follow += A.follow
        Alfa is any stream
        Beta is a empty stream
        """
        calls = deque()

        for A in self.grammar.Terms:
            for stream in A.right:
                streamLen = len(stream)

                for index, unit in enumerate(stream):
                    if(unit.type is TermUnit.NONTERMINAL):
                        B = self.get_term(unit.text)

                        if(A is B):
                            continue

                        if(streamLen > index + 1):
                            beta = stream[index + 1::]

                            firstBeta = self.first(beta)

                            if(TextGrammar.EMPTY_UNIT not in firstBeta):
                                continue

                        # B.follow |= A
                        calls.append((B, A))

        while(len(calls) is not 0):
            tupleBA = calls.popleft()
            B, A = tupleBA

            AinStack = next((x for x in calls if x[0] is A), None)

            if(AinStack):
                calls.append(tupleBA)
                continue

            # print(B.left,"___", A.left)

            B.follow |= A.follow

    def apply_follow_second_rule(self):
        """
        A -> Alfa B Beta, B.follow += first(Beta) - {ɛ}
        Alfa is any stream
        Beta is a not empty stream
        """
        for term in self.grammar.Terms:
            for stream in term.right:
                streamLen = len(stream)

                for index, unit in enumerate(stream):
                    if(unit.type is TermUnit.NONTERMINAL):
                        termB = self.get_term(unit.text)

                        if(streamLen > index + 1):
                            beta = stream[index + 1::]
                            firstBeta = self.first(beta)

                            termB.follow |= set(firstBeta) - {self.grammar.textGrammar.EMPTY_UNIT}

    def apply_follow_first_rule(self):
        """
        S.follow = $
        """
        self.grammar.StartSimbol.follow.add(Grammar.STREAM_END_UNIT)

    def build_follow(self):
        """
        Routine that creates follow sets of the given grammar
        """
        self.apply_follow_first_rule()
        self.apply_follow_second_rule()
        self.apply_follow_third_rule()

    def first_of_non_terminal(self, term):
        if(len(term.first) is not 0):
            return term.first

        first = term.first
        for stream in term.right:
            if(len(stream) is 1 and stream[0] is self.grammar.textGrammar.EMPTY_UNIT):
                first |= {self.grammar.textGrammar.EMPTY_UNIT}
            else:
                first |= self.first(stream) - {self.grammar.textGrammar.EMPTY_UNIT}

        term.first = first

        return first

    def term_has_empty(self, term):
        for stream in term.right:
            for item in stream:
                if(item is TextGrammar.EMPTY_UNIT):
                    return True

        return False

    def get_term(self, termString):
        for i in self.grammar.Terms:
            if(i.left == termString):
                return i

    def first_of_stream(self, stream):
        retorno = set()

        for index, item in enumerate(stream):
            if(item.type is TermUnit.NONTERMINAL):
                term = self.get_term(item.text)

                retorno |= self.first(term)

                if(self.term_has_empty(term)):
                    continue
                else:
                    break

            elif(index is 0 and item.type is TermUnit.TERMINAL or item is TextGrammar.EMPTY_UNIT):
                retorno.add(item)
                break

        return retorno

    def first(self, p) :
        p_type = type(p)
        retorno = None

        if(p_type is list):
            retorno = self.first_of_stream(p)

        elif(p_type is TermUnit):
            term = self.get_term(p)
            retorno = self.first_of_non_terminal(term)

        elif(p_type is Term):
            retorno = self.first_of_non_terminal(p)

        return retorno

    def build_first(self):
        for term in self.grammar.Terms:
            self.first(term)

    def compileGrammar(self):
        if(self.table is None):
            self.build_first()
            self.build_follow()
            self.build_table()

    def compile(self, text):
        self.compileGrammar()
        historic = ''
        lxa = LexicAnalyzer(text, self.grammar)

        stack = [Grammar.STREAM_END_UNIT, self.grammar.StartSimbolUnit]

        # while(lxa.isNotDone()):
        #     print(lxa.getToken())

        current = lxa.getToken()
        while( len(stack) ):#current.unit.type is not TermUnit.STREAM_END):
            hline = str(current.value).ljust(20) + " " + str(stack) + '\n'
            historic += hline

            top = stack.pop()
            # print(hline)

            if(top.type is TermUnit.TERMINAL or top.type is TermUnit.STREAM_END):
                if(top.text == current.value):
                    current = lxa.getToken()

                elif(top.text == 'ide'):
                    current = lxa.getToken()

                elif(top.text == 'num'):
                    current = lxa.getToken()

                else:
                    raise Exception("Error on " + current.value + " at:")

            else:
                tupleCell = self.table[top.text][current.unit.text]

                if(tupleCell != self.ErrorString):
                    for term in tupleCell[1][::-1]:
                        if(term.type is not TermUnit.EMPTY):
                            stack.append(term)

                elif(current.unit.type is TermUnit.STREAM_END):
                    raise Exception("Error: invalid file end")

                else:
                    raise Exception("Error on " + current.value + " at:")

        # print("Compiling succes for entry:\n"+ text)
        return lxa.tokens, historic

Python/Analisador Sintático Descendente Tabular/Services/GrammarTableService.py


import feedparser
import re
import docclass
def read(feed,classifier):
  f=feedparser.parse(feed)

  for entry in f['entries']:
    #print(entry)

    print ()#这一行在干嘛
    print('-----')
    #print('title')
    #print(entry['title'].encode('utf-8'))
    print ('Title:     ',entry['title'].encode('utf-8'))
    print ('Publisher: ',entry['publisher'].encode('utf-8'))
    print ()
    print (entry['summary'].encode('utf-8'))
    '''
    print ('Title:     '+entry['title'].encode('utf-8'))
    print ('Publisher: '+entry['publisher'].encode('utf-8'))
    print ()
    print (entry['summary'].encode('utf-8'))
    '''

    #fulltext='%s\n%s\n%s' % (entry['title'],entry['publisher'],entry['summary'])
    #print(fulltext)
    #aaa=str(classifier.classify(fulltext))
    #print(aaa)
    #print ('Guess: '+str(classifier.classify(fulltext)))
    print ('Guess: '+str(classifier.classify(entry)))

    c1=input('Enter category: ')

    #print('hhh',entry)
    #print(c1)
    #classifier.train(fulltext,c1)
    classifier.train(entry,c1)

def entryfeatures(entry):
  splitter=re.compile('\\W*')
  f={}

  titlewords=[s.lower() for s in splitter.split(entry['title'])
          if len(s)>2 and len(s)<20]
  for w in titlewords: f['Title:'+w]=1

  summarywords=[s.lower() for s in splitter.split(entry['summary'])
          if len(s)>2 and len(s)<20]

  uc=0
  for i in range(len(summarywords)):
    w=summarywords[i]
    f[w]=1
    if w.isupper(): uc+=1

    if i<len(summarywords)-1:
      twowords=' '.join(summarywords[i:i+1])
      f[twowords]=1

  f['Publisher:'+entry['publisher']]=1

  if float(uc)/len(summarywords)>0.3: f['UPPERCASE']=1

  return f


from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
import numpy as np

def logisticRegression():
    data=np.loadtxt("ex2data1.txt",delimiter=",",dtype=np.float64)
    X=data[:,0:-1]
    y=data[:,-1]

    #train_test_split可将数据集随机划分为训练集和测试集
    x_train,x_test,y_train,y_test=train_test_split(X,y,test_size=0.2) #test_size为浮点数 则意思为随机抽取20%的数据作为测试集

    #特征归一化
    scaler=StandardScaler()
    scaler.fit_transform(x_train)
    scaler.fit_transform(x_test)

    #逻辑回归
    model=LogisticRegression()
    model.fit(x_train,y_train)

    #预测
    predict=model.predict(x_test)
    right=sum(predict==y_test)
    predict=np.hstack((predict.reshape(-1,1),y_test.reshape(-1,1)))
    print(predict)
    print('测试集准确率:%f%%'%(right*100.0/predict.shape[0]))

if __name__=='__main__':
    logisticRegression()


#! /usr/bin/env python
# -*- coding: utf-8 -*-

"""Merge two existed dataset."""

import argparse
import logging
from pathlib import Path
from os.path import dirname as parent

from fairseq.molecule_utils.external.fairseq_dataset_build_utils import merge_and_build
from fairseq.molecule_utils.basic import smiles_utils as smu

def main():
    parser = argparse.ArgumentParser(description='Merge two existed dataset.')
    parser.add_argument('-i1', '--input-dir-1', dest='input_dir_1', type=Path, required=True, help='Input data directory 1')
    parser.add_argument('--in1', '--input-name-1', dest='input_name_1', type=str, required=True, help='Input name 1')
    parser.add_argument('-i2', '--input-dir-2', dest='input_dir_2', type=Path, required=True, help='Input data directory 2')
    parser.add_argument('--in2', '--input-name-2', dest='input_name_2', type=str, required=True, help='Input name 2')
    parser.add_argument('-o', '--output-dir', type=Path, required=True, help='Output data directory')
    parser.add_argument('--on', '--output-name', dest='output_name', type=str, default=None,
                        help='Output name, default to input name')
    parser.add_argument('--keep-duplicates', action='store_true')
    parser.add_argument('-v', '--verbose', action='store_true')

    args = parser.parse_args()

    logging.basicConfig(
        format='%(levelname)s:%(filename)s:%(message)s', level=logging.DEBUG if args.verbose else logging.INFO)
    smu.disable_rdkit_log()

    merge_and_build(
        input_dir_1=args.input_dir_1,
        input_name_1=args.input_name_1,
        input_dir_2=args.input_dir_2,
        input_name_2=args.input_name_2,
        output_dir=args.output_dir,
        output_name=args.output_name,
        fairseq_root=Path(parent(parent(parent(__file__)))),
        pre_dicts_root=Path(parent(parent(parent(__file__)))) / 'dict',
        keep_duplicates=args.keep_duplicates,
    )

if __name__ == '__main__':
    main()


import re
from collections import Counter
from pathlib import Path
from shutil import copyfile
from typing import Set, List, Literal, Iterable, Optional

import pandas
from sortedcontainers import SortedDict
from tqdm import tqdm

from .base import BaseTask, CorporaTaskMixin
from .filters.base import CandidatesPairCSV
from .imports import FileID
from .tokenize import TokenizedWordsCSV
from ..utils import logger
from ..workspace import Workspace, WorkspaceCSV

def load_group_words(group: Set[FileID], workspace: Workspace) -> Counter:
    tokenized_folder = workspace.root_path / Path("datasets/tokenized/per_text/")

    group_words = Counter()
    for file_id in group:
        tokenized_file = TokenizedWordsCSV(tokenized_folder / Path(f"{file_id}.csv"))
        try:
            group_words.update(tokenized_file.to_dict())
        except FileNotFoundError:
            logger.warning(f"Couldn't find tokenized text file {file_id}.csv")
            continue

    return group_words

class CorporaCreationTask(BaseTask):
    """Creates corpora based on the list of tokenized words for each text
    in the dataset and the text files families. The output words list are
    compted from the intersection of the words of the texts from the families'
    groups"""
    requires = [
        "datasets/tokenized/all.csv",
        "datasets/families/*/*.txt"
    ]

    creates = [
        "corpora/",
        "corpora/tokenized/*.csv",
    ]

    @staticmethod
    def sort_families_folders(folders: Iterable[Path]) -> List[Path]:
        return sorted(folders, key=lambda x: int(x.stem.split("_")[1]),
                      reverse=True)

    def run(self, workspace: Workspace):
        workspace.corpora.mkdir(parents=True, exist_ok=True)

        logger.info("Building families words lists...")
        families_folder = workspace.datasets / Path("families/")
        # Families *have* to be sorted
        pbar = tqdm(self.sort_families_folders(families_folder.iterdir()))
        previous_group_words: Set[str] = set()
        for family_folder in pbar:
            family_folder: Path
            assert family_folder.is_dir()
            family_id = int(re.fullmatch(r"family_([0-9]+)", family_folder.name)[1])
            assert len(list(family_folder.iterdir())) == family_id

            family_words_dict = None
            groups_paths = self.sort_families_folders(list(family_folder.iterdir()))
            for group_filepath in groups_paths:
                pbar.set_description(f"Family {family_id}: {group_filepath.name}")

                assert group_filepath.is_file()
                with open(group_filepath) as group_file:
                    group = set(file_id for file_id in group_file.read().split("\n") if file_id)
                group_words_dict = load_group_words(group, workspace)

                # the first group is taken as base for the intersection
                if family_words_dict is None:
                    family_words_dict = group_words_dict
                else:
                    # else we only keep the words set that intersect
                    # and increment their counts with the currentgroup's count
                    words_intersection = (set(group_words_dict) & set(family_words_dict))
                    for word in list(family_words_dict):
                        if word in words_intersection:
                            family_words_dict[word] += group_words_dict[word]
                        else:
                            del family_words_dict[word]

            # removing words from "previous" family
            len_before = len(family_words_dict)
            for word in previous_group_words:
                if word in family_words_dict:
                    del family_words_dict[word]
            previous_group_words.update(set(family_words_dict.keys()))

            logger.debug(f"Removed {len_before - len(family_words_dict)} words "
                         f"from smaller corpora in corpus {family_id}")

            logger.debug(f"Writing words list for family {family_id}")
            tokenized_corpora_folder = workspace.corpora / Path("tokenized/")
            tokenized_corpora_folder.mkdir(parents=True, exist_ok=True)
            family_words_csv = TokenizedWordsCSV(tokenized_corpora_folder /
                                                 Path(f"corpus_{family_id}.csv"))
            # sorting dict by word count (decreasing)
            family_words_dict = dict(sorted(family_words_dict.items(),
                                            key=lambda item: item[1],
                                            reverse=True))
            with family_words_csv.dict_writer as dict_writer:
                dict_writer.writeheader()
                for word, count in family_words_dict.items():
                    dict_writer.writerow({
                        "word": word, "count": count
                    })

class ZeroSpeechCSV(WorkspaceCSV):
    header = ["id", "filename", "voice", "frequency", "word", "phones", "length", "correct"]

    def __init__(self, file_path: Path):
        super().__init__(file_path, separator=",", header=self.header)

class BuildZeroSpeechTestSetsTask(BaseTask, CorporaTaskMixin):
    requires = [
        "corpora/wuggy_pairs/*",
        "synth/audio/phonetic/",
    ]

    def __init__(self,
                 output_folder : Path,
                 real_word_synth: Literal["text", "phonetic"],
                 for_corpus: Optional[int] = None):
        super().__init__()
        self.real_word_synth = real_word_synth
        self.for_corpus = for_corpus
        self.output_folder  = output_folder

    def build_frequencies_csv(self,
                              zr_corpus_folder: Path,
                              corpus_id: int,
                              workspace: Workspace):
        family_folder = workspace.datasets / Path(f"families/family_{corpus_id}")
        corpus_csv = TokenizedWordsCSV(workspace.corpora / Path(f"tokenized/corpus_{corpus_id}.csv"))
        corpus_words = set(corpus_csv.to_dict().keys())

        group_paths = sorted(family_folder.iterdir(), key=lambda p: int(p.stem.split("_")[1]))
        freqs_df = pandas.DataFrame()
        for group_filepath in tqdm(group_paths):

            with open(group_filepath) as group_file:
                file_ids = {f_id for f_id in group_file.read().split("\n") if f_id}
            group_words_freqs = load_group_words(file_ids, workspace)

            group_words_freqs = SortedDict({word: freq
                                            for word, freq in group_words_freqs.items()
                                            if word in corpus_words})
            if freqs_df.index.empty:
                freqs_df.index = list(group_words_freqs.keys())

            freqs_df[group_filepath.name] = list(group_words_freqs.values())

        freqs_df.to_csv(zr_corpus_folder / Path("frequencies.csv"),
                        sep=",")

    def build_zr_testset(self,
                         wuggy_pairs_path: Path,
                         zr_corpus_folder: Path,
                         workspace: Workspace):
        zr_audio_folder = zr_corpus_folder / Path("ogg")
        zr_audio_folder.mkdir(parents=True, exist_ok=True)
        zr_csv_path = zr_corpus_folder / Path("gold.csv")
        words_dataset_freqs = TokenizedWordsCSV(workspace.tokenized / "all.csv").to_dict()

        phonetic_synth_folder = workspace.synth / Path("audio/phonetic/")
        text_synth_folder = workspace.synth / Path("audio/text/")
        voices = [p.name for p in phonetic_synth_folder.iterdir()]

        corpus_wuggy_pairs_csv = CandidatesPairCSV(wuggy_pairs_path)
        zr_corpus_csv = ZeroSpeechCSV(zr_csv_path)

        pbar = tqdm(total=corpus_wuggy_pairs_csv.lines_count * len(voices))

        with zr_corpus_csv.dict_writer as dict_writer:
            dict_writer.writeheader()
            for pair_id, (word, phonetic, fake_phonetic) in enumerate(corpus_wuggy_pairs_csv):
                if self.real_word_synth == "text":
                    word_filename = word
                else:
                    word_filename = phonetic.replace(" ", "_")
                fake_word_filename = fake_phonetic.replace(" ", "_")

                for voice in voices:
                    # writing pair entries in the "gold" csv file
                    dict_writer.writerow({
                        "id": pair_id + 1,
                        "filename": f"{word_filename}-{voice}",
                        "voice": voice,
                        "frequency": words_dataset_freqs[word],
                        "word": word,
                        "phones": phonetic,
                        "length": len(phonetic.split(" ")),
                        "correct": 1
                    })
                    dict_writer.writerow({
                        "id": pair_id + 1,
                        "filename": f"{fake_word_filename}-{voice}",
                        "voice": voice,
                        "frequency": "",
                        "word": None,
                        "phones": fake_phonetic,
                        "length": len(fake_phonetic.split(" ")),
                        "correct": 0
                    })

                    # copying audio files for the pair
                    if self.real_word_synth == "text":
                        word_audio_path = text_synth_folder / Path(f"{voice}/{word_filename}.ogg")
                    else:
                        word_audio_path = phonetic_synth_folder / Path(f"{voice}/{word_filename}.ogg")
                    word_cp_path = zr_audio_folder / Path(f"{word_filename}-{voice}.ogg")
                    fake_word_audio_path = phonetic_synth_folder / Path(f"{voice}/{fake_word_filename}.ogg")
                    fake_word_cp_path = zr_audio_folder / Path(f"{fake_word_filename}-{voice}.ogg")

                    if not word_cp_path.is_file():
                        copyfile(word_audio_path, word_cp_path)
                    if not fake_word_cp_path.is_file():
                        copyfile(fake_word_audio_path, fake_word_cp_path)

                    pbar.update()

    def run(self, workspace: Workspace):
        zr_folder = self.output_folder
        zr_folder.mkdir(parents=True, exist_ok=True)
        wuggy_pairs_folder = workspace.corpora / Path("wuggy_pairs")

        corpora = self.find_corpora(wuggy_pairs_folder)
        if self.for_corpus is not None:
            corpora = [(corpus_id, corpus_path)
                       for corpus_id, corpus_path in corpora
                       if corpus_id == self.for_corpus]

        for corpus_id, corpus_csv_path in corpora:
            corpus_zr_folder = zr_folder / Path(f"testset_{corpus_id}")
            logger.info(f"For corpus {corpus_id}")
            logger.info("Building corpus table and aggregating audio files")
            self.build_zr_testset(corpus_csv_path, corpus_zr_folder, workspace)
            logger.info("Building corpus word frequencies table")
            self.build_frequencies_csv(corpus_zr_folder, corpus_id, workspace)


import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import warnings
warnings.filterwarnings('ignore')
import argparse
from typing import List
from src.models.named_entity_recognition_rnn import RNNModel, DataPreprocessor
from src.utils.config_loader import NamedEntityRecognitionConfigReader

def evaluate_model(questions_list: List[str], config: NamedEntityRecognitionConfigReader) -> None:
    """
    Wrapper function that calls the model deserializer and returns prediction
    Args:
        model_file_url: relative path for the model file
        questions_list: list of strings to perform inference on
    Return:
        list of probabilies for positive class for each input word
    """
    pre_processor = None
    trained_model = None
    trained_model, preprocessor_file = RNNModel.load_model(config.h5_model_url, config.class_file_url,
                                                           config.preprocessor_file_url, collect_from_gdrive=False)
    if trained_model is None:
        raise ValueError("there is no corresponding model file")
    pre_processor = DataPreprocessor.load_preprocessor(preprocessor_file)
    questions_list_encoded, questions_list_tokenized, n_tokens =\
        DataPreprocessor.preprocess_data(questions_list, pre_processor)
    labels_list = trained_model.predict(questions_list_encoded, pre_processor["labels_to_idx"], n_tokens)
    display_result = trained_model.visualize(questions_list_tokenized, labels_list)
    print(display_result)

def parse_arguments():
    """
    Parse file arguments
    """
    parser = argparse.ArgumentParser(description="Predict NER for a given input text")
    parser.add_argument('text', help="texts separated by space, each text must be surrounded with quotes", nargs='+')
    return parser.parse_args()

def main():
    """main function"""
    args = parse_arguments()
    qlist = args.text
    root_dir = os.environ.get("MARABOU_HOME")
    if root_dir is None:
        raise ValueError("please make sure to setup the environment variable MARABOU_ROOT to\
                         point for the root of the project")
    config_file_path = os.path.join(root_dir, "marabou/train/config/config_named_entity_recognition.json")
    config_file = NamedEntityRecognitionConfigReader(config_file_path)
    evaluate_model(qlist, config_file)

if __name__ == '__main__':
    main()


"""
129. Sum Root to Leaf Numbers
You are given the root of a binary tree containing digits from 0 to 9 only.
Each root-to-leaf path in the tree represents a number.
- For example, the root-to-leaf path 1->2->3 represents the number 123.

Return the total sum of all root-to-leaf numbers. Test cases are generated so that the answer will fit in a 32-bit integer.

A leaf node is a node with no children.

Example1:
        1
      /   \
    2     3
Input: root = [1,2,3]
Output: 25
Explanation:
The root-to-leaf path 1->2 represents the number 12.
The root-to-leaf path 1->3 represents the number 13.
Therefore, sum = 12 + 13 = 25.

Example2:
               4
            /      \
          9         0
        /   \
      5      1
Input: root = [4,9,0,5,1]
Output: 1026
Explanation:
The root-to-leaf path 4->9->5 represents the number 495.
The root-to-leaf path 4->9->1 represents the number 491.
The root-to-leaf path 4->0 represents the number 40.
Therefore, sum = 495 + 491 + 40 = 1026.
"""

"""
Note:
1. Recursive DFS: O(n) time | O(h) space
2. Iterative DFS: O(n) time | O(h) space
3. Iterative BFS: O(n) time | O(n) space
"""

from typing import List
from collections import deque
import unittest
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    def sumNumbers(self, root: TreeNode) -> int:
        result = [0]
        self.dfs(root, 0, result)
        return result[0]

    def dfs(self, root: TreeNode, temp: int, result: List[int]):
        if not root.left and not root.right:
            result[0] += temp * 10 + root.val
            return
        if root.left:
            self.dfs(root.left, temp * 10 + root.val, result)
        if root.right:
            self.dfs(root.right, temp * 10 + root.val, result)
        return

    def sumNumbers2(self, root: TreeNode) -> int:
        total = 0
        if not root:
            return 0
        stack = [(root, 0)]
        while stack:
            node, currentSum = stack.pop()
            currentSum += node.val
            if not node.left and not node.right:
                total += currentSum
            else:
                currentSum *= 10
                if node.right:
                    stack.append((node.right, currentSum))
                if node.left:
                    stack.append((node.left, currentSum))
        return total

    def sumNumbers3(self, root: TreeNode) -> int:
        queue, result = deque([]), 0
        queue.append(root)
        while queue:
            node = queue.popleft()
            if not node.left and not node.right:
                result += node.val
            if node.left:
                node.left.val += node.val * 10
                queue.append(node.left)

            if node.right:
                node.right.val += node.val * 10
                queue.append(node.right)
        return result

# Unit Tests
funcs = [Solution().sumNumbers, Solution().sumNumbers2, Solution().sumNumbers3]

class TestSumNumbers(unittest.TestCase):
    def testSumNumbers1(self):
        for func in funcs:
            root = TreeNode(1, TreeNode(2), TreeNode(3))
            self.assertEqual(func(root=root), 25)

    def testSumNumbers2(self):
        for func in funcs:
            root = TreeNode(4, TreeNode(
                9, TreeNode(5), TreeNode(1)), TreeNode(0))
            self.assertEqual(func(root=root), 1026)

if __name__ == "__main__":
    unittest.main()


import base64
import collections
import contextlib
import email.message
import http.client
import http.server
import json
import socket
import threading
import time
from typing import BinaryIO
from typing import cast
from typing import Dict
from typing import Iterable
from typing import Iterator
from typing import Tuple
from typing import Union
import unittest
import unittest.mock
import urllib.parse

import http_proxy

LEN = 9000
CHUNK = 1013
DATA = bytes(i % 256 for i in range(LEN))

def get_chunks() -> Iterator[bytes]:
    """Yields fixture data in arbitrary-sized chunks."""
    for i in range(0, LEN, CHUNK):
        yield DATA[i : i + CHUNK]

def parse_headers(fp: BinaryIO) -> http.client.HTTPMessage:
    # mypy 0.790's typeshed is missing this function
    return http.client.parse_headers(fp)  # type: ignore

class FixtureHandler(http.server.BaseHTTPRequestHandler):
    """Handler class for HTTP server test cases."""

    def path_empty(self) -> None:
        """Send an empty response with Content-Length: 0"""
        self.send_response(200)
        self.send_header("Connection", "close")
        self.send_header("Content-Length", "0")
        self.end_headers()

    def path_empty_no_length(self) -> None:
        """Send an empty response with no Content-Length"""
        self.send_response(200)
        self.send_header("Connection", "close")
        self.end_headers()

    def path_empty_with_hop_by_hop(self) -> None:
        """Send an empty response with a nonstandard hop-by-hop header"""
        self.send_response(200)
        self.send_header("Connection", "close, X-Magic")
        self.send_header("X-Magic", "abcd1234")
        self.end_headers()

    def path_bad_status_line(self) -> None:
        """Send a bad HTTP response status line"""
        self.wfile.write(b"HTTP/1.1 whoopsie\r\n")

    def path_data(self) -> None:
        """Send some data with Content-Length"""
        self.send_response(200)
        self.send_header("Connection", "close")
        self.send_header("Content-Length", str(LEN))
        self.end_headers()
        self.wfile.write(DATA)

    def path_data_short(self) -> None:
        """Send some data, truncated according to its Content-Length"""
        self.send_response(200)
        self.send_header("Connection", "close")
        self.send_header("Content-Length", str(LEN))
        self.end_headers()
        self.wfile.write(DATA[: LEN // 2])

    def path_data_no_length(self) -> None:
        """Send some data with no Content-Length"""
        self.send_response(200)
        self.send_header("Connection", "close")
        self.end_headers()
        self.wfile.write(DATA)

    def path_data_with_invalid_length(self) -> None:
        """Send some data with a garbage Content-Length"""
        self.send_response(200)
        self.send_header("Connection", "close")
        self.send_header("Content-Length", "whoopsie")
        self.end_headers()
        self.wfile.write(DATA)

    def path_data_chunked(self) -> None:
        """Send some data using chunked encoding"""
        self.send_response(200)
        self.send_header("Connection", "close, Transfer-Encoding")
        self.send_header("Transfer-Encoding", "chunked")
        self.end_headers()
        for chunk in get_chunks():
            self.wfile.write(b"%x\r\n" % len(chunk))
            self.wfile.write(chunk)
            self.wfile.write(b"\r\n")
        self.wfile.write(b"0\r\n\r\n")

    def path_data_chunked_invalid(self) -> None:
        """Send some chunked encoding with an invalid chunk size"""
        self.send_response(200)
        self.send_header("Connection", "close, Transfer-Encoding")
        self.send_header("Transfer-Encoding", "chunked")
        self.end_headers()
        self.wfile.write(b"whoopsie\r\n")

    def path_data_chunked_short(self) -> None:
        """Send some truncated chunked encoding"""
        self.send_response(200)
        self.send_header("Connection", "close, Transfer-Encoding")
        self.send_header("Transfer-Encoding", "chunked")
        self.end_headers()
        self.wfile.write(b"123\r\n")

    def path_trailers(self) -> None:
        """Send some data using chunked encoding with HTTP trailers"""
        self.send_response(200)
        self.send_header("Connection", "close, Transfer-Encoding")
        self.send_header("Transfer-Encoding", "chunked")
        self.send_header("Trailer", "Checksum")
        self.end_headers()
        for chunk in get_chunks():
            self.wfile.write(b"%x\r\n" % len(chunk))
            self.wfile.write(chunk)
            self.wfile.write(b"\r\n")
        self.wfile.write(b"0\r\n")
        self.wfile.write(b"Checksum: abc123\r\n")
        self.wfile.write(b"\r\n")

    def path_reflect_headers(self) -> None:
        """Send the received headers and status line as JSON"""
        data_dict = {
            "requestline": self.requestline,
            "headers": dict(self.headers.items()),
        }
        data = json.dumps(data_dict).encode()
        self.send_response(200)
        self.send_header("Connection", "close")
        self.send_header("Content-Length", str(len(data)))
        self.end_headers()
        self.wfile.write(data)

    def path_reflect_data(self) -> None:
        """Send the received request body, transforming chunked encoding"""
        data = b""
        if self.headers.get("Transfer-Encoding", "identity") != "identity":
            while True:
                length = int(self.rfile.readline(), 16)
                if length == 0:
                    while True:
                        line = self.rfile.readline()
                        if line in (b"\r\n", b"", b"\n"):
                            break
                    break
                data += self.rfile.read(length)
                self.rfile.read(2)
        elif "Content-Length" in self.headers:
            length = int(self.headers["Content-Length"])
            data = self.rfile.read(length)
        else:
            data = self.rfile.read()
        self.send_response(200)
        self.send_header("Connection", "close")
        self.send_header("Content-Length", str(len(data)))
        self.end_headers()
        self.wfile.write(data)

    def path_forbidden(self) -> None:
        """Send a 403 response"""
        self.send_response(403)
        self.send_header("Connection", "close")
        self.end_headers()

    def dispatch(self) -> None:
        """Generic implementation for the do_* handler methods"""
        path = self.path[1:]
        handler = getattr(self, "path_" + path, None)
        if handler:
            handler()
        else:
            self.send_error(404)

    def do_GET(self) -> None:  # noqa: N802
        """Handle a test GET request"""
        self.dispatch()

    def do_PUT(self) -> None:  # noqa: N802
        """Handle a test PUT request"""
        self.dispatch()

    def do_PATCH(self) -> None:  # noqa: N802
        """Handle a test PATCH request"""
        self.dispatch()

    def do_POST(self) -> None:  # noqa: N802
        """Handle a test POST request"""
        self.dispatch()

    def do_HEAD(self) -> None:  # noqa: N802
        """Handle a test HEAD request"""
        self.dispatch()

    def do_OPTIONS(self) -> None:  # noqa: N802
        """Handle a test OPTIONS request"""
        self.dispatch()

    def do_TRACE(self) -> None:  # noqa: N802
        """Handle a test TRACE request"""
        self.dispatch()

class BaseTest(unittest.TestCase):
    """Base class with common utility functions"""

    def setUp(self) -> None:
        self.proxy = http.server.HTTPServer(
            ("localhost", 0), http_proxy.Handler
        )
        self.proxy_thread = threading.Thread(
            name="proxy", target=self.proxy.serve_forever, daemon=True
        )
        self.httpd = http.server.HTTPServer(("localhost", 0), FixtureHandler)
        self.httpd_thread = threading.Thread(
            name="httpd", target=self.httpd.serve_forever, daemon=True
        )

    def get_conn(self) -> http.client.HTTPConnection:
        """Returns a HTTPConnection to the proxy"""
        host, port = self.proxy.socket.getsockname()
        return http.client.HTTPConnection(host, port=port)

    def putrequest_proxy_to_httpd(
        self, conn: http.client.HTTPConnection, method: str, path: str
    ) -> None:
        host, port = self.httpd.socket.getsockname()
        url = urllib.parse.urlunsplit(
            ("http", "%s:%d" % (host, port), path, None, None)
        )
        conn.putrequest(method, url)

    def check_header_sanity(self, headers: email.message.Message) -> None:
        """Checks response headers for sanity.

        Args:
            headers: A multi-valued dict (email.message.Message subclass) of
                response headers
        """
        counter = collections.Counter(headers.keys())
        duplicates = {
            name: value for name, value in counter.items() if value > 1
        }
        self.assertEqual(duplicates, {})

class HTTPProxyTest(BaseTest):
    """Tests for normal HTTP methods (not CONNECT)."""

    def setUp(self) -> None:
        super().setUp()
        self.proxy_thread.start()
        self.httpd_thread.start()

    def tearDown(self) -> None:
        super().tearDown()
        self.proxy.shutdown()
        self.httpd.shutdown()

    def do_request(
        self,
        method: str,
        path: str,
        headers: Dict[str, str] = None,
        message_body: Union[bytes, Iterable[bytes], BinaryIO] = None,
        encode_chunked: bool = False,
    ) -> http.client.HTTPConnection:
        """Execute a proxied request to the test webserver.

        Args:
            method: The HTTP method string to use
            path: The relative path on the test webserver
            headers: A dict of headers to send
            message_body: An iterable of bytes, or file-like, or bytes.
            encode_chunked: Whether to write message_body using chunked
                encoding.

        Returns:
            An HTTPConnection, with the given request already sent
        """
        conn = self.get_conn()
        self.putrequest_proxy_to_httpd(conn, method, path)
        headers = dict(headers or {})
        connection_header = "close"
        if "Connection" in headers:
            connection_header += ", " + headers["Connection"]
        headers["Connection"] = connection_header
        for name, value in (headers or {}).items():
            conn.putheader(name, value)
        conn.endheaders(
            message_body=message_body, encode_chunked=encode_chunked
        )
        return conn

    def test_get_bad_status_line(self) -> None:
        response = self.do_request("GET", "/bad_status_line").getresponse()
        self.assertEqual(response.status, 502)
        self.check_header_sanity(response.headers)

    def test_get_forbidden(self) -> None:
        response = self.do_request("GET", "/forbidden").getresponse()
        self.assertEqual(response.status, 403)
        self.check_header_sanity(response.headers)

    def test_nonexistent_gateway(self) -> None:
        conn = self.get_conn()
        conn.putrequest("GET", "http://does-not-exist/")
        conn.endheaders()
        response = conn.getresponse()
        self.assertEqual(response.status, 502)
        self.check_header_sanity(response.headers)

    def test_gateway_timeout(self) -> None:
        # Create a listening socket that sends nothing
        server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server.settimeout(10)
        server.bind(("localhost", 0))
        host, port = server.getsockname()
        server.listen(10)

        # Set a low timeout
        http_proxy.Handler.timeout = 1

        with unittest.mock.patch("http_proxy.Handler.timeout", 1):
            conn = self.get_conn()
            conn.putrequest("GET", "http://%s:%d/" % (host, port))
            conn.endheaders()
            response = conn.getresponse()
            self.assertEqual(response.status, 504)
            self.check_header_sanity(response.headers)

    def test_non_http_url(self) -> None:
        conn = self.get_conn()
        conn.putrequest("GET", "https://example.com/")
        conn.endheaders()
        response = conn.getresponse()
        self.assertEqual(response.status, 400)
        self.check_header_sanity(response.headers)

    def test_invalid_http_url(self) -> None:
        conn = self.get_conn()
        conn.putrequest("GET", "http://example.com:whoopsie/")
        conn.endheaders()
        response = conn.getresponse()
        self.assertEqual(response.status, 400)
        self.check_header_sanity(response.headers)

    def test_get_empty(self) -> None:
        response = self.do_request("GET", "/empty").getresponse()
        self.assertEqual(response.status, 200)
        self.check_header_sanity(response.headers)
        self.assertEqual(response.getheader("Content-Length"), "0")
        self.assertEqual(response.read(), b"")

    def test_get_empty_no_length(self) -> None:
        response = self.do_request("GET", "/empty_no_length").getresponse()
        self.assertEqual(response.status, 200)
        self.check_header_sanity(response.headers)
        self.assertEqual(response.getheader("Content-Length"), None)
        self.assertEqual(response.read(), b"")

    def test_get_empty_with_hop_by_hop(self) -> None:
        response = self.do_request(
            "GET", "/empty_with_hop_by_hop"
        ).getresponse()
        self.assertEqual(response.status, 200)
        self.check_header_sanity(response.headers)
        # HTTPMessage not a Container in mypy 0.790
        self.assertNotIn("X-Magic", response.headers)  # type: ignore

    def test_get_data(self) -> None:
        response = self.do_request("GET", "/data").getresponse()
        self.assertEqual(response.status, 200)
        self.check_header_sanity(response.headers)
        self.assertEqual(response.getheader("Content-Length"), str(LEN))
        self.assertEqual(response.read(), DATA)

    def test_get_data_with_invalid_length(self) -> None:
        response = self.do_request(
            "GET", "/data_with_invalid_length"
        ).getresponse()
        self.assertEqual(response.status, 200)
        self.check_header_sanity(response.headers)
        self.assertEqual(response.read(), DATA)

    def test_get_data_short(self) -> None:
        response = self.do_request("GET", "/data_short").getresponse()
        self.assertEqual(response.status, 200)
        self.check_header_sanity(response.headers)
        self.assertEqual(response.getheader("Content-Length"), str(LEN))
        with self.assertRaises(http.client.IncompleteRead):
            response.read()

    def test_get_data_no_length(self) -> None:
        response = self.do_request("GET", "/data_no_length").getresponse()
        self.assertEqual(response.status, 200)
        self.check_header_sanity(response.headers)
        self.assertEqual(response.getheader("Content-Length"), None)
        self.assertEqual(response.read(), DATA)

    def test_get_data_chunked(self) -> None:
        response = self.do_request("GET", "/data_chunked").getresponse()
        self.assertEqual(response.status, 200)
        self.check_header_sanity(response.headers)
        self.assertEqual(response.getheader("Content-Length"), None)
        self.assertEqual(response.read(), DATA)

    def test_get_data_chunked_invalid(self) -> None:
        response = self.do_request(
            "GET", "/data_chunked_invalid"
        ).getresponse()
        self.assertEqual(response.status, 200)
        self.check_header_sanity(response.headers)
        self.assertEqual(response.getheader("Content-Length"), None)
        with self.assertRaises(http.client.IncompleteRead):
            response.read()

    def test_get_data_chunked_short(self) -> None:
        response = self.do_request("GET", "/data_chunked_short").getresponse()
        self.assertEqual(response.status, 200)
        self.check_header_sanity(response.headers)
        self.assertEqual(response.getheader("Content-Length"), None)
        with self.assertRaises(http.client.IncompleteRead):
            response.read()

    def test_basic_auth(self) -> None:
        with unittest.mock.patch("http_proxy.Handler.basic_auth", "secret"):
            headers = {"Proxy-Authorization": "Basic secret"}
            response = self.do_request(
                "GET", "/data", headers=headers
            ).getresponse()
            self.assertEqual(response.status, 200)
            self.check_header_sanity(response.headers)
            self.assertEqual(response.getheader("Content-Length"), str(LEN))
            self.assertEqual(response.read(), DATA)

    def test_basic_auth_fail(self) -> None:
        with unittest.mock.patch("http_proxy.Handler.basic_auth", "secret"):
            headers = {"Proxy-Authorization": "Basic wrong"}
            response = self.do_request(
                "GET", "/data", headers=headers
            ).getresponse()
            self.assertEqual(response.status, 407)
            self.check_header_sanity(response.headers)

    def test_basic_auth_absent(self) -> None:
        with unittest.mock.patch("http_proxy.Handler.basic_auth", "secret"):
            response = self.do_request("GET", "/data").getresponse()
            self.assertEqual(response.status, 407)
            self.check_header_sanity(response.headers)

    def test_basic_auth_malformed(self) -> None:
        with unittest.mock.patch("http_proxy.Handler.basic_auth", "secret"):
            headers = {"Proxy-Authorization": "malformed"}
            response = self.do_request(
                "GET", "/data", headers=headers
            ).getresponse()
            self.assertEqual(response.status, 407)
            self.check_header_sanity(response.headers)

    def test_basic_auth_wrong_scheme(self) -> None:
        with unittest.mock.patch("http_proxy.Handler.basic_auth", "secret"):
            headers = {"Proxy-Authorization": "Digest secret"}
            response = self.do_request(
                "GET", "/data", headers=headers
            ).getresponse()
            self.assertEqual(response.status, 407)
            self.check_header_sanity(response.headers)

    def test_trailers(self) -> None:
        response = self.do_request("GET", "/trailers").getresponse()
        self.assertEqual(response.status, 200)
        self.check_header_sanity(response.headers)
        self.assertEqual(response.getheader("Content-Length"), None)
        # http.client doesn't actually parse trailers, but at least test
        # they're processed correctly
        self.assertNotEqual(response.getheader("Trailer"), None)
        self.assertEqual(response.read(), DATA)

    def test_reflect_headers(self) -> None:
        headers = {
            "X-End-To-End": "foo",
            "X-Hop-By-Hop": "bar",
            "Connection": "close, X-Hop-By-Hop",
        }
        response = self.do_request(
            "GET", "/reflect_headers", headers=headers
        ).getresponse()
        self.assertEqual(response.status, 200)
        self.check_header_sanity(response.headers)
        data = json.loads(response.read().decode())
        seen_headers = data["headers"]
        self.assertEqual(
            seen_headers.get("X-End-To-End"), headers.get("X-End-To-End")
        )
        self.assertNotIn("X-Hop-By-Hop", seen_headers)

    def test_reflect_data(self) -> None:
        response = self.do_request(
            "POST",
            "/reflect_data",
            headers={"Content-Length": str(LEN)},
            message_body=DATA,
        ).getresponse()
        self.assertEqual(response.status, 200)
        self.check_header_sanity(response.headers)
        data = response.read()
        self.assertEqual(data, DATA)

    def test_reflect_data_chunked(self) -> None:
        response = self.do_request(
            "POST",
            "/reflect_data",
            headers={"Transfer-Encoding": "chunked"},
            message_body=get_chunks(),
            encode_chunked=True,
        ).getresponse()
        self.assertEqual(response.status, 200)
        self.check_header_sanity(response.headers)
        data = response.read()
        self.assertEqual(data, DATA)

    def test_body_bad_length(self) -> None:
        response = self.do_request(
            "POST", "/empty", headers={"Content-Length": "whoopsie"}
        ).getresponse()
        self.assertEqual(response.status, 411)
        self.check_header_sanity(response.headers)

    def test_body_required(self) -> None:
        response = self.do_request("POST", "/empty").getresponse()
        self.assertEqual(response.status, 411)
        self.check_header_sanity(response.headers)

    def test_body_invalid_chunk(self) -> None:
        response = self.do_request(
            "POST",
            "/empty",
            headers={"Transfer-Encoding": "chunked"},
            message_body=b"whoopsie\r\n",
        ).getresponse()
        self.assertEqual(response.status, 400)
        self.check_header_sanity(response.headers)

    def test_pipelining(self) -> None:
        # python's http.client doesn't support pipelining!
        # NB: this could hang depending on socket buffer sizes.
        proxy_address = self.proxy.socket.getsockname()
        sock = socket.create_connection(proxy_address, 30)
        rfile = sock.makefile("rb")
        httpd_host = ("%s:%d" % self.httpd.socket.getsockname()).encode()
        sock.sendall(b"GET http://%s/data HTTP/1.1\r\n" % httpd_host)
        sock.sendall(b"\r\n")
        sock.sendall(b"POST http://%s/reflect_data HTTP/1.1\r\n" % httpd_host)
        sock.sendall(b"Content-Length: %d\r\n" % LEN)
        sock.sendall(b"\r\n")
        sock.sendall(DATA)
        sock.sendall(b"PUT http://%s/reflect_data HTTP/1.1\r\n" % httpd_host)
        sock.sendall(b"Transfer-Encoding: chunked\r\n")
        sock.sendall(b"Connection: Transfer-Encoding\r\n")
        sock.sendall(b"\r\n")
        for chunk in get_chunks():
            sock.sendall(b"%x\r\n" % len(chunk))
            sock.sendall(chunk)
            sock.sendall(b"\r\n")
        sock.sendall(b"0\r\n")
        sock.sendall(b"\r\n")
        sock.sendall(b"GET http://%s/data_chunked HTTP/1.1\r\n" % httpd_host)
        sock.sendall(b"\r\n")
        sock.sendall(b"HEAD http://%s/forbidden HTTP/1.1\r\n" % httpd_host)
        sock.sendall(b"Connection: close\r\n")
        sock.sendall(b"\r\n")

        def read_status(file_like: BinaryIO) -> Tuple[str, int, str]:
            line = file_like.readline().decode()
            version, status, reason = line.split(None, 2)
            return version, int(status), reason

        def read_chunks_discard_trailer(
            file_like: BinaryIO,
        ) -> Iterator[bytes]:
            while True:
                size_line = file_like.readline()
                size = int(size_line, 16)
                if size == 0:
                    while True:
                        line = file_like.readline()
                        if line in (b"\r\n", b"", b"\n"):
                            return
                chunk = file_like.read(size)
                self.assertEqual(len(chunk), size)
                yield chunk
                file_like.read(2)

        version, status, _ = read_status(rfile)
        self.assertEqual(version, "HTTP/1.1")
        self.assertEqual(status, 200)
        headers = parse_headers(rfile)
        self.check_header_sanity(headers)
        self.assertEqual(headers["Content-Length"], str(LEN))
        data = rfile.read(LEN)
        self.assertEqual(data, DATA)

        version, status, _ = read_status(rfile)
        self.assertEqual(version, "HTTP/1.1")
        self.assertEqual(status, 200)
        headers = parse_headers(rfile)
        self.check_header_sanity(headers)
        self.assertEqual(headers["Content-Length"], str(LEN))
        data = rfile.read(LEN)
        self.assertEqual(data, DATA)

        version, status, _ = read_status(rfile)
        self.assertEqual(version, "HTTP/1.1")
        self.assertEqual(status, 200)
        headers = parse_headers(rfile)
        self.check_header_sanity(headers)
        self.assertEqual(headers["Content-Length"], str(LEN))
        data = rfile.read(LEN)
        self.assertEqual(data, DATA)

        version, status, _ = read_status(rfile)
        self.assertEqual(version, "HTTP/1.1")
        self.assertEqual(status, 200)
        headers = parse_headers(rfile)
        self.check_header_sanity(headers)
        self.assertEqual(headers["Transfer-Encoding"], "chunked")
        data = b"".join(read_chunks_discard_trailer(rfile))
        self.assertEqual(data, DATA)

        version, status, _ = read_status(rfile)
        self.assertEqual(version, "HTTP/1.1")
        self.assertEqual(status, 403)
        headers = parse_headers(rfile)
        self.check_header_sanity(headers)

        # Test the socket gets closed
        self.assertEqual(rfile.read(), b"")

class ConnectMethodTest(BaseTest):
    """Tests for the CONNECT method"""

    def do_connect_obj(
        self, path: str, headers: Dict[str, str] = None
    ) -> http.client.HTTPConnection:
        """Execute a proxied CONNECT request.

        Args:
            path: The CONNECT method target (normally host:port)
            headers: A dict of headers to send

        Returns:
            An HTTPConnection with the given request already sent
        """
        # http.client doesn't actually support bidirectional CONNECT streaming,
        # but it's nice for testing headers
        conn = self.get_conn()
        conn.putrequest("CONNECT", path)
        headers = dict(headers or {})
        connection_header = "close"
        if "Connection" in headers:
            connection_header += ", " + headers["Connection"]
        headers["Connection"] = connection_header
        for name, value in (headers or {}).items():
            conn.putheader(name, value)
        conn.endheaders()
        return conn

    def do_connect_obj_to_server(
        self, headers: Dict[str, str] = None
    ) -> http.client.HTTPConnection:
        """Execute a proxied CONNECT request to the fixture server socket.

        Args:
            headers: A dict of headers to send

        Returns:
            An HTTPConnection with the given request already sent
        """
        return self.do_connect_obj(
            "%s:%d" % self.server.getsockname(), headers=headers
        )

    def make_connected_pair(self) -> Tuple[socket.socket, socket.socket]:
        """Establish a proxied pair of connected sockets.

        Executes a CONNECT request to the proxy, to the fixture server socket.
        We will assert that the request was successful and the response was
        valid, then return the sockets on either end of the connection.

        Returns:
            A tuple of socket objects; the first is the client connection to
                the proxy, and the second is the server-side incoming
                connection.
        """
        proxy_address = self.proxy.socket.getsockname()
        c_to_s = socket.create_connection(proxy_address, 30)
        server_host, server_port = self.server.getsockname()
        c_to_s.sendall(
            b"CONNECT %s:%d HTTP/1.1\r\n" % (server_host.encode(), server_port)
        )
        c_to_s.sendall(b"\r\n")

        s_to_c, _ = self.server.accept()

        with c_to_s.makefile("rb") as c_rfile:
            version, status_str, _ = c_rfile.readline().decode().split(None, 2)
            status = int(status_str)
            self.assertEqual(version, "HTTP/1.1")
            self.assertGreaterEqual(status, 200)
            self.assertLess(status, 300)
            headers = parse_headers(c_rfile)
            self.check_header_sanity(headers)
            self.assertEqual(headers["Connection"], "close")

        return c_to_s, s_to_c

    def setUp(self) -> None:
        super().setUp()
        self.proxy_thread.start()
        self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.server.bind(("localhost", 0))
        self.server.listen()

    def tearDown(self) -> None:
        super().tearDown()
        self.proxy.shutdown()
        self.server.close()

    def test_basic_auth(self) -> None:
        with unittest.mock.patch("http_proxy.Handler.basic_auth", "secret"):
            headers = {"Proxy-Authorization": "Basic secret"}
            response = self.do_connect_obj_to_server(
                headers=headers
            ).getresponse()
            self.assertEqual(response.status, 200)
            self.check_header_sanity(response.headers)

    def test_basic_auth_fail(self) -> None:
        with unittest.mock.patch("http_proxy.Handler.basic_auth", "secret"):
            headers = {"Proxy-Authorization": "Basic wrong"}
            response = self.do_connect_obj_to_server(
                headers=headers
            ).getresponse()
            self.assertEqual(response.status, 407)
            self.check_header_sanity(response.headers)

    def test_basic_auth_malformed(self) -> None:
        with unittest.mock.patch("http_proxy.Handler.basic_auth", "secret"):
            headers = {"Proxy-Authorization": "malformed"}
            response = self.do_connect_obj_to_server(
                headers=headers
            ).getresponse()
            self.assertEqual(response.status, 407)
            self.check_header_sanity(response.headers)

    def test_basic_auth_wrong_scheme(self) -> None:
        with unittest.mock.patch("http_proxy.Handler.basic_auth", "secret"):
            headers = {"Proxy-Authorization": "Digest secret"}
            response = self.do_connect_obj_to_server(
                headers=headers
            ).getresponse()
            self.assertEqual(response.status, 407)
            self.check_header_sanity(response.headers)

    def test_malformed(self) -> None:
        response = self.do_connect_obj("malformed").getresponse()
        self.assertGreaterEqual(response.status, 400)
        self.assertLess(response.status, 500)
        self.check_header_sanity(response.headers)

    def test_nonexistent_gateway(self) -> None:
        response = self.do_connect_obj("does-not-exist:80").getresponse()
        self.assertEqual(response.status, 502)
        self.check_header_sanity(response.headers)

    def test_gateway_timeout(self) -> None:
        with unittest.mock.patch("http_proxy.Handler.timeout", 1):
            response = self.do_connect_obj("0.0.0.1:80").getresponse()
            self.assertEqual(response.status, 504)
            self.check_header_sanity(response.headers)

    def test_shutdown_on_read_timeout(self) -> None:
        with unittest.mock.patch("http_proxy.Handler.timeout", 1):
            c_to_s, s_to_c = self.make_connected_pair()
            # Client and server send greetings, then get closed due to
            # inactivity
            s_to_c.sendall(b"abcd")
            c_to_s.sendall(b"1234")
            with s_to_c.makefile("rb") as s_rfile:
                self.assertEqual(s_rfile.read(), b"1234")
            with c_to_s.makefile("rb") as c_rfile:
                self.assertEqual(c_rfile.read(), b"abcd")

    def test_connect_and_close_upstream(self) -> None:
        c_to_s, s_to_c = self.make_connected_pair()
        c_rfile = c_to_s.makefile("rb")
        s_rfile = s_to_c.makefile("rb")

        s_to_c.sendall(b"abcd")
        self.assertEqual(c_rfile.read(4), b"abcd")

        c_to_s.sendall(b"1234")
        self.assertEqual(s_rfile.read(4), b"1234")

        s_to_c.sendall(b"goodbye")
        # Note: must close socket *and* makefile()s
        s_to_c.close()
        s_rfile.close()
        self.assertEqual(c_rfile.read(), b"goodbye")

    def test_connect_and_close_client(self) -> None:
        c_to_s, s_to_c = self.make_connected_pair()
        c_rfile = c_to_s.makefile("rb")
        s_rfile = s_to_c.makefile("rb")

        s_to_c.sendall(b"abcd")
        self.assertEqual(c_rfile.read(4), b"abcd")

        c_to_s.sendall(b"1234")
        self.assertEqual(s_rfile.read(4), b"1234")

        c_to_s.sendall(b"goodbye")
        # Note: must close socket *and* makefile()s
        c_to_s.close()
        c_rfile.close()
        self.assertEqual(s_rfile.read(), b"goodbye")

class MainTest(BaseTest):
    def setUp(self) -> None:
        super().setUp()
        self.httpd_thread.start()
        self.main = http_proxy.Main()
        self.main_thread = threading.Thread(
            name="main", target=self.main.run, daemon=True
        )

    def tearDown(self) -> None:
        super().tearDown()
        self.httpd.shutdown()
        self.main.shutdown()

    def get_unused_address(self) -> Tuple[str, int]:
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
            sock.bind(("localhost", 0))
            return cast(Tuple[str, int], sock.getsockname())

    @contextlib.contextmanager
    def start_and_connect(
        self, *args: str
    ) -> Iterator[http.client.HTTPConnection]:
        host, port = self.get_unused_address()
        argv = ["http_proxy", "--bind-host", host, "--port", str(port)]
        argv += args
        with unittest.mock.patch("sys.argv", argv):
            self.main_thread.start()
            deadline = time.time() + 5
            while True:
                try:
                    sock = socket.create_connection((host, port))
                    sock.close()
                    break
                except OSError:
                    pass
                self.assertLess(time.time(), deadline, msg="startup timed out")
                time.sleep(0.1)
            yield http.client.HTTPConnection(host, port=port)

    def test_normal_startup(self) -> None:
        with self.start_and_connect() as conn:
            self.putrequest_proxy_to_httpd(conn, "GET", "/empty")
            conn.endheaders()
            response = conn.getresponse()
            self.assertEqual(response.status, 200)
            self.check_header_sanity(response.headers)
            self.assertEqual(response.getheader("Content-Length"), "0")
            self.assertEqual(response.read(), b"")

    def test_auth_fail(self) -> None:
        with self.start_and_connect("--basic-auth", "test:test") as conn:
            self.putrequest_proxy_to_httpd(conn, "GET", "/empty")
            conn.endheaders()
            response = conn.getresponse()
            self.assertEqual(response.status, 407)
            self.check_header_sanity(response.headers)

    def test_auth_pass(self) -> None:
        with self.start_and_connect("--basic-auth", "test:test") as conn:
            self.putrequest_proxy_to_httpd(conn, "GET", "/empty")
            conn.putheader(
                "Proxy-Authorization",
                "Basic %s" % base64.b64encode("test:test".encode()).decode(),
            )
            conn.endheaders()
            response = conn.getresponse()
            self.assertEqual(response.status, 200)
            self.check_header_sanity(response.headers)
            self.assertEqual(response.getheader("Content-Length"), "0")
            self.assertEqual(response.read(), b"")


import random
import math
import sys
import numpy as np
from PIL import Image, ImageDraw

def cartesianProduct(*arrays):
    la = len(arrays)
    dtype = np.result_type(*arrays)
    arr = np.empty([len(a) for a in arrays] + [la], dtype=dtype)
    for i, a in enumerate(np.ix_(*arrays)):
        arr[...,i] = a
    return arr.reshape(-1, la)

def drawBox(position, color, gameDisplay):
  gameDisplay.rectangle([(position[0]*15, position[1]*15), (((position[0]+1)*15)-1, ((position[1]+1)*15)-1)], color)

def drawTriangle(position, color, gameDisplay):
  basePixel = (position[0] * 15, position[1] * 15)
  gameDisplay.polygon([(basePixel[0]+7, basePixel[1]), (basePixel[0], basePixel[1]+14), (basePixel[0]+14, basePixel[1]+14)], color)

def generateSimHabitat(carnivores):
  #Do Calculations
  smallCarnivore = random.randint(0, carnivores)
  largeCarnivore = carnivores - smallCarnivore

  smallHerbivoresForLarge = random.randint(0, largeCarnivore)
  smallHerbivore = (smallCarnivore * 2) + (smallHerbivoresForLarge * 4)
  largeHerbivore = (largeCarnivore-smallHerbivoresForLarge) * 2

  treesForSmall = random.randint(0, smallHerbivore)
  herbaceousPlantsForSmall = random.randint(0, smallHerbivore-treesForSmall)
  grassesForSmall = smallHerbivore-herbaceousPlantsForSmall-treesForSmall
  treesForLarge = random.randint(0, largeHerbivore)
  herbaceousPlantsForLarge = random.randint(0, largeHerbivore-treesForLarge)
  grassesForLarge = largeHerbivore-herbaceousPlantsForLarge-treesForLarge
  trees = treesForSmall + (treesForLarge * 2)
  herbaceousPlants = (herbaceousPlantsForSmall * 2) + (herbaceousPlantsForLarge * 4)
  grasses = (grassesForSmall * 4) + (grassesForLarge * 8)

  water = (trees * 3) + (herbaceousPlants * 2) + (grasses * 1)

  decomposers = grasses + herbaceousPlants + trees + largeHerbivore + smallHerbivore + largeCarnivore + smallCarnivore
  print()
  print("Number of Decomposers:", decomposers)
  print("Number of Water:", water)
  print("Number of Trees:", trees)
  print("Number of Herbaceous Plants:", herbaceousPlants)
  print("Number of Grasses:", grasses)
  print("Number of Small Herbivores:", smallHerbivore)
  print("Number of Large Herbivores:", largeHerbivore)
  print("Number of Small Carnivores:", smallCarnivore)
  print("Number of Large Carnivores:", largeCarnivore)

  numberOfSquaresNeeded = int(sum([decomposers, water, trees, herbaceousPlants, grasses, smallHerbivore, largeHerbivore, smallCarnivore, largeCarnivore])*1.3)
  print(numberOfSquaresNeeded)
  pixelsPerBox = 15
  pixelsDimension = math.ceil(numberOfSquaresNeeded ** 0.5) * pixelsPerBox
  print(pixelsDimension)
  numberOfSquaresNeeded = int((pixelsDimension / pixelsPerBox) ** 2)
  print(numberOfSquaresNeeded)

  #Generate Image
  black = (0,0,0)
  white = (255,255,255)
  red = (255,0,0)
  green = (0, 255, 0)
  blue = (0, 0, 255)
  brown = (200, 100, 100)
  darkGreen = (0, 150, 0)
  yellow = (255, 255, 0)

  img = Image.new("RGB", (pixelsDimension, pixelsDimension), color = white)
  gameDisplay = ImageDraw.Draw(img)

  dimension = np.array([i for i in range(int(numberOfSquaresNeeded ** 0.5))])
  print('done')
  squareCoords = cartesianProduct(dimension, dimension)
  np.random.shuffle(squareCoords)

  print(1)
  index = 0
  for i in range(decomposers):
    drawBox(squareCoords[index], brown, gameDisplay)
    index += 1
  print(2)
  for i in range(water):
    drawBox(squareCoords[index], blue, gameDisplay)
    index += 1
  print(3)
  for i in range(trees):
    drawBox(squareCoords[index], darkGreen, gameDisplay)
    index += 1
  print(4)
  for i in range(grasses):
    drawBox(squareCoords[index], green, gameDisplay)
    index += 1
  print(5)
  for i in range(herbaceousPlants):
    drawTriangle(squareCoords[index], green, gameDisplay)
    index += 1
  print(6)
  for i in range(smallHerbivore):
    drawTriangle(squareCoords[index], yellow, gameDisplay)
    index += 1
  print(7)
  for i in range(largeHerbivore):
    drawBox(squareCoords[index], yellow, gameDisplay)
    index += 1
  print(8)
  for i in range(smallCarnivore):
    drawTriangle(squareCoords[index], red, gameDisplay)
    index += 1
  print(9)
  for i in range(largeCarnivore):
    drawBox(squareCoords[index], red, gameDisplay)
    index += 1
  print(10)
  print('Saving PNG')
  img.save(f"SimHabitat_{carnivores}.png")

generateSimHabitat(int(input('Number of carnivores: ')))