import warnings
import requests
import os
import twill
import time
import io

import numpy as np
import pandas as pd

from memilio.epidata import defaultDict as dd
from memilio.epidata import geoModificationGermany as geoger
from memilio.epidata import getDataIntoPandasDataFrame as gd

# activate CoW for more predictable behaviour of pandas DataFrames
pd.options.mode.copy_on_write = True

def read_population_data(username, password, read_data, directory):
    '''! Reads Population data either from regionalstatistik.de or from directory

    A request is made using the twill package. Username and Password are required to
    sign in on regionalstatistik.de. After the sign twill navigates to the file to download.

    @param username Username to sign in at regionalstatistik.de.
    @param password Password to sign in at regionalstatistik.de.
    @param read_data False or True. Defines if data is read from file or downloaded.
    @param directory Path to folder where data is read from.
    @return DataFrame
    '''

    filename = '12411-02-03-4'
    if not read_data:
        sign_in_url = 'https://www.regionalstatistik.de/genesis/online?Menu=Anmeldung'

        # sign in to regionalstatistik.de with given username and password
        twill.browser.user_agent = requests.utils.default_headers()[
            'User-Agent']
        twill.commands.go(sign_in_url)
        twill.commands.fv('3', 'KENNUNG', username)
        twill.commands.fv('3', 'PASSWORT', password)
        twill.commands.submit('login', '3')
        # navigate to file as in documentation
        twill.commands.follow('Themen')
        twill.commands.follow(filename[:2])
        # wait 2 seconds to prevent error
        # page needs some time to load
        time.sleep(2)
        twill.commands.follow(filename.split('-')[0])
        twill.commands.follow(filename)
        # start 'Werteabruf'
        twill.commands.submit('45', '3')
        # read csv file (1,4 for xlsx)
        twill.commands.submit('1', '5')

        df_pop_raw = pd.read_csv(io.StringIO(
            twill.browser.html), sep=';', header=6)

    else:
        data_file = os.path.join(directory, filename)
        if os.path.isfile(data_file+'.xlsx'):
            df_pop_raw = pd.read_excel(
                data_file+'.xlsx', engine='openpyxl', sheet_name=filename, header=4)
        elif os.path.isfile(data_file+'.csv'):
            df_pop_raw = pd.read_excel(data_file+'.csv', sep=';', header=6)
        else:
            raise FileNotFoundError(
                'Data file '+filename+' was not found in out_folder/Germany')

    return df_pop_raw

def export_population_dataframe(df_pop, directory, file_format, merge_eisenach):
    '''! Writes population dataframe into directory with new column names and age groups

    @param df_pop Population data DataFrame to be exported
    @param directory Directory where data is written to.
    @param file_format File format which is used for writing the data.
    @param merge_eisenach Defines whether the counties 'Wartburgkreis'
        and 'Eisenach' are listed separately or
        combined as one entity 'Wartburgkreis'.
    @return exported DataFrame
    '''

    new_cols = [
        dd.EngEng['idCounty'],
        dd.EngEng['population'],
        '<3 years', '3-5 years', '6-14 years', '15-17 years',
        '18-24 years', '25-29 years', '30-39 years', '40-49 years',
        '50-64 years', '65-74 years', '>74 years']
    df_pop_export = pd.DataFrame(columns=new_cols)
    df_pop_export[new_cols[0]] = df_pop[dd.EngEng['idCounty']]
    # <3 and 3-5
    df_pop_export[new_cols[2:4]] = df_pop[df_pop.columns[2:4]]
    # 6-14
    df_pop_export[new_cols[4]] = \
        df_pop[df_pop.columns[4:6]].sum(axis=1)
    # 15-17
    df_pop_export[new_cols[5]] = df_pop[df_pop.columns[6]]
    # 18-24
    df_pop_export[new_cols[6]] = \
        df_pop[df_pop.columns[7:9]].sum(axis=1)
    # 25-29
    df_pop_export[new_cols[7]] = df_pop[df_pop.columns[9]]
    # 30-39
    df_pop_export[new_cols[8]] = \
        df_pop[df_pop.columns[10:12]].sum(axis=1)
    # 40-49
    df_pop_export[new_cols[9]] = \
        df_pop[df_pop.columns[12:14]].sum(axis=1)
    # 50-64
    df_pop_export[new_cols[10]] = \
        df_pop[df_pop.columns[14:17]].sum(axis=1)
    # 65-74
    df_pop_export[new_cols[11]] = df_pop[df_pop.columns[17]]
    # >74
    df_pop_export[new_cols[12]] = df_pop[df_pop.columns[18]]

    df_pop_export[dd.EngEng['population']
                  ] = df_pop_export.iloc[:, 2:].sum(axis=1)

    # merge eisenach if no data available
    if '16056' in df_pop_export[dd.EngEng['idCounty']].values:
        if df_pop_export[df_pop_export[dd.EngEng['idCounty']] == '16056'][dd.EngEng['population']].values[0] == 0:
            df_pop_export = geoger.merge_df_counties_all(
                df_pop_export, sorting=[dd.EngEng["idCounty"]],
                columns=dd.EngEng["idCounty"])

    gd.check_dir(directory)

    if len(df_pop_export) == 401:
        filename = 'county_current_population_dim401'
        gd.write_dataframe(df_pop_export, directory, filename, file_format)

    if len(df_pop_export) == 400 or merge_eisenach:
        filename = 'county_current_population'

        # Merge Eisenach and Wartburgkreis
        df_pop_export = geoger.merge_df_counties_all(
            df_pop_export, sorting=[dd.EngEng["idCounty"]],
            columns=dd.EngEng["idCounty"])

        gd.write_dataframe(df_pop_export, directory, filename, file_format)

    return df_pop_export

def assign_population_data(df_pop_raw, counties, age_cols, idCounty_idx):
    '''! Assigns population data of all counties of old dataframe in new created dataframe

    In df_pop_raw there might be additional information like federal states,
    governing regions etc. which is not necessary for the dataframe.
    Also checks for incomplete data.

    @param df_pop_raw Raw Population DataFrame read from regionalstatistik.de
    @param counties List of counties to be assigned in new DataFrame
    @param age_cols Age groups in old DataFrame
    @param idCountyidx indexes in old DataFrame where data of corresponding county starts
    @return new DataFrame
    '''

    new_cols = {dd.EngEng['idCounty']: counties[:, 1],
                dd.EngEng['county']: counties[:, 0]}

    # number of age_cols
    num_age_groups = len(age_cols)

    # add age_cols with zero initialization to new_cols
    new_cols.update({age: 0 for age in age_cols})

    df_pop = pd.DataFrame(new_cols)

    empty_data = ['.', '-']
    for start_idx in idCounty_idx:

        county_id = df_pop_raw.loc[start_idx, dd.EngEng['idCounty']]

        # check for empty rows
        if df_pop_raw.loc[start_idx: start_idx + num_age_groups - 1,
                          dd.EngEng['number']].values.any() in empty_data:
            if not df_pop_raw.loc[start_idx: start_idx + num_age_groups - 1,
                                  dd.EngEng['number']].values.all() in empty_data:
                raise gd.DataError(
                    'Error. Partially incomplete data for county ' +
                    county_id)

        # county information needed
        elif county_id in counties[:, 1]:
            # direct assignment of population data found
            df_pop.loc[df_pop[dd.EngEng['idCounty']] == df_pop_raw.loc
                       [start_idx, dd.EngEng['idCounty']],
                       age_cols] = df_pop_raw.loc[start_idx: start_idx + num_age_groups - 1, dd.EngEng
                                                  ['number']].values.astype(int)
        # Berlin and Hamburg
        elif county_id + '000' in counties[:, 1]:
            # direct assignment of population data found
            df_pop.loc[df_pop[dd.EngEng['idCounty']] == df_pop_raw.loc
                       [start_idx, dd.EngEng['idCounty']] + '000',
                       age_cols] = df_pop_raw.loc[start_idx: start_idx + num_age_groups - 1, dd.EngEng
                                                  ['number']].values.astype(int)

        # additional information for local entities not needed
        elif county_id in ['03241001', '05334002', '10041100']:
            pass
        # Germany, federal states, and governing regions
        elif len(county_id) < 5:
            pass
        else:
            print('no data for ' + df_pop_raw.loc
                  [start_idx, dd.EngEng['idCounty']])
            raise gd.DataError(
                'Error. County ID in input population data '
                'found which could not be assigned.')

    return df_pop

def test_total_population(df_pop, age_cols):
    """! Tests if total population matches expectation
    @param df_pop Population Dataframe with all counties
    @param age_cols All age groups in DataFrame"""

    total_sum_2020 = 83155031
    total_sum_2021 = 83237124

    if df_pop[age_cols].sum().sum() != total_sum_2021:
        if df_pop[age_cols].sum().sum() == total_sum_2020:
            warnings.warn('Using data of 2020. Newer data is available.')
        else:
            raise gd.DataError('Total Population does not match expectation.')

def get_population_data(read_data=dd.defaultDict['read_data'],
                        file_format=dd.defaultDict['file_format'],
                        out_folder=dd.defaultDict['out_folder'],
                        no_raw=dd.defaultDict['no_raw'],
                        merge_eisenach=True,
                        username='',
                        password=''):
    """! Download age-stratified population data for the German counties.

    The data we use is:
    Official 'Bevölkerungsfortschreibung' 12411-02-03-4:
    'Bevölkerung nach Geschlecht und Altersgruppen (17)'
    of regionalstatistik.de.
    ATTENTION: The raw file cannot be downloaded
    automatically by our scripts without an Genesis Online account. In order to
    work on this dataset, please enter your username and password or manually download it from:

    https://www.regionalstatistik.de/genesis/online -> "1: Gebiet, Bevölkerung,
    Arbeitsmarkt, Wahlen" -> "12: Bevölkerung" -> "12411 Fortschreibung des
    Bevölkerungsstandes" ->  "12411-02-03-4: Bevölkerung nach Geschlecht und
    Altersgruppen (17) - Stichtag 31.12. - regionale Tiefe: Kreise und
    krfr. Städte".

    Download the xlsx or csv file and put it under dd.defaultDict['out_folder'],
    this normally is Memilio/data/pydata/Germany.
    The folders 'pydata/Germany' have to be created if they do not exist yet.
    Then this script can be run.

    @param read_data False or True. Defines if data is read from file or
        downloaded. Default defined in defaultDict.
    @param file_format File format which is used for writing the data.
        Default defined in defaultDict.
    @param out_folder Path to folder where data is written in folder
        out_folder/Germany. Default defined in defaultDict.
    @param no_raw True or False. Defines if unchanged raw data is written or
        not. Default defined in defaultDict. Currently not used.
    @param merge_eisenach [Default: True] or False. Defines whether the
        counties 'Wartburgkreis' and 'Eisenach' are listed separately or
        combined as one entity 'Wartburgkreis'.
    @param username Username to sign in at regionalstatistik.de.
    @param password Password to sign in at regionalstatistik.de.
    @return DataFrame with adjusted population data for all ages to current level.
    """

    directory = os.path.join(out_folder, 'Germany')
    gd.check_dir(directory)

    df_pop_raw = read_population_data(username, password, read_data, directory)

    column_names = list(df_pop_raw.columns)
    # rename columns
    rename_columns = {
        column_names[0]: dd.EngEng['idCounty'],
        column_names[1]: dd.EngEng['county'],
        column_names[2]: dd.EngEng['ageRKI'],
        column_names[3]: dd.EngEng['number'],
        column_names[4]: dd.EngEng['male'],
        column_names[5]: dd.EngEng['female']
    }
    df_pop_raw.rename(columns=rename_columns, inplace=True)
    # remove date and explanation rows at end of table
    df_pop_raw = df_pop_raw[:np.where(df_pop_raw[dd.EngEng['idCounty']].str.contains(
        '__') == True)[0][0]].reset_index(drop=True)
    # get indices of counties first lines
    idCounty_idx = df_pop_raw.groupby(
        dd.EngEng['idCounty']).head(1).index.tolist()

    # read county list and create output data frame
    counties = np.array(geoger.get_county_names_and_ids(
        merge_berlin=True, merge_eisenach=merge_eisenach, zfill=True))
    age_cols = df_pop_raw.loc[
        idCounty_idx[0]: idCounty_idx[1] - 2,
        dd.EngEng['ageRKI']].values.copy()
    for i in range(len(age_cols)):
        if i == 0:
            upper_bound = str(int(age_cols[i][
                age_cols[i].index('unter ')+6:].split(' ')[0])-1)
            age_cols[i] = '0-' + upper_bound
        elif i == len(age_cols)-1:
            lower_bound = age_cols[i].split(' ')[0]
            age_cols[i] = lower_bound + '-99'
        else:
            lower_bound = age_cols[i].split(' ')[0]
            upper_bound = str(int(age_cols[i][
                age_cols[i].index('unter ')+6:].split(' ')[0])-1)
            age_cols[i] = lower_bound + '-' + upper_bound

    df_pop = assign_population_data(
        df_pop_raw, counties, age_cols, idCounty_idx)

    test_total_population(df_pop, age_cols)

    df_pop_export = export_population_dataframe(
        df_pop, directory, file_format, merge_eisenach)

    return df_pop_export

def main():
    """! Main program entry."""

    arg_dict = gd.cli("population")
    get_population_data(**arg_dict)

if __name__ == "__main__":
    main()


"""Defines a class for different POPCON algorithms."""
from __future__ import annotations

import inspect
from collections.abc import Callable, Sequence
from functools import wraps
from typing import Any, Optional, Union
from warnings import warn

import xarray as xr

from ..unit_handling import convert_to_default_units

FunctionType = Callable[..., dict[str, Any]]

class Algorithm:
    """A class which handles the input and output of POPCON algorithms."""

    def __init__(self, function: FunctionType, return_keys: list[str], name: Optional[str] = None):
        """Initialise an Algorithm.

        Args:
            function: a callable function
            return_keys: the arguments which are returned from the function
            name: Descriptive name for algorithm
        """
        self._function = function
        self._name = self._function.__name__ if name is None else name

        self._signature = inspect.signature(function)
        for p in self._signature.parameters.values():
            if p.kind not in (
                inspect.Parameter.POSITIONAL_OR_KEYWORD,
                inspect.Parameter.KEYWORD_ONLY,
                inspect.Parameter.VAR_KEYWORD,
            ):
                raise ValueError(
                    f"Algorithm only supports functions with keyword arguments, but {function}, has {p.kind} parameter {p.name}"
                )
        self.input_keys = list(self._signature.parameters.keys())
        self.return_keys = return_keys

        self.default_values = {
            key: val.default for key, val in self._signature.parameters.items() if val.default is not inspect.Parameter.empty
        }
        self.default_keys = list(self.default_values.keys())

        self.required_input_keys = [key for key in self.input_keys if key not in self.default_keys]

        self.__doc__ = self._make_docstring()

        self.run = self._make_run(self._function)

    def _make_docstring(self) -> str:
        """Makes a doc-string detailing the function inputs and outputs."""
        return_string = (
            f"Algorithm: {self._name}\n" + "Inputs:\n" + ", ".join(self.input_keys) + "\n" + "Outputs:\n" + ", ".join(self.return_keys)
        )
        return return_string

    def __repr__(self) -> str:
        """Return a simple string description of the Algorithm."""
        return f"Algorithm: {self._name}"

    @classmethod
    def _make_run(cls, func: FunctionType) -> Callable[..., xr.Dataset]:
        """Helper to create the `run()` function with correct doc string.

        Args:
            func: function to be wrapped

        Returns: a xarray DataSet of the result
        """

        @wraps(func)
        def run(**kwargs: Any) -> xr.Dataset:
            result = func(**kwargs)
            dataset = xr.Dataset(result)
            return dataset

        return run

    def update_dataset(self, dataset: xr.Dataset, in_place: bool = False) -> Optional[xr.Dataset]:
        """Retrieve inputs from passed dataset and return a new dataset combining input and output quantities.

        Specifying in_place=True modifies the dataset in place (changing the input), whereas in_place=False will
        return a copy of the dataset with the outputs appended.

        Args:
            dataset: input dataset
            in_place: modify the dataset in place, otherwise return a modified dataset keeping the input unchanged.

        Returns: modified dataset
        """
        if not in_place:
            dataset = dataset.copy(deep=True)

        input_values = {}
        for key in self.input_keys:
            if key in dataset.keys():
                input_values[key] = dataset[key]
            elif key in self.default_keys:
                input_values[key] = self.default_values[key]
            else:
                sorted_dataset_keys = ", ".join(sorted(dataset.keys()))  # type:ignore[arg-type]
                sorted_default_keys = ", ".join(sorted(self.default_keys))
                raise KeyError(f"Key '{key}' not in dataset keys [{sorted_dataset_keys}] or default values [{sorted_default_keys}]")

        result = self._function(**input_values)

        for key, val in result.items():
            dataset[key] = val

        if not in_place:
            return dataset
        else:
            return None

    def __add__(self, other: Union[Algorithm, CompositeAlgorithm]) -> CompositeAlgorithm:
        """Build a CompositeAlgorithm composed of this Algorithm and another Algorithm or CompositeAlgorithm."""
        if isinstance(other, CompositeAlgorithm):
            return CompositeAlgorithm(algorithms=[self, *other.algorithms])
        else:
            return CompositeAlgorithm(algorithms=[self, other])

    @classmethod
    def from_single_function(
        cls, func: Callable, return_keys: list[str], name: Optional[str] = None, skip_unit_conversion: bool = False
    ) -> Algorithm:
        """Build an Algorithm which wraps a single function."""

        @wraps(func)
        def wrapped_function(**kwargs: Any) -> dict:
            result = func(**kwargs)

            if not isinstance(result, tuple):
                result = (result,)

            result_dict = {}
            for i, key in enumerate(return_keys):
                if skip_unit_conversion:
                    result_dict[key] = result[i]
                else:
                    result_dict[key] = convert_to_default_units(result[i], key)

            return result_dict

        return cls(wrapped_function, return_keys, name=name)

    def validate_inputs(
        self, configuration: Union[dict, xr.Dataset], quiet: bool = False, raise_error_on_missing_inputs: bool = False
    ) -> bool:
        """Check that all required inputs are defined, and warn if inputs are unused."""
        return _validate_inputs(self, configuration, quiet=quiet, raise_error_on_missing_inputs=raise_error_on_missing_inputs)

class CompositeAlgorithm:
    """A class which combined multiple Algorithms into a single object which behaves like an Algorithm."""

    def __init__(self, algorithms: Sequence[Union[Algorithm, CompositeAlgorithm]], name: Optional[str] = None):
        """Initialise a CompositeAlgorithm, combining several other Algorithms.

        Args:
            algorithms: a list of Algorithms, in the order that they should be executed.
            name: a name used to refer to the composite algorithm.
        """
        if not (isinstance(algorithms, Sequence) and all(isinstance(alg, (Algorithm, CompositeAlgorithm)) for alg in algorithms)):
            raise TypeError("Should pass a list of algorithms or composites to CompositeAlgorithm.")

        self.algorithms: list[Algorithm] = []

        # flattens composite algorithms into their respective list of plain Algorithms
        for alg in algorithms:
            if isinstance(alg, Algorithm):
                self.algorithms.append(alg)
            else:
                self.algorithms.extend(alg.algorithms)

        self.input_keys: list[str] = []
        self.required_input_keys: list[str] = []
        self.return_keys: list[str] = []
        pars: list[inspect.Parameter] = []

        # traverse list of algorithms in order.
        # If an ouput from the set of previous algorithms provides an input to a following algorithm
        # the input is not turned into an input to the CompositeAlgorithm
        for alg in self.algorithms:
            alg_sig = inspect.signature(alg.run)
            for key in alg.default_keys:
                if key not in self.return_keys:
                    self.input_keys.append(key)
                    pars.append(alg_sig.parameters[key])
            for key in alg.required_input_keys:
                if key not in self.return_keys:
                    self.input_keys.append(key)
                    self.required_input_keys.append(key)
                    pars.append(alg_sig.parameters[key])

            for key in alg.return_keys:
                if key not in self.return_keys:
                    self.return_keys.append(key)

        # create a signature for the run() function
        # This is a purely aesthetic change, that ensures the run() function
        # has a helpful tooltip in editors and in the documentation

        # 1. make sure the list of pars doesn't have any duplicates, if there are duplicates
        # we pick the first one. We don't assert that the types of two parameters are compatible
        # that's not easy to do.
        seen_pars: dict[str, int] = {}
        pars = [p for i, p in enumerate(pars) if seen_pars.setdefault(p.name, i) == i]

        # ensure POSITIONAL_OR_KEYWORD are before kw only
        pars = sorted(pars, key=lambda p: p.kind)

        def_pars = [p for p in pars if p.default != inspect.Parameter.empty]
        non_def_pars = [p for p in pars if p.default == inspect.Parameter.empty]

        # methods are immutable and we don't want to set a signature on the class' run() method
        # thus we wrap the original run method and then assign the __signature__ to the wrapped
        # wrapper function
        def _wrap(f: Callable[..., xr.Dataset]) -> Callable[..., xr.Dataset]:
            def wrapper(**kwargs: Any) -> xr.Dataset:
                return f(**kwargs)

            wrapper.__doc__ = f.__doc__

            return wrapper

        self.run = _wrap(self._run)
        # ignore due to mypy bug/missing feature https://github.com/python/mypy/issues/3482
        self.run.__signature__ = inspect.Signature(  # type:ignore[attr-defined]
            non_def_pars + def_pars, return_annotation=xr.Dataset
        )
        self._name = name
        self.__doc__ = self._make_docstring()

    def _make_docstring(self) -> str:
        """Makes a doc-string detailing the function inputs and outputs."""
        components = f"[{', '.join(alg._name for alg in self.algorithms)}]"

        return_string = (
            f"CompositeAlgorithm: {self._name}\n"
            if self._name is not None
            else "CompositeAlgorithm\n"
            f"Composed of {components}\n"
            f"Inputs:\n{', '.join(self.input_keys)}\n"
            f"Outputs:\n{', '.join(self.return_keys)}"
        )
        return return_string

    def __repr__(self) -> str:
        """Return a simple string description of the CompositeAlgorithm."""
        return f"CompositeAlgorithm: {self._name}"

    def _run(self, **kwargs: Any) -> xr.Dataset:
        """Run the sub-Algorithms, one after the other and return a xarray.Dataset of the results.

        Will throw a warning if parameters are not used by any sub-Algorithm.
        """
        result = kwargs

        parameters_extra = set(kwargs) - set(self.required_input_keys)
        parameters_missing = set(self.required_input_keys) - set(kwargs)
        if parameters_missing:
            raise TypeError(f"CompositeAlgorithm.run() missing arguments: {', '.join(parameters_missing)}")
        if parameters_extra:
            warn(f"Not all input parameters were used. Unused parameters: [{', '.join(parameters_extra)}]", stacklevel=3)

        for alg in self.algorithms:

            alg_kwargs = {key: result[key] for key in result.keys() if key in alg.input_keys}

            alg_result = alg.run(**alg_kwargs)
            result.update(alg_result)  # type:ignore[arg-type]  # dict.update() doesn't like KeysView[Hashable]

        return xr.Dataset(result)

    def update_dataset(self, dataset: xr.Dataset, in_place: bool = False) -> Optional[xr.Dataset]:
        """Retrieve inputs from passed dataset and return a new dataset combining input and output quantities.

        Specifying in_place=True modifies the dataset in place (changing the input), whereas in_place=False will
        return a copy of the dataset with the outputs appended.

        N.b. will not throw a warning if the dataset contains unused elements.

        Args:
            dataset: input dataset
            in_place: modify the dataset in place, otherwise return a modified dataset keeping the input unchanged.

        Returns: modified dataset
        """
        if not in_place:
            dataset = dataset.copy(deep=True)

        for alg in self.algorithms:
            # We've already used copy on the dataset, so can now call update_dataset with
            # in_place = True for each of the algorithms.
            alg.update_dataset(dataset, in_place=True)

        if not in_place:
            return dataset
        else:
            return None

    def __add__(self, other: Union[Algorithm, CompositeAlgorithm]) -> CompositeAlgorithm:
        """Build a CompositeAlgorithm composed of this CompositeAlgorithm and another Algorithm or CompositeAlgorithm."""
        if isinstance(other, Algorithm):
            return CompositeAlgorithm(algorithms=[*self.algorithms, other])
        else:
            return CompositeAlgorithm(algorithms=[*self.algorithms, *other.algorithms])

    def validate_inputs(  # noqa: PLR0912
        self,
        configuration: Union[dict, xr.Dataset],
        quiet: bool = False,
        raise_error_on_missing_inputs: bool = True,
        warn_for_overridden_variables: bool = False,
    ) -> bool:
        """Check that all required inputs are defined, and warn if inputs are unused."""
        # Check if variables are being silently internally overwritten
        config_keys = list(configuration.keys())
        key_setter = {key: ["INPUT"] for key in config_keys}

        for algorithm in self.algorithms:
            for key in algorithm.return_keys:
                if key not in key_setter.keys():
                    key_setter[key] = [algorithm._name]
                else:
                    key_setter[key].append(algorithm._name)

        overridden_variables = []
        for variable, algs in key_setter.items():
            if len(algs) > 1:
                overridden_variables.append(f"{variable}: ({', '.join(algs)})")

        if warn_for_overridden_variables and len(overridden_variables) > 0:
            warn(
                f"The following variables were overridden internally (given as variable: (list of algorithms setting variable)): {', '.join(overridden_variables)}",
                stacklevel=3,
            )

        # Check that algorithms are ordered such that dependent algorithms follow those setting their required input keys
        available_parameters = config_keys.copy()
        out_of_order_parameters = {}
        for algorithm in self.algorithms:
            for key in algorithm.required_input_keys:
                if key not in available_parameters:
                    out_of_order_parameters[key] = algorithm
            for key in algorithm.return_keys:
                available_parameters.append(key)

        if len(out_of_order_parameters) > 0:
            message = ""
            for key, algorithm in out_of_order_parameters.items():
                if key in key_setter and len(key_setter.get(key, [])) > 0:
                    message += f"{key} needed by {algorithm} defined by output of {key_setter[key]}."
            if len(message) > 0:
                message = f"Algorithms out of order. {message}. Rearrange the list of algorithms so that dependent algorithm are after algorithms setting their inputs."
                if raise_error_on_missing_inputs:
                    raise RuntimeError(message)
                if not quiet:
                    warn(message, stacklevel=3)

            _validate_inputs(self, configuration, quiet=quiet, raise_error_on_missing_inputs=raise_error_on_missing_inputs)

            return False
        else:
            return _validate_inputs(self, configuration, quiet=quiet, raise_error_on_missing_inputs=raise_error_on_missing_inputs)

def _validate_inputs(
    algorithm: Union[Algorithm, CompositeAlgorithm],
    configuration: Union[dict, xr.Dataset],
    quiet: bool = False,
    raise_error_on_missing_inputs: bool = False,
) -> bool:
    """Check that all required inputs are defined, and warn if inputs are unused."""
    config_keys = list(configuration.keys())

    unused_config_keys = config_keys.copy()
    missing_input_keys = set(algorithm.required_input_keys)

    for key in config_keys:
        if key in missing_input_keys:
            missing_input_keys.remove(key)

        if key in algorithm.input_keys:
            # required_input_keys gives the list of keys which must
            # be provided, while input_puts gives the list of keys
            # which can be provided (but which might have default values).
            unused_config_keys.remove(key)

    if len(missing_input_keys) == 0 and len(unused_config_keys) == 0:
        return True

    elif len(missing_input_keys) > 0 and len(unused_config_keys) > 0:
        message = f"Missing input parameters [{', '.join(missing_input_keys)}]. Also had unused input parameters [{', '.join(unused_config_keys)}]."
        if raise_error_on_missing_inputs:
            raise RuntimeError(message)

    elif len(missing_input_keys) > 0:
        message = f"Missing input parameters [{', '.join(missing_input_keys)}]."
        if raise_error_on_missing_inputs:
            raise RuntimeError(message)

    else:
        message = f"Unused input parameters [{', '.join(unused_config_keys)}]."

    if not quiet:
        warn(message, stacklevel=3)
    return False


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Note: To use the 'upload' functionality of this file, you must:
#   $ pip install twine

import io
import os
import sys
from shutil import rmtree

from setuptools import setup, Command

# Package meta-data.
NAME = "yam-py"
DESCRIPTION = "HTML Parsing for Humans."
URL = "https://github.com/plugarut/yam-py"
EMAIL = "plugaru.tudor@gmail.com"
AUTHOR = "Tudor Plugaru"
VERSION = "0.1.0"

# What packages are required for this module to be executed?
REQUIRED = ["requests", "requests-html", "click", "colorama"]

# The rest you shouldn't have to touch too much :)
# ------------------------------------------------
# Except, perhaps the License and Trove Classifiers!
# If you do change the License, remember to change the Trove Classifier for that!

here = os.path.abspath(os.path.dirname(__file__))

# Import the README and use it as the long-description.
# Note: this will only work if 'README.rst' is present in your MANIFEST.in file!
with io.open(os.path.join(here, "README.md"), encoding="utf-8") as f:
    long_description = "\n" + f.read()

# Where the magic happens:
setup(
    name=NAME,
    version=VERSION,
    description=DESCRIPTION,
    long_description=long_description,
    author=AUTHOR,
    author_email=EMAIL,
    url=URL,
    python_requires=">=3.6.0",
    # If your package is a single module, use this instead of 'packages':
    py_modules=["yam_py"],
    install_requires=REQUIRED,
    include_package_data=True,
    license="MIT",
    classifiers=[
        # Trove classifiers
        # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: Implementation :: CPython",
        "Programming Language :: Python :: Implementation :: PyPy",
    ],
    entry_points={"console_scripts": ["yam-py=yam_py:cli"]},
)


from random import choice

from backend.database.db_schema import Medicine, db
from sqlalchemy.sql import func

def get_medicine_detail(medicine_id: int):
    medicine_entity: Medicine = db.get_or_404(Medicine, medicine_id)
    return {"id": medicine_entity.id,
            "name": medicine_entity.name,
            "sukl_code": medicine_entity.sukl_code,
            "contraindications": medicine_entity.contraindications,
            "adverse_effects": medicine_entity.adverse_effects,
            "active_ingredients": [{"id": ingredient_with_dosage.active_ingredient.id,
                                    "name": ingredient_with_dosage.active_ingredient.name,
                                    "dosage": ingredient_with_dosage.dosage,
                                    "units": ingredient_with_dosage.units
                                    } for ingredient_with_dosage
                                   in medicine_entity.active_ingredients_with_dosage],
            "negative_interactions": [{"id": medicine.id,
                                       "name": medicine.name} for medicine
                                      in get_negative_interactions_for_medicine(medicine_entity)],
            "alternative_medicines": [{"id": medicine.id,
                                       "name": medicine.name,
                                       "sukl_code": medicine.sukl_code,
                                       "contraindications": medicine.contraindications,
                                       "adverse_effects": medicine.adverse_effects,
                                       "active_ingredients": [
                                           {"id": ingredient_with_dosage.active_ingredient.id,
                                            "name": ingredient_with_dosage.active_ingredient.name,
                                            "dosage": ingredient_with_dosage.dosage,
                                            "units": ingredient_with_dosage.units
                                            } for ingredient_with_dosage
                                           in medicine.active_ingredients_with_dosage],
                                       "pdf_dropbox_link": medicine.pdf_dropbox_link
                                       } for medicine
                                      in get_alternative_medicines(medicine_entity)],
            "pdf_dropbox_link": medicine_entity.pdf_dropbox_link
            }

def get_negative_interactions_for_medicine(medicine: Medicine):
    return list(medicine.negative_interactions_1) + list(medicine.negative_interactions_2)

def get_alternative_medicines(medicine: Medicine):
    active_ingredients_with_dosage = medicine.active_ingredients_with_dosage
    return [medicine_with_dosage.medicine for active_ingredients_with_dosage in
            active_ingredients_with_dosage
            for medicine_with_dosage in
            list(active_ingredients_with_dosage.active_ingredient.medicines_with_dosage)
            if medicine_with_dosage.medicine != medicine]

def get_random_medicine_and_adverse_effect():
    random_medicine = Medicine.query.order_by(func.random()).first()
    random_adverse_effect = choice(random_medicine.adverse_effects)
    return {"name": random_medicine.name,
            "random_adverse_effect": random_adverse_effect}


import os

import numpy as np
from PIL import Image
import skimage
import skimage.io
from torch.utils.data import Dataset
from tqdm import tqdm

def read_rgb_image(path_to_image):
    image = skimage.img_as_ubyte(skimage.io.imread(path_to_image))
    if len(image.shape) == 2:
        image = image[:, :, np.newaxis]
    if image.shape[-1] == 1:
        image = image * np.ones((1,1,3), dtype=np.uint8)
    return image

class KITTIStereoRAM(Dataset):
    def __init__(self, root, train=True, transforms=None):
        super(KITTIStereoRAM, self).__init__()

        self.root = root
        if train:
            path_to_dataset = os.path.join(root, 'train')
        else:
            path_to_dataset = os.path.join(root, 'val')

        self.transforms = transforms
        self.left_images = []
        self.right_images = []
        self.targets = []
        self.valid_pixels_masks = []
        for img_name in tqdm(os.listdir(os.path.join(path_to_dataset, 'colored_0'))):
            left_img_path = os.path.join(path_to_dataset, 'colored_0', img_name)
            right_img_path = os.path.join(path_to_dataset, 'colored_1', img_name)
            disp_img_path = os.path.join(path_to_dataset, 'disp_noc', img_name)

            left_img = read_rgb_image(left_img_path)
            right_img = read_rgb_image(right_img_path)

            # disparity normalization
            disp_img = skimage.io.imread(disp_img_path)[:,:,np.newaxis]
            valid_pixels_mask = disp_img > 0
            disp_img = disp_img.astype(np.float32) / 256.

            self.left_images.append(Image.fromarray(left_img))
            self.right_images.append(Image.fromarray(right_img))
            self.targets.append(disp_img)
            self.valid_pixels_masks.append(valid_pixels_mask)

    def __len__(self):
        return len(self.left_images)

    def __getitem__(self, index):
        left_img = self.left_images[index]
        right_img = self.right_images[index]
        disp = self.targets[index]
        valid_pixels_mask = self.valid_pixels_masks[index]

        if self.transforms is not None:
            left_img, right_img, disp, valid_pixels_mask = self.transforms(
                    left_img, right_img, disp, valid_pixels_mask)
        return left_img, right_img, disp, valid_pixels_mask


import re
from functools import partial
from pathlib import Path
from typing import List, Optional

import sublime
import sublime_plugin

from ..core.git import git_status_porcelain
from ..core.parser import parse_git_status
from ..core.git_file import GitFile

class GitFilesGotoCommand(sublime_plugin.WindowCommand):
    def run(self):
        cwd = Path(self.window.extract_variables()["folder"])
        output = git_status_porcelain(cwd)
        git_files = parse_git_status(output)

        active_view = self.window.active_view()
        self._highlighted_view = None  # type: Optional[sublime.View]

        if len(git_files) > 0:
            self.window.show_quick_panel(
                items=self.to_quick_panel_item(git_files),
                on_select=partial(self.on_select, active_view, cwd, git_files),
                on_highlight=partial(self.on_highlight, cwd, git_files),
            )
        else:
            sublime.message_dialog("GitFiles: No changed files.")

    def to_quick_panel_item(
        self, git_files: List[GitFile]
    ) -> List[sublime.QuickPanelItem]:
        items = []
        for item in git_files:
            git_status = item.git_status
            items.append(
                sublime.QuickPanelItem(
                    item.file_name,
                    item.file_path,
                    git_status.description(),
                    git_status.kind(),
                )
            )
        return items

    def __open_file(
        self, cwd, items, index: int, on_highlight: bool
    ) -> Optional[sublime.View]:
        if index == -1:
            return None

        rename_match = re.search(r"(.+) -> (.*)", items[index].file_path)
        file_path = (
            rename_match.group(2)
            if rename_match is not None
            else items[index].file_path
        )
        full_path = Path(cwd, file_path)
        flag = sublime.TRANSIENT if on_highlight else sublime.NewFileFlags.NONE
        if full_path.is_file():
            return self.window.open_file(
                str(full_path), flag, self.window.active_group()
            )
        return None

    def on_select(self, active_view: Optional[sublime.View], cwd, items, index: int):
        if index == -1:
            if active_view is not None and active_view.is_valid():
                self.window.focus_view(active_view)
                active_view.show(active_view.sel()[0])

            if self._highlighted_view:
                sheet = self._highlighted_view.sheet()
                if sheet and sheet.is_transient():
                    self._highlighted_view.close()

        self.__open_file(cwd, items, index, False)

    def on_highlight(self, cwd, items, index: int):
        self._highlighted_view = self.__open_file(cwd, items, index, True)


from flask import Blueprint, request
from google.cloud import datastore
from helper import validate_mimetype
import constants
import json

bp = Blueprint('users', __name__, url_prefix='/users')
client = datastore.Client()

@bp.route('', methods=['GET'])
def get_users():
    if request.method == 'GET':
        # Checking to see if the request Accept Header is in a valid format.
        chk_content_type = validate_mimetype(None, None, request.accept_mimetypes, 'application/json')
        if chk_content_type is not None:
            return chk_content_type
        q = client.query(kind=constants.users)
        results = list(q.fetch())
        for el in results:
            el["id"] = el.key.id
        return json.dumps(results)
    else:
        return 'That is a bad method!'


#coding=utf-8
#此文件用于指定后台返回前台的协议

# 返回包封装类

import json
from bson import json_util

class DataProtocol(object):

    SUCCESS = 0
    ARGUMENT_ERROR = -1 #参数错误
    DB_ERROR = -2      #数据库访问错误
    DB_DUPLICATE_ERROR = -3 # 数据重复

    IS_NOT_AJAX_REQUEST = -4 #非 ajax请求却请求了ajax/xxx的文件
    DATAPROTOCOL_ERROR = -5  # -5 ：数据封装问题
    FILE_ERROR = -6 #文件读取失败
    AUTH_ERROR = -7
    USER_EXISTED_ERROR = -8 #用户已经存在，注册失败
    LOGIN_FAIL = -9
    SERVER_ERROR = -10

    PERMISSION_DENY = -11 #没有权限
    USER_EXISTED_ERROR = -12 #用户不存在

    AT_LEAST_ONE_TRUNK_ERROR = -20001 #只要要保存一辆货车
    BILL_REMOVED = -30001  #bill已经被删除
    BILL_NOT_OWN = -30002   #用户不用有该bill
    BILL_NOT_WAIT = -30003   #bill不再有效期
    USER_INVALID = -30004  #请求中用户不存在或者不是有效的对象(管理员)

    ALREADY_COMMENTED = -40001 #已经评论过了
    CANNOT_SELF_COMMENT = -40002 #不能给自己评论呢
    MESSAGE_DUPLICATE_ERROR = -50001 #重复添加message
    # code: 0 代表成功 ，负数代表失败，-1 ~ -100 通用失败， -1000 ~ -9999 自定义错误代码

    ALREADY_EXIST = -50001   #要添加的东西已经存在

    # datatype include "html","json","string","nothing"
    #"nothing " is default

    #MSG
    COMMON_MSG = "服务器繁忙，请稍后再试"
    ARGUMENT_ERROR_MSG ="请求失败，请重试"
    DB_ERROR = "服务器繁忙，请稍后再试"
    IS_NOT_AJAX_REQUEST_MSG="非法请求，请用正常方式请求"
    AT_LEAST_ONE_TRUNK_ERROR_MSG = "至少要保存一辆货车"
    ALREADY_COMMENTED_MSG = "已经评论过了"
    CANNOT_SELF_COMMENT_MSG = "不能给自己评论"
    BILL_REMOVED_MSG = "订单已经被删除"

    #什么都不判断直接输出return，不保证数据正确，只能被getJson()调用
    @staticmethod
    def _wrapJson(code, msg, data, datatype="nothing"):
        retDict = {
            "code":code,
            "msg":msg,
            "data":data,
            "datatype":datatype
            }
        return json.dumps(retDict,default=json_util.default)

    @staticmethod
    def _wrapError(msg="数据封装错误"):
        return DataProtocol._wrapJson(DataProtocol.DATAPROTOCOL_ERROR, msg,"","nothing")

    @staticmethod
    def getJson(code, msg="", data=None, datatype="nothing"):
        retJson = None
        if not isinstance(code,int):
            try:
                code = int(code)
            except:
                retJson = DataProtocol._wrapError("code is not int")

        if not isinstance(msg, str):
            try:
                msg = str(msg)
            except:
                retJson = DataProtocol._wrapError("msg is not string")

        if not isinstance(datatype, str):
            try:
                datatypeTmp = str(datatype)
            except:
                retJson = DataProtocol._wrapError("datatype is not string")
            else:
                if datatype == "string" or datatype == "json" or datatype == "html":
                    if not isinstance(data, str):
                        try:
                            dataTmpStr = str(data)
                        except:
                            retJson = DataProtocol._wrapError("datatype said is "+ datatype+ " but data is "+ str(type(data)))
                        else:
                            data = dataTmpStr
                elif datatype == "nothing":
                    pass
                else:
                    retJson = DataProtocol._wrapError("unknown datatype")

        if msg == "":
            if code == DataProtocol.ARGUMENT_ERROR:
                msg = DataProtocol.ARGUMENT_ERROR_MSG
            elif code ==DataProtocol.DB_ERROR:
                msg = DataProtocol.DB_ERROR_MSG
            elif code ==DataProtocol.IS_NOT_AJAX_REQUEST:
                msg = DataProtocol.IS_NOT_AJAX_REQUEST_MSG
            elif code ==DataProtocol.AT_LEAST_ONE_TRUNK_ERROR:
                msg = DataProtocol.AT_LEAST_ONE_TRUNK_ERROR_MSG
            elif code ==DataProtocol.ALREADY_COMMENTED:
                msg = DataProtocol.ALREADY_COMMENTED_MSG
            elif code ==DataProtocol.CANNOT_SELF_COMMENT:
                msg = DataProtocol.CANNOT_SELF_COMMENT_MSG
            else:
                msg = DataProtocol.COMMON_MSG

        # print "----------getJson",code,msg,data,datatype
        if not retJson is None:
            return retJson
        else:
            return DataProtocol._wrapJson(code,msg,data,datatype);

    @staticmethod
    def getSuccessJson(data=None,datatype="nothing"):
        return DataProtocol.getJson(0,"success",data, datatype)

#单元测试-_-
#if __name__ == "__main__":
#     print r"DataProtocol.getSuccessJson(\"hello, world\",\"string\")", DataProtocol.getSuccessJson("hello, world","string")
#     print r"DataProtocol.getSuccessJson(\"hello, world\",\"(123,123)\")", DataProtocol.getSuccessJson("hello, world","(123,123)")
#     print r"DataProtocol.getSuccessJson(1,2)", DataProtocol.getSuccessJson(1,2)
#     print DataProtocol.getJson("0","success")
#     print DataProtocol.getJson(0,"success")
#     print DataProtocol.getJson("o","success")
#     print DataProtocol.getJson(0,"success",data={"a":1},datatype="json")
#     print DataProtocol.getJson(0,"success",data=(1,2,3),datatype="json")
#     print DataProtocol.getJson("-4",(123,123),data=(1,2,3),datatype="json")
#     print DataProtocol.getJson(5,None,None,None)


import torch
import torch.nn as nn
from torch import optim
import torch.nn.functional as F
import numpy as np
import operator
from queue import PriorityQueue

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
MAX_LENGTH = 10
SOS_token = 0
EOS_token = 1
EOS_char_token = 0

class DecoderRNN(nn.Module):
    def __init__(self, hidden_size, output_size):
        super(DecoderRNN, self).__init__()
        self.hidden_size = hidden_size

        self.embedding = nn.Embedding(output_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size)
        self.out = nn.Linear(hidden_size, output_size)
        self.softmax = nn.LogSoftmax(dim=1)

    def forward(self, input, hidden):
        output = self.embedding(input).view(1, 1, -1)
        output = F.relu(output)
        output, hidden = self.gru(output, hidden)
        output = self.softmax(self.out(output[0]))
        return output, hidden

class AttnDecoderRNN(nn.Module):
    def __init__(self, hidden_size, output_size, dropout_p=0.1, max_length=MAX_LENGTH):
        super(AttnDecoderRNN, self).__init__()
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.dropout_p = dropout_p
        self.max_length = max_length

        self.embedding = nn.Embedding(self.output_size, self.hidden_size)
        self.attn = nn.Linear(self.hidden_size * 2, self.max_length)
        self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)
        self.dropout = nn.Dropout(self.dropout_p)
        self.gru = nn.GRU(self.hidden_size, self.hidden_size)
        self.out = nn.Linear(self.hidden_size, self.output_size)

    def forward(self, input, hidden, encoder_outputs):
        embedded = self.embedding(input).view(1, 1, -1)
        embedded = self.dropout(embedded)

        attn_weights = F.softmax(self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)
        attn_applied = torch.bmm(attn_weights.unsqueeze(0), encoder_outputs.unsqueeze(0))

        output = torch.cat((embedded[0], attn_applied[0]), 1)
        output = self.attn_combine(output).unsqueeze(0)

        output = F.relu(output)
        output, hidden = self.gru(output, hidden)

        output = F.log_softmax(self.out(output[0]), dim=1)

        return output, hidden, attn_weights

    def init_hidden(self):
        return torch.zeros(1, 1, self.hidden_size, device=device)

class EncoderRNN(nn.Module):
    def __init__(self, n_chars, n_words, hidden_size, char_embedding_dim=25, char_representation_dim=25):
        """
        Input parameters:
            n_chars = Number of unique characters in input language
            n_words = Number of unique words in input language
            hidden_size = Dimension of GRU input and output.
            char_embedding_dim = Dimension of the character embeddings
            char_representation_dim = Output dimension from the CNN encoder for character
        """
        super(EncoderRNN, self).__init__()

        # Parameters
        self.n_chars = n_chars
        self.char_embedding_dim = char_embedding_dim
        self.char_representation_dim = char_representation_dim
        self.n_words = n_words
        self.hidden_size = hidden_size

        # Character-level encoder
        self.char_embedding_layer = nn.Embedding(n_chars, char_embedding_dim)
        self.char_cnn3_layer = nn.Conv2d(in_channels=1, out_channels=char_representation_dim, kernel_size=(3, char_embedding_dim), padding=(2, 0))

        # Word embedding layer (Dimension of the word embeddings is automatically derived as hidden_size - char_representation_dim)
        self.word_embedding_layer = nn.Embedding(n_words, hidden_size - char_representation_dim)

        # GRU layer
        self.gru = nn.GRU(hidden_size, hidden_size)

        # Weights
        self.init_embedding(self.char_embedding_layer.weight)

    def forward(self, char_indexes, word_index, hidden):
        # Get char representation
        char_embedding = self.char_embedding_layer(char_indexes).unsqueeze(1)
        char_cnn3 = self.char_cnn3_layer(char_embedding).squeeze(-1).unsqueeze(1)
        char_representation = F.max_pool2d(char_cnn3, kernel_size=(1, char_cnn3.size(-1))).squeeze(-1)

        # Get word embedding
        word_embedding = self.word_embedding_layer(word_index).view(1, 1, -1)

        # Concatenate char representation with word embedding
        combined = torch.cat((char_representation, word_embedding), dim=2)

        # Feed combined and hidden to GRU
        output, hidden = self.gru(combined, hidden)

        return output, hidden

    def init_hidden(self):
        return torch.zeros(1, 1, self.hidden_size, device=device)

    @staticmethod
    def init_embedding(input_embedding):
        """
        Initialize embedding
        """
        bias = np.sqrt(3.0 / input_embedding.size(1))
        nn.init.uniform_(input_embedding, -bias, bias)

class BeamSearchNode(object):
    def __init__(self, hidden_state, previous_node, word_id, log_prob, length):
        self.hidden_state = hidden_state
        self.prev_node = previous_node
        self.word_id = word_id
        self.log_p = log_prob
        self.length = length

    def eval(self, alpha=1.0):
        reward = 0
        # Add here a function for shaping a reward
        return self.log_p / float(self.length - 1 + 1e-6) + alpha * reward

def beam_decode(decoder_hiddens, decoder, encoder_outputs=None):

    beam_width = 5
    top_k = 1  # how many sentences do you want to generate
    decoded_batch = []

    # print(target_tensor.shape)
    # print(decoder_hiddens.shape)
    # print(encoder_outputs.shape)

    # decoding goes sentence by sentence
    for idx in range(decoder_hiddens.size(1)):
        decoder_hidden = decoder_hiddens[:, idx, :]
        encoder_output = encoder_outputs[:, idx, :]

        # Start with the start of the sentence token
        decoder_input = torch.LongTensor([[SOS_token]]).to(device)

        # Number of sentence to generate
        endnodes = []
        number_required = min((top_k + 1), top_k - len(endnodes))

        # Starting node - hidden vector, previous node, word id, logp, length
        node = BeamSearchNode(decoder_hidden, None, decoder_input, 0, 1)
        nodes = PriorityQueue()

        # Start the queue
        nodes.put((-node.eval(), node))
        qsize = 1

        # Start beam search
        while True:

            # Give up when decoding takes too long
            if qsize > 2000:
                break

            # Fetch the best node
            score, n = nodes.get()
            decoder_input = n.word_id
            decoder_hidden = n.hidden_state

            if n.word_id.item() == EOS_token and n.prev_node is not None:
                endnodes.append((score, n))
                # If we reached maximum # of sentences required
                if len(endnodes) >= number_required:
                    break
                else:
                    continue

            # Decode for one step using decoder
            decoder_output, decoder_hidden, decoder_attention = decoder(decoder_input, decoder_hidden, encoder_output)

            # PUT HERE REAL BEAM SEARCH OF TOP
            log_prob, indexes = torch.topk(decoder_output, beam_width)
            # print(indexes)
            next_nodes = []

            for new_k in range(beam_width):
                decoded_t = indexes[0][new_k].view(1, -1)
                log_p = log_prob[0][new_k].item()

                node = BeamSearchNode(decoder_hidden, n, decoded_t, n.log_p + log_p, n.length + 1)
                score = -node.eval()
                next_nodes.append((score, node))

            # Put them into queue
            for i in range(len(next_nodes)):
                score, next_node = next_nodes[i]
                nodes.put((score, next_node))
                # increase qsize
            qsize += len(next_nodes) - 1

        # Choose nbest paths, back trace them
        if len(endnodes) == 0:
            endnodes = [nodes.get() for _ in range(top_k)]

        utterances = []
        for score, n in sorted(endnodes, key=operator.itemgetter(0)):
            utterance = []
            utterance.append(n.word_id)
            # back trace
            while n.prev_node is not None:
                n = n.prev_node
                utterance.append(n.word_id)

            utterance = utterance[::-1]
            utterances.append(utterance)

        decoded_batch.append(utterances)

    return decoded_batch


import os
import requests
import pypandoc

def download_image(url):
    response = requests.get(url)
    return response.content

def convert_doc_to_md(doc_url, access_token):
    headers = {"Authorization": "Bearer " + access_token}
    response = requests.get(doc_url, headers=headers)
    response.raise_for_status()
    doc_json = response.json()
    title = doc_json["title"]
    content_blocks = doc_json["content"]
    md_text = f"# {title}\n\n"
    for block in content_blocks:
        block_type = block["type"]
        if block_type == "text":
            md_text += block["content"] + "\n\n"
        elif block_type == "image":
            image_url = block["url"]
            image_name = os.path.basename(image_url)
            image_content = download_image(image_url)
            with open(image_name, "wb") as f:
                f.write(image_content)
            md_text += f"![{image_name}]({image_name})\n\n"
    md_text = pypandoc.convert_text(md_text, "md", format="html")
    return md_text

if __name__ == "__main__":
    doc_url = "https://test.feishu.cn/docx/test"
    access_token = "xxxxxxxxxxxxxxxxxx"
    md_text = convert_doc_to_md(doc_url, access_token)
    with open("output.md", "w", encoding="utf-8") as f:
        f.write(md_text)


import datetime
import json
import urllib.request

from celery import shared_task
from django.core.mail import send_mail

from Task17.settings import API_KEY
from weather.models import City, Forecast

@shared_task
def send_email(interval):
    cities = City.objects.all()
    for city in cities:
        for sub in city.subscriptions.all():
            if sub.interval == interval:
                source = urllib.request.urlopen(
                    f'https://api.openweathermap.org/data/2.5/weather?q={city}&appid={API_KEY}&units=metric').read()
                list_of_data = json.loads(source)
                weather = str(list_of_data['weather'][0]['main'])
                country_code = str(list_of_data['sys']['country'])
                longitude = str(list_of_data['coord']['lon'])
                latidude = str(list_of_data['coord']['lat'])
                timezone = str(list_of_data['timezone'] // 3600)
                temperature = str(list_of_data['main']['temp'])
                temp_min = str(list_of_data['main']['temp_min'])
                temp_max = str(list_of_data['main']['temp_max'])
                pressure = str(list_of_data['main']['pressure'])
                humidity = str(list_of_data['main']['humidity'])
                feels_like = str(list_of_data['main']['feels_like'])
                visibility = str(list_of_data['visibility'])
                wind_speed = str(list_of_data['wind']['speed'])
                clouds = str(list_of_data['clouds']['all'])
                sunrise = datetime.datetime.fromtimestamp(list_of_data['sys']['sunrise']).strftime('%H:%M:%S')
                sunset = datetime.datetime.fromtimestamp(list_of_data['sys']['sunset']).strftime('%H:%M:%S')

                context = f"Thanks to our service, you get a weather forecast in the city {city} once every {interval}." \
                          f"
City name: {city}
" \
                          f"
country code: {country_code}, timezone: {timezone}
" \
                          f"
longitude: {longitude}, latidude: {latidude}
" \
                          f"
" \
                          f"
weather: {weather}
" \
                          f"
temperature: {temperature + 'C'}, temperature min: {temp_min + 'C'}, temperature max: {temp_max + 'C'}
" \
                          f"
feels_like: {feels_like}
" \
                          f"
" \
                          f"
pressure: {pressure}, humidity: {humidity}, visibility: {visibility}
" \
                          f"
wind speed: {wind_speed}, clouds: {clouds}
" \
                          f"
sunrise: {sunrise}, sunset: {sunset}
"
                f, created = Forecast.objects.get_or_create(
                    city=city,
                    weather=weather,
                    temp=temperature,
                    humidity=humidity,
                    pressure=pressure,
                    temp_min=temp_min,
                    temp_max=temp_max,
                    visibility=visibility,
                    wind_speed=wind_speed,
                    clouds=clouds,
                    sunrise=sunrise,
                    sunset=sunset,
                )
                send_mail(f'Forecast in {city}!',
                          f'The weather in {city}:',
                          'Weather Reminder Service',
                          [sub.user.email],
                          html_message=context,
                          fail_silently=False
                          )


"""
Calibrate smoth rssi for the analysis
"""

from datetime import datetime, timedelta
import logging
import time
from common.telnet import Telnet
from programs.info_connected_stations import get_connected_stations, parse_telnet_output

logger = logging.getLogger(__name__)

COMMANDS = {
    "get station info": "echo -n 'EE''EE '; wl -i BAND sta_info MAC_ADDRESS; echo 'FF''FF'",
}

VALUES_TO_CALIBRATE = ["smoothed rssi"]

def run_calibrate_station(
    telnet: Telnet,
    station_mac: str,
):
    """Entry point for initial calibrate stations program"""

    logger.info(
        f"RUNNING PROGRAM: initial calibrate station MAC: {station_mac}")

    now = datetime.now()

    while True:
        next_sample_at = now + timedelta(seconds=1)

        # Get number of connected stations
        total_connections, connected_stations_2_4GHz, connected_stations_5GHz = get_connected_stations(
            telnet)

        connected_stations = connected_stations_2_4GHz + connected_stations_5GHz
        if total_connections > 0:
            # Check if station to calibrate is connected
            if station_mac in connected_stations:
                connected_to_band = "5GHz" if station_mac in connected_stations_5GHz else "2.4GHz"
                # retrieve station info
                sta_info_command = COMMANDS["get station info"].replace(
                    "MAC_ADDRESS", station_mac)
                if connected_to_band == "5GHz":
                    sta_info_command = sta_info_command.replace("BAND", "wl0")
                elif connected_to_band == "2.4GHz":
                    sta_info_command = sta_info_command.replace("BAND", "wl2")
                else:
                    sta_info_command = sta_info_command.replace("BAND", "wl1")
                station_info_result_brut = telnet.send_command_and_read_result(
                    sta_info_command)
                _raw_station_info = str(parse_telnet_output(
                    station_info_result_brut)).split("\r\n")[1:-1]

                # create entry
                for key in VALUES_TO_CALIBRATE:
                    for line in _raw_station_info:
                        if key in line:
                            value = line.split(key)[1][1:].replace("= ", "")
                            logger.info(f"{key}: {value}")
            else:
                logger.error(f"The station {station_mac} is not connected")
        else:
            logger.error(f"The station {station_mac} is not connected")

        now = datetime.now()

        while now < next_sample_at:
            time.sleep(0.1)
            now = datetime.now()


import time
import json
import requests
from datetime import timedelta
from flask import current_app as app

from extensions.flask_redis import redis_store

def check_pending_job(user_id):
    redis_user_pending_job_key = f'{app.config["REDIS_PREFIX_USER_PENDING_JOB"]}:{user_id}'
    if redis_store.connection.exists(redis_user_pending_job_key):
        return True
    else:
        return False

def add_pending_job(user_id):
    redis_user_pending_job_key = f'{app.config["REDIS_PREFIX_USER_PENDING_JOB"]}:{user_id}'
    redis_store.connection.set(redis_user_pending_job_key, 1, timedelta(minutes=app.config["REDIS_PREFIX_USER_JOB_PENDING_TIME"]))

def remove_pending_job(user_id):
    redis_user_pending_job_key = f'{app.config["REDIS_PREFIX_USER_PENDING_JOB"]}:{user_id}'
    redis_store.connection.delete(redis_user_pending_job_key)

def add_new_job(user_id, logged_in_user_role=None):
    redis_user_job_key = f'{app.config["REDIS_PREFIX_USER_JOB"]}:{user_id}'
    user_job_limit = int(app.config["REDIS_PREFIX_USER_JOB_LIMIT"])

    privilege_roles = ['root']
    if logged_in_user_role in privilege_roles:
        add_pending_job(user_id)
        return True

    if check_pending_job(user_id):
        return False

    if redis_store.connection.exists(redis_user_job_key):
        job_count = int(redis_store.connection.get(redis_user_job_key))
        if job_count < user_job_limit:
            redis_store.connection.incr(redis_user_job_key, 1)
            add_pending_job(user_id)
            return True
        else:
            return False
    else:
        redis_store.connection.set(redis_user_job_key, 1, timedelta(minutes=app.config["REDIS_PREFIX_USER_JOB_TIMEOUT"]))
        add_pending_job(user_id)
        return True


import datetime

from pytz import timezone

from processors import NexusTileProcessor

EPOCH = timezone('UTC').localize(datetime.datetime(1970, 1, 1))

class NormalizeTimeBeginningOfMonth(NexusTileProcessor):
    def process_nexus_tile(self, nexus_tile):
        the_tile_type = nexus_tile.tile.WhichOneof("tile_type")

        the_tile_data = getattr(nexus_tile.tile, the_tile_type)

        time = the_tile_data.time

        timeObj = datetime.datetime.utcfromtimestamp(time)

        timeObj = timeObj.replace(day=1)

        timeObj = timezone('UTC').localize(timeObj)

        the_tile_data.time = int((timeObj - EPOCH).total_seconds())

        yield nexus_tile


# -*- coding: utf-8 -*-
from django.http import Http404
from ..models import UserSocialProfile, Poll
from ..forms import PollForm
from django.conf import settings
from django.http import Http404

class polls_help():

    def is_party_manager(self, request):
        '''To find out if user manager of the app or not'''
        manager_social_id = settings.PARTY_MANAGER_VK_SOCIAL_ID
        user = request.user
        # find the profile
        try:
            social_user = UserSocialProfile.objects.get(user=user)
        except:
            raise Http404("Такого профиля социальной сети ВКонтакте нет в базе.")

        if social_user.social_id == manager_social_id:
            return True
        else:
            return None

    def arrange_poll_form(self, request, form):
        '''To arrange params for the poll form'''
        user = request.user
        try:
            vk_social_user = UserSocialProfile.objects.get(user=user)
        except:
            raise Http404("Такого профиля социальной сети ВКонтакте нет в базе.")
        return {'form': form, 'vk_social_user': vk_social_user}


from fastapi import FastAPI, Request, Response, status
from fastapi.middleware.cors import CORSMiddleware

from src import __version__
from src.settings import init_db
from src.utils import get_features_from_db, get_feature_from_nominatim
from src.validators import authenticate, validate_zoom_level

app = FastAPI(version=__version__)

app.add_middleware(
    CORSMiddleware,
    allow_credentials=True,
    allow_headers=["*"],
    allow_methods=["*"],
    allow_origins=["*"],
)

@app.on_event("startup")
async def start_db():
    await init_db()

@app.get("/")
def home():
    return {"version": app.version}

@app.get("/polygons")
async def polygons(zoom_level: str, codes: str, request: Request):
    authenticate(request)

    validate_zoom_level(zoom_level)

    return await get_features_from_db(zoom_level, codes)

@app.post("/add_polygon", status_code=201)
async def add_polygon(zoom_level: str, code: str, request: Request, response: Response):
    authenticate(request)

    validate_zoom_level(zoom_level)

    feature, from_db = await get_feature_from_nominatim(zoom_level, code)

    if from_db:
        response.status_code = status.HTTP_200_OK

    return feature


''' Classes used to represent the concept of a parameter in a .c3d file.
'''
import struct
import numpy as np
from .utils import DEC_to_IEEE, DEC_to_IEEE_BYTES

class ParamData(object):
    '''A class representing a single named parameter from a C3D file.

    Attributes
    ----------
    name : str
        Name of this parameter.
    dtype: DataTypes
        Reference to the DataTypes object associated with the file.
    desc : str
        Brief description of this parameter.
    bytes_per_element : int, optional
        For array data, this describes the size of each element of data. For
        string data (including arrays of strings), this should be -1.
    dimensions : list of int
        For array data, this describes the dimensions of the array, stored in
        column-major (Fortran) order. For arrays of strings, the dimensions here will be
        the number of columns (length of each string) followed by the number of
        rows (number of strings).
    bytes : str
        Raw data for this parameter.
    '''

    def __init__(self,
                 name,
                 dtype,
                 desc='',
                 bytes_per_element=1,
                 dimensions=None,
                 bytes=b'',
                 handle=None):
        '''Set up a new parameter, only the name is required.'''
        self.name = name
        self.dtypes = dtype
        self.desc = desc
        self.bytes_per_element = bytes_per_element
        self.dimensions = dimensions or []
        self.bytes = bytes
        if handle:
            self.read(handle)

    def __repr__(self):
        return '<Param: {}>'.format(self.desc)

    @property
    def num_elements(self) -> int:
        '''Return the number of elements in this parameter's array value.'''
        e = 1
        for d in self.dimensions:
            e *= d
        return e

    @property
    def total_bytes(self) -> int:
        '''Return the number of bytes used for storing this parameter's data.'''
        return self.num_elements * abs(self.bytes_per_element)

    @property
    def binary_size(self) -> int:
        '''Return the number of bytes needed to store this parameter.'''
        return (
            1 +  # group_id
            2 +  # next offset marker
            1 + len(self.name.encode('utf-8')) +  # size of name and name bytes
            1 +  # data size
            # size of dimensions and dimension bytes
            1 + len(self.dimensions) +
            self.total_bytes +  # data
            1 + len(self.desc.encode('utf-8'))  # size of desc and desc bytes
        )

    def write(self, group_id, handle):
        '''Write binary data for this parameter to a file handle.

        Parameters
        ----------
        group_id : int
            The numerical ID of the group that holds this parameter.
        handle : file handle
            An open, writable, binary file handle.
        '''
        name = self.name.encode('utf-8')
        handle.write(struct.pack('bb', len(name), group_id))
        handle.write(name)
        handle.write(struct.pack('<h', self.binary_size - 2 - len(name)))
        handle.write(struct.pack('b', self.bytes_per_element))
        handle.write(struct.pack('B', len(self.dimensions)))
        handle.write(struct.pack('B' * len(self.dimensions), *self.dimensions))
        if self.bytes is not None and len(self.bytes) > 0:
            handle.write(self.bytes)
        desc = self.desc.encode('utf-8')
        handle.write(struct.pack('B', len(desc)))
        handle.write(desc)

    def read(self, handle):
        '''Read binary data for this parameter from a file handle.

        This reads exactly enough data from the current position in the file to
        initialize the parameter.
        '''
        self.bytes_per_element, = struct.unpack('b', handle.read(1))
        dims, = struct.unpack('B', handle.read(1))
        self.dimensions = [struct.unpack('B', handle.read(1))[
            0] for _ in range(dims)]
        self.bytes = b''
        if self.total_bytes:
            self.bytes = handle.read(self.total_bytes)
        desc_size, = struct.unpack('B', handle.read(1))
        self.desc = desc_size and self.dtypes.decode_string(handle.read(desc_size)) or ''

    def _as(self, dtype):
        '''Unpack the raw bytes of this param using the given struct format.'''
        return np.frombuffer(self.bytes, count=1, dtype=dtype)[0]

    def _as_array(self, dtype, copy=True):
        '''Unpack the raw bytes of this param using the given data format.'''
        if not self.dimensions:
            return [self._as(dtype)]
        elems = np.frombuffer(self.bytes, dtype=dtype)
        # Reverse shape as the shape is defined in fortran format
        view = elems.reshape(self.dimensions[::-1])
        if copy:
            return view.copy()
        return view

class ParamReadonly(object):
    ''' Wrapper exposing readonly attributes of a `c3d.parameter.ParamData` entry.
    '''

    def __init__(self, data):
        self._data = data

    def __eq__(self, other):
        return self._data is other._data

    @property
    def name(self) -> str:
        ''' Get the parameter name. '''
        return self._data.name

    @property
    def desc(self) -> str:
        ''' Get the parameter descriptor. '''
        return self._data.desc

    @property
    def dtypes(self):
        ''' Convenience accessor to the `c3d.dtypes.DataTypes` instance associated with the parameter. '''
        return self._data.dtypes

    @property
    def dimensions(self) -> (int, ...):
        ''' Shape of the parameter data (Fortran format). '''
        return self._data.dimensions

    @property
    def num_elements(self) -> int:
        '''Return the number of elements in this parameter's array value.'''
        return self._data.num_elements

    @property
    def bytes_per_element(self) -> int:
        '''Return the number of bytes used to store each data element.'''
        return self._data.bytes_per_element

    @property
    def total_bytes(self) -> int:
        '''Return the number of bytes used for storing this parameter's data.'''
        return self._data.total_bytes

    @property
    def binary_size(self) -> int:
        '''Return the number of bytes needed to store this parameter.'''
        return self._data.binary_size

    @property
    def int8_value(self):
        '''Get the parameter data as an 8-bit signed integer.'''
        return self._data._as(self.dtypes.int8)

    @property
    def uint8_value(self):
        '''Get the parameter data as an 8-bit unsigned integer.'''
        return self._data._as(self.dtypes.uint8)

    @property
    def int16_value(self):
        '''Get the parameter data as a 16-bit signed integer.'''
        return self._data._as(self.dtypes.int16)

    @property
    def uint16_value(self):
        '''Get the parameter data as a 16-bit unsigned integer.'''
        return self._data._as(self.dtypes.uint16)

    @property
    def int32_value(self):
        '''Get the parameter data as a 32-bit signed integer.'''
        return self._data._as(self.dtypes.int32)

    @property
    def uint32_value(self):
        '''Get the parameter data as a 32-bit unsigned integer.'''
        return self._data._as(self.dtypes.uint32)

    @property
    def uint_value(self):
        ''' Get the parameter data as a unsigned integer of appropriate type. '''
        if self.bytes_per_element >= 4:
            return self.uint32_value
        elif self.bytes_per_element >= 2:
            return self.uint16_value
        else:
            return self.uint8_value

    @property
    def int_value(self):
        ''' Get the parameter data as a signed integer of appropriate type. '''
        if self.bytes_per_element >= 4:
            return self.int32_value
        elif self.bytes_per_element >= 2:
            return self.int16_value
        else:
            return self.int8_value

    @property
    def float_value(self):
        '''Get the parameter data as a floating point value of appropriate type.'''
        if self.bytes_per_element > 4:
            if self.dtypes.is_dec:
                raise AttributeError("64 bit DEC floating point is not supported.")
            # 64-bit floating point is not a standard
            return self._data._as(self.dtypes.float64)
        elif self.bytes_per_element == 4:
            if self.dtypes.is_dec:
                return DEC_to_IEEE(self._data._as(np.uint32))
            else:  # is_mips or is_ieee
                return self._data._as(self.dtypes.float32)
        else:
            raise AttributeError("Only 32 and 64 bit floating point is supported.")

    @property
    def bytes_value(self) -> bytes:
        '''Get the raw byte string.'''
        return self._data.bytes

    @property
    def string_value(self):
        '''Get the parameter data as a unicode string.'''
        return self.dtypes.decode_string(self._data.bytes)

    @property
    def int8_array(self):
        '''Get the parameter data as an array of 8-bit signed integers.'''
        return self._data._as_array(self.dtypes.int8)

    @property
    def uint8_array(self):
        '''Get the parameter data as an array of 8-bit unsigned integers.'''
        return self._data._as_array(self.dtypes.uint8)

    @property
    def int16_array(self):
        '''Get the parameter data as an array of 16-bit signed integers.'''
        return self._data._as_array(self.dtypes.int16)

    @property
    def uint16_array(self):
        '''Get the parameter data as an array of 16-bit unsigned integers.'''
        return self._data._as_array(self.dtypes.uint16)

    @property
    def int32_array(self):
        '''Get the parameter data as an array of 32-bit signed integers.'''
        return self._data._as_array(self.dtypes.int32)

    @property
    def uint32_array(self):
        '''Get the parameter data as an array of 32-bit unsigned integers.'''
        return self._data._as_array(self.dtypes.uint32)

    @property
    def int64_array(self):
        '''Get the parameter data as an array of 32-bit signed integers.'''
        return self._data._as_array(self.dtypes.int64)

    @property
    def uint64_array(self):
        '''Get the parameter data as an array of 32-bit unsigned integers.'''
        return self._data._as_array(self.dtypes.uint64)

    @property
    def float32_array(self):
        '''Get the parameter data as an array of 32-bit floats.'''
        # Convert float data if not IEEE processor
        if self.dtypes.is_dec:
            # _as_array but for DEC
            if not self.dimensions:
                return [self.float_value]
            return DEC_to_IEEE_BYTES(self._data.bytes).reshape(self.dimensions[::-1])  # Reverse fortran format
        else:  # is_ieee or is_mips
            return self._data._as_array(self.dtypes.float32)

    @property
    def float64_array(self):
        '''Get the parameter data as an array of 64-bit floats.'''
        # Convert float data if not IEEE processor
        if self.dtypes.is_dec:
            raise ValueError('Unable to convert bytes encoded in a 64 bit floating point DEC format.')
        else:  # is_ieee or is_mips
            return self._data._as_array(self.dtypes.float64)

    @property
    def float_array(self):
        '''Get the parameter data as an array of 32 or 64 bit floats.'''
        # Convert float data if not IEEE processor
        if self.bytes_per_element == 4:
            return self.float32_array
        elif self.bytes_per_element == 8:
            return self.float64_array
        else:
            raise TypeError("Parsing parameter bytes to an array with %i bit " % self.bytes_per_element +
                            "floating-point precission is not unsupported.")

    @property
    def int_array(self):
        '''Get the parameter data as an array of integer values.'''
        # Convert float data if not IEEE processor
        if self.bytes_per_element == 1:
            return self.int8_array
        elif self.bytes_per_element == 2:
            return self.int16_array
        elif self.bytes_per_element == 4:
            return self.int32_array
        elif self.bytes_per_element == 8:
            return self.int64_array
        else:
            raise TypeError("Parsing parameter bytes to an array with %i bit integer values is not unsupported." %
                            self.bytes_per_element)

    @property
    def uint_array(self):
        '''Get the parameter data as an array of integer values.'''
        # Convert float data if not IEEE processor
        if self.bytes_per_element == 1:
            return self.uint8_array
        elif self.bytes_per_element == 2:
            return self.uint16_array
        elif self.bytes_per_element == 4:
            return self.uint32_array
        elif self.bytes_per_element == 8:
            return self.uint64_array
        else:
            raise TypeError("Parsing parameter bytes to an array with %i bit integer values is not unsupported." %
                            self.bytes_per_element)

    @property
    def bytes_array(self):
        '''Get the parameter data as an array of raw byte strings.'''
        # Decode different dimensions
        if len(self.dimensions) == 0:
            return np.array([])
        elif len(self.dimensions) == 1:
            return np.array(self._data.bytes)
        else:
            # Convert Fortran shape (data in memory is identical, shape is transposed)
            word_len = self.dimensions[0]
            dims = self.dimensions[1:][::-1]  # Identical to: [:0:-1]
            byte_steps = np.cumprod(self.dimensions[:-1])[::-1]
            # Generate mult-dimensional array and parse byte words
            byte_arr = np.empty(dims, dtype=object)
            for i in np.ndindex(*dims):
                # Calculate byte offset as sum of each array index times the byte step of each dimension.
                off = np.sum(np.multiply(i, byte_steps))
                byte_arr[i] = self._data.bytes[off:off+word_len]
            return byte_arr

    @property
    def string_array(self):
        '''Get the parameter data as a python array of unicode strings.'''
        # Decode different dimensions
        if len(self.dimensions) == 0:
            return np.array([])
        elif len(self.dimensions) == 1:
            return np.array([self.string_value])
        else:
            # Parse byte sequences
            byte_arr = self.bytes_array
            # Decode sequences
            for i in np.ndindex(byte_arr.shape):
                byte_arr[i] = self.dtypes.decode_string(byte_arr[i])
            return byte_arr

    @property
    def any_value(self):
        ''' Get the parameter data as a value of 'traditional type'.

        Traditional types are defined in the Parameter section in the [user manual].

        Returns
        -------
        value : int, float, or str
            Depending on the `bytes_per_element` field, a traditional type can
            be a either a signed byte, signed short, 32-bit float, or a string.

        [user manual]: https://www.c3d.org/docs/C3D_User_Guide.pdf
        '''
        if self.bytes_per_element >= 4:
            return self.float_value
        elif self.bytes_per_element >= 2:
            return self.int16_value
        elif self.bytes_per_element == -1:
            return self.string_value
        else:
            return self.int8_value

    @property
    def any_array(self):
        ''' Get the parameter data as an array of 'traditional type'.

        Traditional types are defined in the Parameter section in the [user manual].

        Returns
        -------
        value : array
            Depending on the `bytes_per_element` field, a traditional type can
            be a either a signed byte, signed short, 32-bit float, or a string.

        [user manual]: https://www.c3d.org/docs/C3D_User_Guide.pdf
        '''
        if self.bytes_per_element >= 4:
            return self.float_array
        elif self.bytes_per_element >= 2:
            return self.int16_array
        elif self.bytes_per_element == -1:
            return self.string_array
        else:
            return self.int8_array

    @property
    def _as_any_uint(self):
        ''' Attempt to parse the parameter data as any unsigned integer format.
            Checks if the integer is stored as a floating point value.

            Can be used to read 'POINT:FRAMES' or 'POINT:LONG_FRAMES'
            when not accessed through `c3d.manager.Manager.last_frame`.
        '''
        if self.bytes_per_element >= 4:
            # Check if float value representation is an integer
            value = self.float_value
            if float(value).is_integer():
                return int(value)
            return self.uint32_value
        elif self.bytes_per_element >= 2:
            return self.uint16_value
        else:
            return self.uint8_value

class Param(ParamReadonly):
    ''' Wrapper exposing both readable and writable attributes of a `c3d.parameter.ParamData` entry.
    '''
    def __init__(self, data):
        super(Param, self).__init__(data)

    def readonly(self):
        ''' Returns a readonly `c3d.parameter.ParamReadonly` instance. '''
        return ParamReadonly(self._data)

    @property
    def bytes(self) -> bytes:
        ''' Get or set the parameter bytes. '''
        return self._data.bytes

    @bytes.setter
    def bytes(self, value):
        self._data.bytes = value


from flask import jsonify, request, abort
from flask_sqlalchemy import SQLAlchemy

from routes import routes
import views

db = SQLAlchemy

# GET /api/search?kw={kw}
@routes.route("/api/search", methods=['GET'])
def get_search():
    kw = request.args.get('kw', type=str, default='')  # 검색어
    page = request.args.get('page', type=int, default=1)
    try:
        if kw:
            puppies, meta = views.get_search(kw, page)
            return jsonify({
                'success': True,
                'data': puppies,
                'meta': meta
            })

    except:
        abort(500, "Something wrong in database")


import torch
from collections import Counter

def get_entities(seq,id2label,markup='bios'):
    """Gets entities from sequence.
    note: BIOS
    Args:
        seq (list): sequence of labels.
    Returns:
        list: list of (chunk_type, chunk_start, chunk_end).
    Example:
        # >>> seq = ['B-PER', 'I-PER', 'O', 'S-LOC']
        # >>> get_entity_bios(seq)
        [['PER', 0,1], ['LOC', 3, 3]]
    """
    chunks = []
    chunk = [-1, -1, -1]
    for indx, tag in enumerate(seq):
        if not isinstance(tag, str):
            tag = id2label[tag]
        if tag.startswith("S-"):
            if chunk[2] != -1:
                chunks.append(chunk)
            chunk = [-1, -1, -1]
            chunk[1] = indx
            chunk[2] = indx
            chunk[0] = tag.split('-')[1]
            chunks.append(chunk)
            chunk = (-1, -1, -1)
        if tag.startswith("B-"):
            if chunk[2] != -1:
                chunks.append(chunk)
            chunk = [-1, -1, -1]
            chunk[1] = indx
            chunk[0] = tag.split('-')[1]
        elif tag.startswith('I-') and chunk[1] != -1:
            _type = tag.split('-')[1]
            if _type == chunk[0]:
                chunk[2] = indx
            if indx == len(seq) - 1:
                chunks.append(chunk)
        else:
            if chunk[2] != -1:
                chunks.append(chunk)
            chunk = [-1, -1, -1]
    return chunks

class F1(object):
    def __init__(self):
        self.origins = []
        self.founds = []
        self.rights = []
    def compute(self, origin, found, right):
        recall = 0 if origin == 0 else (right / origin)
        precision = 0 if found == 0 else (right / found)
        f1 = 0. if recall + precision == 0 else (2 * precision * recall) / (precision + recall)
        return recall, precision, f1
    def result(self):
        num_examples = len(self.origins)
        origins = 0
        founds = 0
        rights = 0
        for i in range(num_examples):
            origin = self.origins[i]
            found = self.founds[i]
            right = self.rights[i]
            origins += len(origin)
            founds += len(found)
            rights += len(right)
        # print('origins: {}'.format(origins))
        # print('founds: {}'.format(founds))
        # print('rights: {}'.format(rights))
        recall, precision, f1 = self.compute(origins, founds, rights)
        return {'acc': precision, 'recall': recall, 'f1': f1}

    def update(self, pred, label):
        self.origins.append(label)
        self.founds.append(pred)
        self.rights.append(list(set(pred) & set(label)))

class FunctionAcc(object):
    def __init__(self, end_id):
        self.correct = 0
        self.tot = 0
        self.end_id = end_id
    def result(self):
        acc = self.correct / self.tot
        return acc
    def update(self, pred, label):
        match = True
        for i in range(min(len(pred), len(label))):
            if label[i] != pred[i]:
                match = False
                break
            if pred[i] == self.end_id and label[i] == self.end_id:
                break
        if match:
            self.correct += 1
        self.tot += 1

class SeqEntityScore(object):
    def __init__(self, id2label,markup='bios'):
        self.id2label = id2label
        self.markup = markup
        self.reset()

    def reset(self):
        self.origins = []
        self.founds = []
        self.rights = []

    def compute(self, origin, found, right):
        recall = 0 if origin == 0 else (right / origin)
        precision = 0 if found == 0 else (right / found)
        f1 = 0. if recall + precision == 0 else (2 * precision * recall) / (precision + recall)
        return recall, precision, f1

    def result(self):
        class_info = {}
        origin_counter = Counter([x[0] for x in self.origins])
        found_counter = Counter([x[0] for x in self.founds])
        right_counter = Counter([x[0] for x in self.rights])
        for type_, count in origin_counter.items():
            origin = count
            found = found_counter.get(type_, 0)
            right = right_counter.get(type_, 0)
            recall, precision, f1 = self.compute(origin, found, right)
            class_info[type_] = {"acc": round(precision, 4), 'recall': round(recall, 4), 'f1': round(f1, 4)}
        origin = len(self.origins)
        found = len(self.founds)
        right = len(self.rights)
        recall, precision, f1 = self.compute(origin, found, right)
        return {'acc': precision, 'recall': recall, 'f1': f1}, class_info

    def update(self, label_paths, pred_paths):
        '''
        labels_paths: [[],[],[],....]
        pred_paths: [[],[],[],.....]

        :param label_paths:
        :param pred_paths:
        :return:
        Example:
            >>> labels_paths = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]
            >>> pred_paths = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]
        '''
        for label_path, pre_path in zip(label_paths, pred_paths):
            label_entities = get_entities(label_path, self.id2label,self.markup)
            pre_entities = get_entities(pre_path, self.id2label,self.markup)
            self.origins.extend(label_entities)
            self.founds.extend(pre_entities)
            self.rights.extend([pre_entity for pre_entity in pre_entities if pre_entity in label_entities])