import copy
import jax
import math
from typing import Callable, Optional, Any, List, Sequence
from jax import numpy as jnp
from flax import linen as nn
from functools import partial
from .utils import _make_divisible
from .layers.efficientnet_helpers import SqueezeAndExcitation, ConvNormActivation
from .layers.drop import DropPath

# __all__ = [
#     "EfficientNet",
#     "efficientnet_b0",
#     "efficientnet_b1",
#     "efficientnet_b2",
#     "efficientnet_b3",
#     "efficientnet_b4",
#     "efficientnet_b5",
#     "efficientnet_b6",
#     "efficientnet_b7",
# ]

class MBConvConfig:
    # Stores information listed at Table 1 of the EfficientNet paper
    def __init__(
            self,
            expand_ratio: float,
            kernel: int,
            stride: int,
            input_channels: int,
            out_channels: int,
            num_layers: int,
            width_mult: float,
            depth_mult: float,
    ) -> None:
        self.expand_ratio = expand_ratio
        self.kernel = kernel
        self.stride = stride
        self.input_channels = self.adjust_channels(input_channels, width_mult)
        self.out_channels = self.adjust_channels(out_channels, width_mult)
        self.num_layers = self.adjust_depth(num_layers, depth_mult)

    def __repr__(self) -> str:
        s = self.__class__.__name__ + "("
        s += "expand_ratio={expand_ratio}"
        s += ", kernel={kernel}"
        s += ", stride={stride}"
        s += ", input_channels={input_channels}"
        s += ", out_channels={out_channels}"
        s += ", num_layers={num_layers}"
        s += ")"
        return s.format(**self.__dict__)

    @staticmethod
    def adjust_channels(channels: int, width_mult: float, min_value: Optional[int] = None) -> int:
        return _make_divisible(channels * width_mult, 8, min_value)

    @staticmethod
    def adjust_depth(num_layers: int, depth_mult: float):
        return int(math.ceil(num_layers * depth_mult))

class MBConv(nn.Module):
    cnf: MBConvConfig
    drop_path_prob: float
    norm_layer: Callable[..., nn.Module]
    se_layer: Callable[..., nn.Module] = SqueezeAndExcitation
    dtype: Any = jnp.float32

    @nn.compact
    def __call__(self, inputs, train: bool = True):
        if not (1 <= self.cnf.stride <= 2):
            raise ValueError("wrong stride value")

        input_channels = inputs.shape[-1]
        use_res_connect = self.cnf.stride == 1 and input_channels == self.cnf.out_channels
        activation_layer = nn.silu

        expanded_channels = self.cnf.adjust_channels(input_channels, self.cnf.expand_ratio)
        outputs = inputs
        if expanded_channels != input_channels:
            outputs = ConvNormActivation(
                expanded_channels,
                kernel_size=1,
                norm_layer=self.norm_layer,
                activation_layer=activation_layer
            )(outputs, train=train)

        # depthwise
        outputs = ConvNormActivation(
            expanded_channels,
            kernel_size=self.cnf.kernel,
            stride=self.cnf.stride,
            groups=expanded_channels,
            norm_layer=self.norm_layer,
            activation_layer=activation_layer,
        )(outputs, train=train)

        # squeeze and excitation
        squeeze_channels = max(1, input_channels // 4)
        outputs = self.se_layer(expanded_channels, squeeze_channels, activation=activation_layer)(outputs)

        # project
        outputs = ConvNormActivation(
            self.cnf.out_channels, kernel_size=1,
            norm_layer=self.norm_layer, activation_layer=None
        )(outputs, train=train)

        if use_res_connect:
            if self.drop_path_prob != 0.:
                outputs = DropPath(self.drop_path_prob)(outputs, train=train)
            outputs = outputs + inputs
        return outputs

class EfficientNet(nn.Module):
    inverted_residual_setting: List[MBConvConfig]
    drop_rate: float
    drop_path_prob: float = 0.2
    num_classes: Optional[int] = 1000
    block: Optional[Callable[..., nn.Module]] = None
    norm_layer: Optional[Callable[..., nn.Module]] = None
    global_pool: str = "avg"
    dtype: Any = jnp.float32

    @nn.compact
    def __call__(self, inputs, train: bool = True):
        if not self.inverted_residual_setting:
            raise ValueError("The inverted_residual_setting should not be empty")
        elif not (
                isinstance(self.inverted_residual_setting, Sequence)
                and all([isinstance(s, MBConvConfig) for s in self.inverted_residual_setting])
        ):
            raise TypeError("The inverted_residual_setting should be List[MBConvConfig]")

        if self.block is None:
            block = MBConv
        else:
            block = self.block

        if self.norm_layer is None:
            norm_layer = nn.BatchNorm
        else:
            norm_layer = self.norm_layer

        activation_layer = nn.silu

        firstconv_output_channels = self.inverted_residual_setting[0].input_channels
        outputs = ConvNormActivation(firstconv_output_channels, kernel_size=3, stride=2,
                                     norm_layer=norm_layer, activation_layer=activation_layer)(inputs, train=train)

        total_stage_blocks = sum(cnf.num_layers for cnf in self.inverted_residual_setting)
        stage_block_id = 0

        for cnf in self.inverted_residual_setting:
            for ix in range(cnf.num_layers):

                block_cnf = copy.copy(cnf)

                if ix > 0:
                    block_cnf.input_channels = block_cnf.out_channels
                    block_cnf.stride = 1

                sd_prob = self.drop_path_prob * float(stage_block_id) / total_stage_blocks

                outputs = block(block_cnf, norm_layer=norm_layer, drop_path_prob=sd_prob, dtype=self.dtype)(outputs,
                                                                                train=train)
                stage_block_id += 1

        lastconv_output_channels = 4 * self.inverted_residual_setting[-1].out_channels

        outputs = ConvNormActivation(lastconv_output_channels, kernel_size=1, norm_layer=norm_layer,
                                     activation_layer=activation_layer, dtype=self.dtype)(outputs, train=train)

        if self.global_pool == "avg":
            outputs = jnp.mean(outputs, axis=(1, 2))
        elif self.global_pool == "max":
            outputs = jnp.max(outputs, axis=(1, 2))
        else:
            raise ValueError("Unsupported pool type value : {}".format(self.global_pool))

        if self.num_classes is not None:
            if self.drop_rate != 0.:
                outputs = nn.Dropout(self.drop_rate, deterministic=not train, name="fc_dropout")(outputs)
            outputs = nn.Dense(self.num_classes, name="fc")(outputs)

        return outputs

def _efficientnet(
        arch: str,
        width_mult: float,
        depth_mult: float,
        dropout: float,
        **kwargs: Any,
) -> EfficientNet:
    bneck_conf = partial(MBConvConfig, width_mult=width_mult, depth_mult=depth_mult)
    inverted_residual_setting = [
        bneck_conf(1, 3, 1, 32, 16, 1),
        bneck_conf(6, 3, 2, 16, 24, 2),
        bneck_conf(6, 5, 2, 24, 40, 2),
        bneck_conf(6, 3, 2, 40, 80, 3),
        bneck_conf(6, 5, 1, 80, 112, 3),
        bneck_conf(6, 5, 2, 112, 192, 4),
        bneck_conf(6, 3, 1, 192, 320, 1),
    ]
    model = EfficientNet(inverted_residual_setting, dropout, **kwargs)
    return model

def efficientnet_b0(**kwargs: Any):
    return _efficientnet("efficientnet_b0", 1.0, 1.0, 0.2, **kwargs)

def efficientnet_b1(**kwargs: Any) -> EfficientNet:
    return _efficientnet("efficientnet_b1", 1.0, 1.1, 0.2, **kwargs)

def efficientnet_b2(**kwargs: Any) -> EfficientNet:
    return _efficientnet("efficientnet_b2", 1.1, 1.2, 0.3, **kwargs)

def efficientnet_b3(**kwargs: Any) -> EfficientNet:
    return _efficientnet("efficientnet_b3", 1.2, 1.4, 0.3, **kwargs)

def efficientnet_b4(**kwargs: Any) -> EfficientNet:
    return _efficientnet("efficientnet_b4", 1.4, 1.8, 0.4, **kwargs)

def efficientnet_b5(**kwargs: Any) -> EfficientNet:
    return _efficientnet(
        "efficientnet_b5",
        1.6,
        2.2,
        0.4,
        norm_layer=partial(nn.BatchNorm, epsilon=0.001, momentum=0.01),
        **kwargs,
    )

def efficientnet_b6(**kwargs: Any) -> EfficientNet:
    return _efficientnet(
        "efficientnet_b6",
        1.8,
        2.6,
        0.5,
        norm_layer=partial(nn.BatchNorm, epsilon=0.001, momentum=0.01),
        **kwargs,
    )

def efficientnet_b7(**kwargs: Any) -> EfficientNet:
    return _efficientnet(
        "efficientnet_b7",
        2.0,
        3.1,
        0.5,
        norm_layer=partial(nn.BatchNorm, epsilon=0.001, momentum=0.01),
        **kwargs,
    )


# To change this license header, choose License Headers in Project Properties.
# To change this template file, choose Tools | Templates
# and open the template in the editor.
import enum
import unittest

from prompt_toolkit.document import Document
from promshell.prometheus import ArgDescription, OptionKind
from shell.completion import ArgumentCompleter

class TestArgCompleter(unittest.TestCase):

    ARG_DESCRIPTION = [
        ArgDescription(
                kind=OptionKind.POSITIONAL,
                name ='<pos1>',
                value=[]),
        ArgDescription(
                kind=OptionKind.SINGLE,
                name ='-o1',
                value=['val1', 'val2']),
        ArgDescription(
                kind=OptionKind.SINGLE,
                name ='-o2',
                value=['<o2_value>'])
    ]

    class CaseKind(enum.IntEnum):
        DASH_ONLY = 0
        DASH_ONLY_SPACE = 1
        SINGLE_OPTION = 2
        SINGLE_OPTION_W_SPACE = 3
        SINGLE_OPTION_W_VALUE = 4
        SINGLE_OPTION_W_VALUE_AND_SPACE = 5
        SINGLE_OPTION_W_VALUE_W_POSITIONAL = 6
        SINGLE_OPTION_W_VALUE_W_POSITIONAL_AND_SPACE = 7
        ONE_POSITIONAL = 8
        ONE_POSITIONAL_AND_SPACE = 9
        POSITIONAL_W_OPTION = 10
        POSITIONAL_W_OPTION_AND_SPACE = 11
        POSITIONAL_W_OPTION_W_VALUE = 12
        POSITIONAL_W_OPTION_W_VALUE_AND_SPACE = 13
        TWO_OPTIONS = 14
        TWO_OPTIONS_AND_SPACE = 15
        TWO_OPTIONS_W_VALUE = 16
        TWO_OPTIONS_W_VALUE_AND_SPACE = 17

    TEST_CASES = [
        dict(line='-', expected=['<pos1>', '-o1', '-o2']), #DASH_ONLY
        dict(line='- ', expected=['-o1', '-o2']), #DASH_ONLY_SPACE
        dict(line='-o1', expected=['<pos1>', '-o1', '-o2']), #SINGLE_OPTION
        dict(line='-o1 ', expected=ARG_DESCRIPTION[1].choices), #3INGLE_OPTION_W_SPACE
        dict(line='-o1 value', expected=['val1', 'val2']), #SINGLE_OPTION_W_VALUE
        dict(line='-o1 value ', expected=['<pos1>', '-o2']), #SINGLE_OPTION_W_VALUE_AND_SPACE
        dict(line='-o1 value pos1', expected=['<pos1>', '-o2']), #SINGLE_OPTION_W_VALUE_W_POSITIONAL
        dict(line='-o1 value pos1 ', expected=['-o2']),  #SINGLE_OPTION_W_VALUE_W_POSITIONAL_AND_SPACE
        dict(line='pos1', expected=['<pos1>', '-o1', '-o2']),  # ONE_POSITIONAL
        dict(line='pos1 ', expected=['-o1', '-o2']),  #ONE_POSITIONAL_AND_SPACE
        dict(line='pos1 -o1', expected=['-o1', '-o2']),  # POSITIONAL_W_OPTION
        dict(line='pos1 -o1 ', expected=ARG_DESCRIPTION[1].choices),  # POSITIONAL_W_OPTION_AND_SPACE
        dict(line='pos1 -o1 value', expected=ARG_DESCRIPTION[1].choices),  # POSITIONAL_W_OPTION_W_VALUE
        dict(line='pos1 -o1 value ', expected=['-o2']),  # POSITIONAL_W_OPTION_W_VALUE_AND_SPACE
        dict(line='-o1 -o2', expected=ARG_DESCRIPTION[1].choices),  # TWO_OPTIONS
        dict(line='-o1 -o2 ', expected=ARG_DESCRIPTION[2].choices),  # TWO_OPTIONS_AND_SPACE
        dict(line='-o1 -o2 val', expected=ARG_DESCRIPTION[2].choices),  # TWO_OPTIONS_W_VALUE
        dict(line='-o1 -o2 val ', expected=['<pos1>']),  # TWO_OPTIONS_W_VALUE_AND_SPACE
    ]

    #    def setUp(self):
    #        self.foo = Tests()

    #def tearDown(self):
    #    self.foo.dispose()
    #    self.foo = None

    def check_completions(self, arg_description, test_case):
        expected_completions = test_case['expected'].copy()
        expected_completions.append('')
        completer = ArgumentCompleter(arg_description)
        document = Document(test_case['line'], cursor_position=0)

        completed = []
        for item in completer.get_completions(document, None):
            completed.append(item.text)

        self.assertEqual(expected_completions, completed)

    def test_dash_only(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.DASH_ONLY.value])

    def test_dash_only_space(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.DASH_ONLY_SPACE.value])

    def test_single_option(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.SINGLE_OPTION.value])

    def test_single_option_with_space(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.SINGLE_OPTION_W_SPACE.value])

    def test_single_option_with_value(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.SINGLE_OPTION_W_VALUE.value])

    def test_single_option_with_value_and_space(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.SINGLE_OPTION_W_VALUE_AND_SPACE.value])

    def test_single_option_with_value_with_pos(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.SINGLE_OPTION_W_VALUE_W_POSITIONAL.value])

    def test_one_positional(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.ONE_POSITIONAL.value])

    def test_one_positional_and_space(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.ONE_POSITIONAL_AND_SPACE.value])

    def test_one_positional_with_option(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.POSITIONAL_W_OPTION.value])

    def test_one_positional_with_option_and_space(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.POSITIONAL_W_OPTION_AND_SPACE.value])

    def test_one_positional_with_option_with_value(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.POSITIONAL_W_OPTION_W_VALUE.value])

    def test_one_positional_with_option_with_value_and_space(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.POSITIONAL_W_OPTION_W_VALUE_AND_SPACE.value])

    def test_two_options(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.TWO_OPTIONS.value])

    def test_two_options_and_space(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.TWO_OPTIONS_AND_SPACE.value])

    def test_two_options_with_value(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.TWO_OPTIONS_W_VALUE.value])

    def test_two_options_with_value_and_space(self):
        self.check_completions(
            TestArgCompleter.ARG_DESCRIPTION,
            TestArgCompleter.TEST_CASES[TestArgCompleter.CaseKind.TWO_OPTIONS_W_VALUE_AND_SPACE.value])


from interpret_text.experimental.classical import ClassicalTextExplainer
from sklearn.preprocessing import LabelEncoder
from utils_test import setup_mnli_test_train_split
import pickle
import os

DOCUMENT = "rare bird has more than enough charm to make it memorable."

class TestClassicalExplainer(object):
    def test_working(self):
        assert True

    def test_explain_model_local_default(self):
        """
        Test for explain_local of classical explainer
        :return:
        """
        X_train, X_test, y_train, y_test = setup_mnli_test_train_split()
        label_encoder = LabelEncoder()
        y_train = label_encoder.fit_transform(y_train)
        explainer = ClassicalTextExplainer()
        classifier, best_params = explainer.fit(X_train, y_train)
        explainer.preprocessor.labelEncoder = label_encoder

        local_explanation = explainer.explain_local(DOCUMENT)
        assert len(local_explanation.local_importance_values) == len(local_explanation.features)

    def test_explain_model_local_with_predicted_label(self):
        """
        Test for explain_local of classical explainer
        :return:
        """
        X_train, X_test, y_train, y_test = setup_mnli_test_train_split()

        label_encoder = LabelEncoder()
        y_train = label_encoder.fit_transform(y_train)
        explainer = ClassicalTextExplainer()
        classifier, best_params = explainer.fit(X_train, y_train)
        explainer.preprocessor.labelEncoder = label_encoder
        y = classifier.predict(DOCUMENT)
        predicted_label = label_encoder.inverse_transform(y)
        local_explanation = explainer.explain_local(DOCUMENT, predicted_label)
        assert len(local_explanation.local_importance_values) == len(local_explanation.features)

    def test_pickle(self, tmpdir):
        """
        Test for pickling of classical explainer
        :return:
        """
        X_train, X_test, y_train, y_test = setup_mnli_test_train_split()
        label_encoder = LabelEncoder()
        y_train = label_encoder.fit_transform(y_train)
        explainer = ClassicalTextExplainer()
        classifier, best_params = explainer.fit(X_train, y_train)
        explainer_file_name = 'explainer.pkl'
        explainer_file_path = tmpdir.mkdir('explainers').join(explainer_file_name)
        with open(explainer_file_path, 'wb') as explainer_save_file:
            pickle.dump(explainer, explainer_save_file)
        with open(explainer_file_path, 'rb') as explainer_load_file:
            explainer = pickle.load(explainer_load_file)
        os.remove(explainer_file_path)
        explainer.preprocessor.labelEncoder = label_encoder
        local_explanation = explainer.explain_local(DOCUMENT)
        assert len(local_explanation.local_importance_values) == len(local_explanation.features)


# -*- coding: utf-8 -*-
"""
Created on Sat Feb 11 12:04:43 2017

@author: vbhaumik
"""
import numpy as np
import matplotlib.pyplot as plt
data = np.random. randn(1000)
f,(ax1,ax2) = plt.subplots(1 , 2, figsize=(6,3))
# histogram (pdf)
ax1.hist (data , bins=30, normed=True, color='b' )
# empirical cdf
ax2.hist (data , bins=30, normed=True, color='r',cumulative=True)
plt.savefig('histogram.pdf')


# import csv
import json
import sys
import traceback
from functools import partial
from typing import Dict, List, Optional, Tuple

from pka_lookup_pubchem import pka_lookup_pubchem
from tinydb import Query, TinyDB
from tinydb.middlewares import CachingMiddleware
from tinydb.storages import JSONStorage

debug = False

# Create print alias to pretty print a dict (sorted keys, indent)
def pprint(nested_element):
    """Pretty printing a nested element such as dict"""

    style = partial(json.dumps, indent=4, sort_keys=True, ensure_ascii=False)
    print(style(nested_element))

def search_pka(identifier: str, database) -> Optional[List[Tuple[Dict]]]:
    """Search for pKa in current local database, if not found
     then search Pubchem. If result is found from Pubchem, then
     add to the local database

    Parameters
    ----------
    identifier : str
        search string. Could be one of (ranked likely search)
        - CAS number ~ InChIKey ~ InChI > IUPAC name >> SMILES
    database : tinyDB tiny.database object
        the name of the database the search query need

    Returns
    -------
    Optional[List[Tuple[Dict]]]: either
        - The record(s) found in local database in the following format:
            [
                (record1_id, record1 [Dict]),
                (record2_id, record2 [Dict]),
                ...
            ]
        - The record found in Pubchem: i.e.
            {"Canonical_SMILES": "CO",
            "IUPAC_Name": "methanol",
            "InChI": "InChI=1S/CH4O/c1-2/h2H,1H3",
            "InChIKey": "OKKJLVBELUTLKV-UHFFFAOYSA-N",
            "Isomeric_SMILES": "CO",
            "Pubchem_CID": "887",
            "Substance_CASRN": "67-56-1",
            "pKa": "15.3",
            "reference": "Serjeant, E.P., Dempsey B.;...",
            "source": "Pubchem"}
        - None: if not found any

    """
    global debug

    if len(sys.argv) == 2 and sys.argv[1] in ['--debug=True', '--debug=true', '--debug', '-d']:
        debug = True

    try:
        # Search local DB first:
        db_result = search_db(identifier=identifier, database=database)

        if db_result:
            # return db_result
            return [record for _, record in db_result]

        # If record(s) NOT found, search in Pubchem
        elif not db_result:
            # Get pubchem result
            pubchem_result = pka_lookup_pubchem(identifier)

            # Get pubchem result InChIKey
            pubchem_result_inchikey = pubchem_result.get('InChIKey') if pubchem_result else None

            # Add pubchem result (if found) into local DB:
            if pubchem_result_inchikey:
                '''Use pubchem_result_inchikey to search in current db:
                A bit redundant to check if current DB has the entry again
                but this is for EXACT match'''
                current_db_result = search_db(identifier=pubchem_result_inchikey, database=db)
                record_existed = False
                if current_db_result:
                    for db_id, result in current_db_result:
                        if pubchem_result == result:
                            # Do not add into current db
                            if debug:
                                print('Will not add into current DB')    # for trouble shooting
                            record_existed = True
                            break

                # Add record from pubchem
                if not record_existed:
                    if debug:
                        print('Will add into current DB')    # for trouble shooting
                        # print('Exiting record id with same InChIKey: {}'.format(db_id))    # for trouble shooting

                    # Add into current DB
                    db.insert(pubchem_result)
                    return pubchem_result

    except Exception as error:
        if debug:
            traceback_str = ''.join(traceback.format_exception(etype=type(error), value=error, tb=error.__traceback__))
            print(traceback_str)

        return None

def search_db(identifier: str, database) -> Optional[List[Tuple[Dict]]]:
    """Return a list of (result'ID, result) from pka search using local database

    Parameters
    ----------
    identifier : str
        search keywords. Could be one of (ranked likely search)
        - CAS number ~ InChIKey ~ InChI > IUPAC name >> SMILES
    database : tinyDB tiny.database object
        the name of the database the search query need

    Returns
    -------
    Optional[List[Tuple[Dict]]]
        The record(s) found in local database in the following format:
            [
                (record1_id, record1 [Dict]),
                (record2_id, record2 [Dict]),
                ...
            ]
        None: if not found any
    """

    if identifier and database:
        results = database.search(
                    (Query()['Substance_CASRN'] == identifier) |
                    (Query()['IUPAC_Name'] == identifier) |
                    (Query()['InChI'] == identifier) |
                    (Query()['InChIKey'] == identifier) |
                    (Query()['Original_SMILES'] == identifier) |
                    (Query()['Structure_SMILES'] == identifier) |
                    (Query()['Canonical_SMILES'] == identifier) |
                    (Query()['Isomeric_SMILES'] == identifier)
                )
        return [(record.doc_id, record) for record in results]

if __name__ == "__main__":
    # Access local tinyDB
    db = TinyDB('src/data/tinydb_db.json',
                sort_keys=True,
                indent=4,
                storage=CachingMiddleware(JSONStorage))    # Using caching for faster performance

    try:
        identifiers = [
            # '64-19-7',    # acetic acid   >>> pKa = 4.76 at 25 °C
            # '2950-43-8',    # Hydroxylamine-O-sulfonic acid, no result
            # 'InChI=1S/C6H6S/c7-6-4-2-1-3-5-6/h1-5,7H',    # thiophenol
            # 'C1=CC=C(C=C1)O',    # phenol   >>> pKa = 9.9
            '596-09-8',
            '141-78-6',
            '110-82-7',
            '67-63-0',
            '75-09-2',
        ]

        for identifier in identifiers:
            print('Searching for pKa of structure with identifier: {}'.format(identifier))
            result = search_pka(identifier=identifier, database=db)
            pprint(result)

    except Exception as error:
        if debug:
            traceback_str = ''.join(traceback.format_exception(etype=type(error), value=error, tb=error.__traceback__))
            print(traceback_str)

    finally:
        # Because CachingMIddleware is used, db.close() is required to save to disk after some operations!
        # db.storage.flush()
        db.close()


## Log Transform

import scipy.misc
import matplotlib.pyplot as plt
from scipy.misc.pilutil import Image
import numpy

# opening the image and converting it to grayscale
a = Image.open('lenna.png').convert('L')

# a is converted to an ndarray
b = scipy.misc.fromimage(a)

# gamma is initialized
gamma1 = .5
gamma2 = 4

# b is converted to type float
b1 = b.astype(float)

# maximum value in b1 is determined
b2 = numpy.max(b1)

# Log transform is computed
c = (255.0*numpy.log(1+b1))/numpy.log(1+b2)

# c is converted to type int
c1 = c.astype(int)

# c1 is converted from ndarray to image
d1= scipy.misc.toimage(c1)

# make new figure with 2 subfigures each subfigure can have an image in it
fig = plt.figure()
image1 = plt.subplot(121)
image2 = plt.subplot(122)

#put the images into the window
_ = image1.imshow(a)
_ = image2.imshow(d1)

#hide axis and show window with images
image1.axis("on")
image2.axis("on")
plt.show()


import warnings
warnings.filterwarnings('ignore')
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
plt.style.use('fivethirtyeight')
from pylab import rcParams
rcParams['figure.figsize'] = 10, 7

df = pd.read_csv('Electric_Production.csv')

df.columns=['Date', 'Consumption']
df.dropna()
df['Date'] = pd.to_datetime(df['Date'])
df.set_index('Date', inplace=True)
df.head()

plt.xlabel("Date")
plt.ylabel("Consumption")
plt.title("production graph")
plt.plot(df)

df.plot(style='k.')

from statsmodels.tsa.seasonal import seasonal_decompose
result = seasonal_decompose(df, model='multiplicative')
result.plot()
plt.show()


import aiohttp
import asyncio
import async_timeout

async def main():
    pass

async def request(url):
    async def get(url):
        if request_type == 'get':
            async with aiohttp.ClientSession() as session:
                body = await fetch(session, url, None, 'get')
                return body
        if headers is None:
            async with aiohttp.ClientSession() as session:
                body = await fetch(session, url, data, request_type)
                return body
        else:
            async with aiohttp.ClientSession(headers=headers) as session:
                body = await fetch(session, url, data, request_type)
                return body

async def fetch(session, url, data, request_type):
    with async_timeout.timeout(10):
        # database read API
        if request_type == 'get':
            async with session.get(url) as response:
                response_content = await response.read()
                if response.status != 200:
                    raise ConnectionError(f'url:{url}, response:{response_content.decode()}')
                return response_content
        # database write API
        elif type(data) == dict:
            async with session.post(url, json=data) as response:
                response_content = await response.read()
                if response.status != 200:
                    raise ConnectionError(f'url:{url}, data:{data}, response:{response_content.decode()}')
                return response_content
        # Google TTS
        else:
            async with session.post(url, data=data) as response:
                response_content = await response.read()
                if response.status != 200:
                    raise ConnectionError(f'url:{url}, data:{data}, response:{response_content.decode()}')
                return response_content

if __name__ == '__main__':
    loop = asyncio.get_running_loop()
    if loop is None:
        loop = asyncio.get_event_loop()
    loop.run_until_complete(main())


import json
import sys
import struct

if len(sys.argv) < 2:
    exit(-1)

print("Reading " + sys.argv[1] + "...")

inputFileName = sys.argv[1]
with open(inputFileName) as inputFile:
    jsonData = json.load(inputFile)

print("Done!")

width = jsonData["map"]["width"]
height = jsonData["map"]["height"]
mapData = jsonData["map"]["data"]

print("Map: " + width + "x" + height)

destFileName = inputFileName.replace(".json", ".map")

print("Writing " + destFileName + "...")

destFile = open(destFileName, "w")
destFile.write(struct.pack('B', int(width)))
destFile.write(struct.pack('B', int(height)))
for i in mapData:
    destFile.write(bytearray(struct.pack('B', int(i))))

print("Done!")


class Employee:

   #class variable
   num_of_emps=0
   raise_amount = 1.04

   def __init__(self,first,last,pay):
       self.first=first
       self.last=last
       self.pay=pay
       self.email=f'{first}{last}@company.com'

       Employee.num_of_emps+=1

   def full_name(self):
       return '{}{}'.format(self.first, self.last)

   def apply_raise(self):
       self.pay=int(self.pay*self.raise_amount)

   #We work with the class instead of instance when using this method
   @classmethod
   def set_raise_amt(cls,amount):
       cls.raise_amount=amount

   @classmethod
   def from_string(cls,emp_str):
       first,last,pay=emp_str.split('-')
       return cls(first,last,pay)

   #Do not take an instance or method as first argument
   #Used when the method does not have to use an instance or the class
   #any where within the function
   @staticmethod
   def is_workday(day):
       if (day.weekday() == 5) or (day.weekday() == 6):
           return False
       return True

emp_1=Employee('Caleb', 'Mbugua',90000)
emp_2=Employee('Mercy', 'Wanjiru',70000)

#calling the classmethod
# Employee.set_raise_amt(1.09)

# #class methods as alternative constructors
# ##When i want to do some manipulation to the data
# ##then pass my instance,this is a very good candidate for a class method

# emp_str_1='Mbugua-Kimani-10000'
# emp_str_2='Kigen-Andrew-7000'
# emp_str_3='Lucy-Wamboi-30000'

# #class method as an aternative constructor
# new_emp_1=Employee.from_string(emp_str_1)

# print(new_emp_1.email)
# print(new_emp_1.pay)

##Static Methods
#Are normal methods used in a class and they neither take the instance or the class
#as the first argument

import datetime
my_date=datetime.date(2021,6,5)

#There are times i want to call methods without necessarily having
#to instantate a class..I can call these from the class directly.
print(Employee.is_workday(my_date))

##They are methods that have a logical connection with the class

#i can also run my class method from and instance but this is not advisable

# emp_1.set_raise_amt(1.25)

# print(emp_1.raise_amount)
# print(emp_2.raise_amount)

#Regular methods in a class automatically take the instance as the first argument(self)

#class Methods take in the class as the first argument

#whenw e are working with class variables we are working with the class instead of the instance

#we add a decorator class


# coding: utf-8

import os,glob,re
import numpy as np
import tensorflow as tf
from numpy.random import randint, choice
from multiprocessing import Pool
os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
import horovod.tensorflow as hvd
hvd_valid = True

hvd.init()

in_dim=(3,10)
out_dim=(3,)
epochs=2000
drop=0.1

strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()

with strategy.scope():
      layers={}

      layers[0]=tf.keras.layers.Input(shape=in_dim)

      layers[1]=tf.keras.layers.Dense(32)(layers[0])
      layers[2]=tf.keras.layers.Dense(3)(layers[1])

      model = tf.keras.models.Model(layers[0], layers[2])
      model.summary()

      model.compile(optimizer=tf.keras.optimizers.Adamax(), loss='mse')

      checkpoint_path = "time.ckpt"
      checkpoint_dir = os.path.dirname(checkpoint_path)

      #model.load_weights(checkpoint_path)

      cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
                                                       save_weights_only=True,
                                                       verbose=1)

      n=10000
      data=np.load('63.npy')[:n]
      print(data)
      for e in range(10000):
          print("hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh")
          ins=data[:-10]
          ins=np.array([data[i:i+10] for i in range(n-10)])
          outs=data[10:]
          while True:
              print("##################################################################")
              history = model.fit(ins,outs, epochs=epochs, validation_split=0.02, callbacks=[cp_callback])


import sqlite3

class Wish:
    def __init__(self, db):
        self.db = db
        self.cursor = db.cursor()

    def __repr__(self):
        return f'Wish<{self.title}'

    def __str__(self):
        return f'Wish<{self.title}'

    def get_all(self):
        sql_code = '''SELECT * FROM wish'''
        try:
            print(self.cursor.execute(sql_code))
            wishes = self.cursor.fetchall()
            print('wishes: ', wishes)
            if wishes: return wishes
        except sqlite3.Error as e:
            print('Ошибка чтения из БД', e)
        return []

    def add_wish(self, title, price, url, description):
        self.title = title
        try:
            self.cursor.execute('INSERT INTO wish VALUES(NULL, ?, ?, ?, ?)',
                                (title, price, url, description))
            self.db.commit()
        except sqlite3.Error as e:
            print('Ошибка добавление в БД', e)
            return False
        return True

    def edit_wish(self, wish_id, title, price, url, description):
        try:
            self.cursor.execute('UPDATE wish SET title=?, price=?, url=?, description=? WHERE id=?',
                                (title, price, url, description, str(wish_id)))
            self.db.commit()
        except sqlite3.Error as e:
            print('Ошибка обновления записи в БД', e)
            return False
        return True

    def delete_wish(self, wish_id):
        try:
            self.cursor.execute('DELETE FROM wish WHERE id=' + str(wish_id))
            self.db.commit()
        except sqlite3.Error as e:
            print('Ошибка удаления записи', e)
            return False
        return True

    def get_wish(self, wish_id):
        try:
            self.cursor.execute(f"SELECT * FROM wish WHERE id = {wish_id} LIMIT 1")
            res = self.cursor.fetchone()
            if res:
                print(*res)
                return res
        except sqlite3.Error as e:
            print("Ошибка получения статьи из БД " + str(e))

        return (False, False)


# Author  : Ki-Hwan Kim
# Purpose : Update function for the CPML (Convolutional Perfectly Matched Layer)
# Target  : GPU using PyOpenCL
# Created : 2012-01-30
# Modified:

import numpy as np
import pyopencl as cl

from kemp.fdtd3d.util import common, common_gpu
from fields import Fields

class Pml:
    def __init__(self, fields, directions, npml=10, sigma_max=4, kappa_max=1, alpha_max=0, m_sigma=3, m_alpha=1):
        """
        """

        common.check_type('fields', fields, Fields)
        common.check_type('directions', directions, (list, tuple), str)
        common.check_type('npml', npml, int)
        common.check_type('sigma_max', sigma_max, (int, float))
        common.check_type('kappa_max', kappa_max, (int, float))
        common.check_type('alpha_max', alpha_max, (int, float))
        common.check_type('m_sigma', m_sigma, (int, float))
        common.check_type('m_alpha', m_alpha, (int, float))

        assert len(directions) == 3
        for axis in directions:
            assert axis in ['+', '-', '+-', '']

        # local variables
        dt = fields.dt
        nx, ny, nz = fields.ns
        dtype = fields.dtype
        context = fields.context
        queue = fields.queue

        # allocations
        fx = np.zeros((2*npml, ny, nz), dtype)
        fy = np.zeros((nx, 2*npml, nz), dtype)
        fz = np.zeros((nx, ny, 2*npml), dtype)
        psix_bufs = [cl.Buffer(context, cl.mem_flags.READ_WRITE, fx.nbytes) for i in range(4)]
        psiy_bufs = [cl.Buffer(context, cl.mem_flags.READ_WRITE, fy.nbytes) for i in range(4)]
        psiz_bufs = [cl.Buffer(context, cl.mem_flags.READ_WRITE, fz.nbytes) for i in range(4)]
        for psi_bufs, f in zip([psix_bufs, psiy_bufs, psiz_bufs], [fx, fy, fz]):
            for psi_buf in psi_bufs:
                cl.enqueue_copy(queue, psi_buf, f)
        del fx, fy, fz

        i_half = np.arange(0.5, npml, 1, dtype)
        i_one = np.arange(1, npml+1, 1, dtype)
        sigma_half = sigma_max * (i_half / npml) ** m_sigma
        sigma_one = sigma_max * (i_one / npml) ** m_sigma
        kappa_half = 1 + (kappa_max - 1) * (i_half / npml) ** m_sigma
        kappa_one = 1 + (kappa_max - 1) * (i_one / npml) ** m_sigma
        alpha_half = alpha_max * ((npml - i_half) / npml) ** m_alpha
        alpha_one = alpha_max * ((npml - i_one) / npml) ** m_alpha
        pcb_half = np.exp(-(sigma_half / kappa_half + alpha_half) * dt)
        pcb_one = np.exp(-(sigma_one / kappa_one + alpha_one) * dt)
        pca_half = sigma_half / (sigma_half + alpha_half * kappa_half) * (pcb_half - 1)
        pca_one = sigma_one / (sigma_one + alpha_one * kappa_one) * (pcb_one - 1)

        pc_bufs = [cl.Buffer(context, cl.mem_flags.READ_ONLY, pcb_half.nbytes) for i in range(4)]
        for pc_buf, pc in zip(pc_bufs, [pcb_half, pcb_one, pca_half, pca_one]):
            cl.enqueue_copy(queue, pc_buf, pc)

        # modify reciprocal ds
        if kappa_max != 1:
            erds, hrds = fields.get_rds()
            for erd, hrd, pms in zip(erds, hrds, directions):
                if '+' in pms:
                    erd[-npml-1:-1] /= kappa_half
                    hrd[-npml:] /= kappa_one
                if '-' in pms:
                    erd[:npml] /= kappa_one
                    hrd[1:npml+1] /= kappa_half

        # program
        nmax = {'x': 'npml*ny*nz', 'y': 'nx*npml*nz', 'z': 'nx*ny*npml'}
        idx_psi = {'+': 'idx+nmax', '-': 'idx'}
        idx_pc = {'x': {'+': 'idx/(ny*nz)', '-': 'npml-1-idx/(ny*nz)'}, \
                  'y': {'+': '(idx/nz)%npml', '-': 'npml-1-(idx/nz)%npml'}, \
                  'z': {'+': 'idx%npml', '-': 'npml-1-idx%npml'}}
        if0 = {'x': 'idx', 'y': 'idx+(idx/(npml*nz))*(ny-npml)*nz', 'z': 'idx+(idx/npml)*(nz-npml)'}
        idx_f1 = {'E': {'x': {'+': if0['x']+'+(nx-npml-1)*ny*nz', '-': if0['x']}, \
                        'y': {'+': if0['y']+'+(ny-npml-1)*nz', '-': if0['y']}, \
                        'z': {'+': if0['z']+'+(nz-npml-1)', '-': if0['z']}}, \
                  'H': {'x': {'+': if0['x']+'+(nx-npml)*ny*nz', '-': if0['x']+'+ny*nz'}, \
                        'y': {'+': if0['y']+'+(ny-npml)*nz', '-': if0['y']+'+nz'}, \
                        'z': {'+': if0['z']+'+(nz-npml)', '-': if0['z']+'+1'}}}
        idx_f2 = {'E': {'x': 'if1+ny*nz', 'y': 'if1+nz', 'z': 'if1+1'}, \
                  'H': {'x': 'if1', 'y': 'if1', 'z': 'if1'}}
        idx_f3 = {'E': {'x': 'if1', 'y': 'if1', 'z': 'if1'}, \
                  'H': {'x': 'if1-ny*nz', 'y': 'if1-nz', 'z': 'if1-1'}}

        values_const = ['', '0.5', '0.5']
        values_array = [', __global DTYPE *cf1, __global DTYPE *cf2', 'cf1[if1]', 'cf2[if1]']

        programs = {}
        for xyz in ['x', 'y', 'z']:
            programs[xyz] = {}
            for eh in ['E', 'H']:
                programs[xyz][eh] = {}
                for pm in ['+', '-']:
                    macros = ['NMAX', 'IDX_PSI', 'IDX_PC', 'IDX_F1', 'IDX_F2', 'IDX_F3', 'ARGS_CF', 'CF1', 'CF2', 'DTYPE', 'PRAGMA_fp64']
                    values = [nmax[xyz], idx_psi[pm], idx_pc[xyz][pm], idx_f1[eh][xyz][pm], idx_f2[eh][xyz], idx_f3[eh][xyz]]
                    if (eh == 'E' and fields.ce_on) or (eh == 'H' and fields.ch_on):
                        values += values_array
                    else:
                        values += values_const
                    values += fields.dtype_str_list
                    ksrc = common.replace_template_code(open(common_gpu.src_path + 'cpml.cl').read(), macros, values)
                    programs[xyz][eh][pm] = cl.Program(context, ksrc).build()

        # arguments
        ex, ey, ez, hx, hy, hz = fields.eh_bufs
        if fields.ce_on:
            cex, cey, cez = fields.ce_bufs
            ces = {'x': [cey, cez], 'y': [cez, cex], 'z': [cex, cey]}
        else:
            ces = {'x': [], 'y': [], 'z': []}
        if fields.ch_on:
            chx, chy, chz = fields.ch_bufs
            chs = {'x': [chz, chy], 'y': [chx, chz], 'z': [chy, chx]}
        else:
            chs = {'x': [], 'y': [], 'z': []}
        psi_eyx, psi_ezx, psi_hyx, psi_hzx = psix_bufs
        psi_ezy, psi_exy, psi_hzy, psi_hxy = psiy_bufs
        psi_exz, psi_eyz, psi_hxz, psi_hyz = psiz_bufs
        pcb_half, pcb_one, pca_half, pca_one = pc_bufs

        args0 = fields.ns + [np.int32(npml)]
        arguments = {'x': {'E': {'+': args0 + [ey, ez, hz, hy, psi_eyx, psi_ezx, pcb_half, pca_half] + ces['x'], \
                                 '-': args0 + [ey, ez, hz, hy, psi_eyx, psi_ezx, pcb_one, pca_one] + ces['x']}, \
                           'H': {'+': args0 + [hz, hy, ey, ez, psi_hzx, psi_hyx, pcb_one, pca_one] + chs['x'], \
                                 '-': args0 + [hz, hy, ey, ez, psi_hzx, psi_hyx, pcb_half, pca_half] + chs['x']}}, \
                     'y': {'E': {'+': args0 + [ez, ex, hx, hz, psi_ezy, psi_exy, pcb_half, pca_half] + ces['y'], \
                                 '-': args0 + [ez, ex, hx, hz, psi_ezy, psi_exy, pcb_one, pca_one] + ces['y']}, \
                           'H': {'+': args0 + [hx, hz, ez, ex, psi_hxy, psi_hzy, pcb_one, pca_one] + chs['y'], \
                                 '-': args0 + [hx, hz, ez, ex, psi_hxy, psi_hzy, pcb_half, pca_half] + chs['y']}}, \
                     'z': {'E': {'+': args0 + [ex, ey, hy, hx, psi_exz, psi_eyz, pcb_half, pca_half] + ces['z'], \
                                 '-': args0 + [ex, ey, hy, hx, psi_exz, psi_eyz, pcb_one, pca_one] + ces['z']}, \
                           'H': {'+': args0 + [hy, hx, ex, ey, psi_hyz, psi_hxz, pcb_one, pca_one] + chs['z'], \
                                 '-': args0 + [hy, hx, ex, ey, psi_hyz, psi_hxz, pcb_half, pca_half] + chs['z']}}}

        # global variables
        self.mainf = fields
        self.directions = directions
        self.psix_bufs = psix_bufs
        self.psiy_bufs = psiy_bufs
        self.psiz_bufs = psiz_bufs
        self.pc_bufs = pc_bufs
        self.programs = programs
        self.arguments = arguments

        nn = {'x': int(npml*ny*nz), 'y': int(nx*npml*nz), 'z': int(nx*ny*npml)}
        self.gs = {}
        for key, val in nn.items():
            remainder = val % fields.ls
            self.gs[key] = val if remainder == 0 else val - remainder + fields.ls

        # append to the update list
        self.priority_type = 'pml'
        fields.append_instance(self)

    def update(self, eh):
        for xyz, pms in zip(['x', 'y', 'z'], self.directions):
            for pm in pms:
                self.programs[xyz][eh][pm].update(self.mainf.queue, (self.gs[xyz],), (self.mainf.ls,), *self.arguments[xyz][eh][pm])

    def update_e(self):
        self.update('E')

    def update_h(self):
        self.update('H')


import argparse
import collections
import torch
import numpy as np
import data_loader.data_loaders as module_data
import model.loss as module_loss
import model.metric as module_metric
import model.model as module_arch
from parse_config import ConfigParser
from trainer import Trainer
from playground.corrupt import Fault
from trainer.fault_trainer import  FaultTrainer
from utils import prepare_device

# fix random seeds for reproducibility
SEED = 123
torch.manual_seed(SEED)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(SEED)

def main(config):
    logger = config.get_logger('train')

    # setup data_loader instances
    data_loader = config.init_obj('data_loader', module_data)
    print("Training Size:", data_loader.dataset.data.shape)
    valid_data_loader = data_loader.split_validation()
    print("Validation Size:",valid_data_loader.dataset.data.shape)

    # build model architecture, then print to console
    model = config.init_obj('arch', module_arch)
    logger.info(model)

    # prepare for (multi-device) GPU training
    device, device_ids = prepare_device(config['n_gpu'])
    model = model.to(device)
    if len(device_ids) > 1:
        model = torch.nn.DataParallel(model, device_ids=device_ids)

    # get function handles of loss and metrics
    criterion = getattr(module_loss, config['loss'])
    metrics = [getattr(module_metric, met) for met in config['metrics']]

    # build optimizer, learning rate scheduler. delete every lines containing lr_scheduler for disabling scheduler
    trainable_params = filter(lambda p: p.requires_grad, model.parameters())
    optimizer = config.init_obj('optimizer', torch.optim, trainable_params)
    lr_scheduler = config.init_obj('lr_scheduler', torch.optim.lr_scheduler, optimizer)

    # construct faults given index and layer
    try:
        layers = config['fault']['layer'].split('_')
        fault_indices = config ['fault']['index']
        if isinstance(fault_indices,str):
            fault_indices=[[int(c) for c in fault_index.split(',')] for fault_index in fault_indices.split('_') ]
        times =list(map(int ,config['fault']['time'].split('_')))
        assert len(times)==len(layers)==len(fault_indices)
        faults = []
        for layer,fault_indice,time in zip(layers,fault_indices,times):
            fault = Fault(fault_layer=layer,fault_index=fault_indice,time=time)
            faults.append(fault)
    except:
        faults = None
    trainer = FaultTrainer(model, criterion, metrics, optimizer,
                      config=config,
                      device=device,
                      data_loader=data_loader,
                      valid_data_loader=valid_data_loader,
                      lr_scheduler=lr_scheduler,
                           fault=faults)

    trainer.train()

if __name__ == '__main__':
    args = argparse.ArgumentParser(description='PyTorch Template')
    args.add_argument('-c', '--config', default=None, type=str,
                      help='config file path (default: None)')
    args.add_argument('-r', '--resume', default=None, type=str,
                      help='path to latest checkpoint (default: None)')
    args.add_argument('-d', '--device', default=None, type=str,
                      help='indices of GPUs to enable (default: all)')

    # custom cli options to modify configuration from default values given in json file.
    CustomArgs = collections.namedtuple('CustomArgs', 'flags type target')
    options = [
        CustomArgs(['--lr', '--learning_rate'], type=float, target='optimizer;args;lr'),
        CustomArgs(['--bs', '--batch_size'], type=int, target='data_loader;args;batch_size'),
        CustomArgs(['--l', '--layer'], type=str, target='fault;layer'),
        CustomArgs(['--t', '--time'], type=str, target='fault;time'),
        CustomArgs(['--i', '--index'], type=str, target='fault;index')
    ]
    config = ConfigParser.from_args(args, options)
    main(config)


#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import uuid

import mock

from heat.common import exception
from heat.common import template_format
from heat.engine import environment
from heat.engine import resource
from heat.engine import rsrc_defn
from heat.engine import scheduler
from heat.engine import stack as parser
from heat.engine import template
from heat.tests import common
from heat.tests import utils

from ..resources import cloud_dns  # noqa

domain_only_template = '''
{
  "AWSTemplateFormatVersion" : "2010-09-09",
  "Description" : "Dns instance running on Rackspace cloud",
  "Parameters" : {
    "UnittestDomain" : {
      "Description" : "Domain for unit tests",
      "Type" : "String",
      "Default" : 'dnsheatunittest.com'
    },
    "dnsttl" : {
      "Description" : "TTL for the domain",
      "Type" : "Number",
      "MinValue" : '301',
      "Default" : '301'
    },
    "name": {
      "Description" : "The cloud dns instance name",
      "Type": "String",
      "Default": "CloudDNS"
    }
  },
  "Resources" : {
    "domain" : {
      "Type": "Rackspace::Cloud::DNS",
      "Properties" : {
        "name" : "dnsheatunittest.com",
        "emailAddress" : "admin@dnsheatunittest.com",
        "ttl" : 3600,
        "comment" : "Testing Cloud DNS integration with Heat"
      }
    }
  }
}
'''

class FakeDnsInstance(object):
    def __init__(self):
        self.id = 4
        self.resource_id = 4

    def get(self):
        pass

    def delete(self):
        pass

class RackspaceDnsTest(common.HeatTestCase):

    def setUp(self):
        super(RackspaceDnsTest, self).setUp()
        # Test environment may not have pyrax client library installed and if
        # pyrax is not installed resource class would not be registered.
        # So register resource provider class explicitly for unit testing.
        resource._register_class("Rackspace::Cloud::DNS", cloud_dns.CloudDns)
        self.create_domain_only_args = {
            "name": 'dnsheatunittest.com',
            "emailAddress": 'admin@dnsheatunittest.com',
            "ttl": 3600,
            "comment": 'Testing Cloud DNS integration with Heat',
            "records": None
        }
        self.update_domain_only_args = {
            "emailAddress": 'updatedEmail@example.com',
            "ttl": 5555,
            "comment": 'updated comment'
        }

    def _setup_test_cloud_dns_instance(self, name, parsed_t):
        stack_name = '%s_stack' % name
        t = parsed_t
        templ = template.Template(
            t, env=environment.Environment({'name': 'test'}))
        self.stack = parser.Stack(utils.dummy_context(),
                                  stack_name,
                                  templ,
                                  stack_id=str(uuid.uuid4()))

        instance = cloud_dns.CloudDns(
            '%s_name' % name,
            templ.resource_definitions(self.stack)['domain'],
            self.stack)
        return instance

    def _stubout_create(self, instance, fake_dnsinstance, **create_args):
        mock_client = self.m.CreateMockAnything()
        self.m.StubOutWithMock(instance, 'cloud_dns')
        instance.cloud_dns().AndReturn(mock_client)
        self.m.StubOutWithMock(mock_client, "create")
        mock_client.create(**create_args).AndReturn(fake_dnsinstance)
        self.m.ReplayAll()

    def _stubout_update(
            self,
            instance,
            fake_dnsinstance,
            updateRecords=None,
            **update_args):
        mock_client = self.m.CreateMockAnything()
        self.m.StubOutWithMock(instance, 'cloud_dns')
        instance.cloud_dns().AndReturn(mock_client)
        self.m.StubOutWithMock(mock_client, "get")
        mock_domain = self.m.CreateMockAnything()
        mock_client.get(fake_dnsinstance.resource_id).AndReturn(mock_domain)
        self.m.StubOutWithMock(mock_domain, "update")
        mock_domain.update(**update_args).AndReturn(fake_dnsinstance)
        if updateRecords:
            fake_records = list()
            mock_domain.list_records().AndReturn(fake_records)
            mock_domain.add_records([{
                'comment': None,
                'priority': None,
                'type': 'A',
                'name': 'ftp.example.com',
                'data': '192.0.2.8',
                'ttl': 3600}])
        self.m.ReplayAll()

    def _get_create_args_with_comments(self, record):
        record_with_comment = [dict(record[0])]
        record_with_comment[0]["comment"] = None
        create_record_args = dict()
        create_record_args['records'] = record_with_comment
        create_args = dict(
            self.create_domain_only_args.items() + create_record_args.items())
        return create_args

    def test_create_domain_only(self):
        """
        Test domain create only without any records.
        """
        fake_dns_instance = FakeDnsInstance()
        t = template_format.parse(domain_only_template)
        instance = self._setup_test_cloud_dns_instance('dnsinstance_create', t)
        create_args = self.create_domain_only_args
        self._stubout_create(instance, fake_dns_instance, **create_args)
        scheduler.TaskRunner(instance.create)()
        self.assertEqual((instance.CREATE, instance.COMPLETE), instance.state)
        self.m.VerifyAll()

    def test_create_domain_with_a_record(self):
        """
        Test domain create with an A record.  This should not have a
        priority field.
        """
        fake_dns_instance = FakeDnsInstance()
        t = template_format.parse(domain_only_template)
        a_record = [{
            "type": "A",
            "name": "ftp.example.com",
            "data": "192.0.2.8",
            "ttl": 3600
        }]
        t['Resources']['domain']['Properties']['records'] = a_record
        instance = self._setup_test_cloud_dns_instance('dnsinstance_create', t)
        create_args = self._get_create_args_with_comments(a_record)
        self._stubout_create(instance, fake_dns_instance, **create_args)
        scheduler.TaskRunner(instance.create)()
        self.assertEqual((instance.CREATE, instance.COMPLETE), instance.state)
        self.m.VerifyAll()

    def test_create_domain_with_mx_record(self):
        """
        Test domain create with an MX record.  This should have a
        priority field.
        """
        fake_dns_instance = FakeDnsInstance()
        t = template_format.parse(domain_only_template)
        mx_record = [{
            "type": "MX",
            "name": "example.com",
            "data": "mail.example.com",
            "priority": 5,
            "ttl": 3600
        }]
        t['Resources']['domain']['Properties']['records'] = mx_record
        instance = self._setup_test_cloud_dns_instance('dnsinstance_create', t)
        create_args = self._get_create_args_with_comments(mx_record)
        self._stubout_create(instance, fake_dns_instance, **create_args)
        scheduler.TaskRunner(instance.create)()
        self.assertEqual((instance.CREATE, instance.COMPLETE), instance.state)
        self.m.VerifyAll()

    def test_check(self):
        t = template_format.parse(domain_only_template)
        instance = self._setup_test_cloud_dns_instance('dnsinstance_create', t)

        mock_get = mock.Mock()
        instance.cloud_dns = mock.Mock()
        instance.cloud_dns.return_value.get = mock_get
        scheduler.TaskRunner(instance.check)()
        self.assertEqual('CHECK', instance.action)
        self.assertEqual('COMPLETE', instance.status)

        mock_get.side_effect = cloud_dns.NotFound('boom')
        exc = self.assertRaises(exception.ResourceFailure,
                                scheduler.TaskRunner(instance.check))
        self.assertEqual('CHECK', instance.action)
        self.assertEqual('FAILED', instance.status)
        self.assertIn('boom', str(exc))

    def test_update(self, updateRecords=None):
        """
        Helper function for testing domain updates.
        """
        fake_dns_instance = FakeDnsInstance()
        t = template_format.parse(domain_only_template)
        instance = self._setup_test_cloud_dns_instance('dnsinstance_update', t)
        instance.resource_id = 4
        update_args = self.update_domain_only_args
        self._stubout_update(
            instance,
            fake_dns_instance,
            updateRecords,
            **update_args)

        uprops = dict(instance.properties)
        uprops.update({
            'emailAddress': 'updatedEmail@example.com',
            'ttl': 5555,
            'comment': 'updated comment',
        })
        if updateRecords:
            uprops['records'] = updateRecords
        ut = rsrc_defn.ResourceDefinition(instance.name,
                                          instance.type(),
                                          uprops)
        instance.state_set(instance.CREATE, instance.COMPLETE)
        scheduler.TaskRunner(instance.update, ut)()
        self.assertEqual((instance.UPDATE, instance.COMPLETE), instance.state)
        self.m.VerifyAll()

    def test_update_domain_only(self):
        """
        Test domain update without any records.
        """
        self.test_update()

    def test_update_domain_with_a_record(self):
        """
        Test domain update with an A record.
        """
        a_record = [{'type': 'A',
                     'name': 'ftp.example.com',
                     'data': '192.0.2.8',
                     'ttl': 3600}]
        self.test_update(updateRecords=a_record)

    def test_update_record_only(self):
        """Helper function for testing domain updates."""
        fake_dns_instance = FakeDnsInstance()
        t = template_format.parse(domain_only_template)
        instance = self._setup_test_cloud_dns_instance('dnsinstance_update', t)
        instance.resource_id = 4
        update_records = [{'type': 'A',
                           'name': 'ftp.example.com',
                           'data': '192.0.2.8',
                           'ttl': 3600}]

        mock_client = self.m.CreateMockAnything()
        self.m.StubOutWithMock(instance, 'cloud_dns')
        instance.cloud_dns().AndReturn(mock_client)
        self.m.StubOutWithMock(mock_client, "get")
        mock_domain = self.m.CreateMockAnything()
        mock_client.get(fake_dns_instance.resource_id).AndReturn(mock_domain)

        # mock_domain.update shouldn't be called in this scenario, so
        # stub it out but don't record a call to it
        self.m.StubOutWithMock(mock_domain, "update")

        fake_records = list()
        mock_domain.list_records().AndReturn(fake_records)
        mock_domain.add_records([{
            'comment': None,
            'priority': None,
            'type': 'A',
            'name': 'ftp.example.com',
            'data': '192.0.2.8',
            'ttl': 3600}])
        self.m.ReplayAll()

        uprops = dict(instance.properties)
        uprops['records'] = update_records
        ut = rsrc_defn.ResourceDefinition(instance.name,
                                          instance.type(),
                                          uprops)
        instance.state_set(instance.CREATE, instance.COMPLETE)

        scheduler.TaskRunner(instance.update, ut)()
        self.assertEqual((instance.UPDATE, instance.COMPLETE), instance.state)
        self.m.VerifyAll()


from pwz_bench.utility import *

from itertools import chain
from typing import List

__all__ = ['gen_pwz_binary_pygram_ml']

PWZ_BINARY_PYGRAM_ML = """\
open Pytokens
open Pwz_binary

let rec {grammar_rules}
"""

M_BOT = 'm_bottom'

def gen_pwz_binary_pygram_ml(desc: GrammarDescription) -> List[str]:
    prefix = 'pwz_binary_rule_'
    lines = PWZ_BINARY_PYGRAM_ML.format(
        grammar_rules='\n    and '.join(chain(
            (f"{prefix}{token} = {{ m = {M_BOT}; e' = Tok {token_pair_of_token(token)} }}"
             for token in chain(desc.tokens.named,
                                desc.tokens.nameless,
                                map(lambda p: p[0], desc.tokens.typed))),
            map(lambda r: generate_pygram_rule(r, desc, prefix), desc.base_grammar.rules)
        ))
    ).split('\n')
    return lines

def generate_pygram_rule(rule: Rule, desc: GrammarDescription, prefix: str) -> str:
    if len(rule.groups) != 1:
        raise RuntimeError(f"Rule {rule.name} has {len(rule.groups)} groups; expected 1.")
    group = rule.groups[0]
    if len(group.productions) == 1:
        return f"{prefix}{rule.name} = {str_of_production(group.productions[0], desc, prefix, rule.name)}"
    else:
        production_strings = list(map(lambda p: str_of_production(p[1], desc, prefix, rule.name, p[0]),
                                      enumerate(group.productions, start=1)))
        return f"{prefix}{rule.name} = " + build_binary_alt(production_strings)

def build_binary_alt(elements: List[str]) -> str:
    result = f"{{ m = {M_BOT}; e' = Alt (ref (Some {elements[-2]}), ref (Some {elements[-1]})) }}"
    for element in reversed(elements[:-2]):
        result = f"{{ m = {M_BOT}; e' = Alt (ref (Some {element}), ref (Some {result})) }}"
    return result

def str_of_production(production: Production, desc: GrammarDescription, prefix: str, rule_name: str,
                      production_no: int = 0) -> str:
    parts: List[str] = []
    for component in production.components:
        if isinstance(component, Terminal):
            parts.append(prefix + desc.terminal_names[component])
        elif isinstance(component, NonTerminal):
            parts.append(prefix + component.name)
        else:
            raise RuntimeError(f"Invalid component instance in production of class {component.__class__.__name__}.")
    production_name = rule_name
    if production_no > 0:
        production_name += f"-{production_no}"
    if len(parts) == 0:
        return f"{{ m = {M_BOT}; e' = Eps \"{production_name}\" }}"
    elif len(parts) == 1:
        return f"{{ m = {M_BOT}; e' = Red ((fun t -> Pyast.Ast (\"{production_name}\", [t])), {parts[0]}) }}"
    elif len(parts) == 2:
        return f"{{ m = {M_BOT}; e' = Seq (\"{production_name}\", {parts[0]}, {parts[1]}) }}"
    else:
        return build_binary_seq(production_name, parts)

def build_binary_seq(label: str, elements: List[str]) -> str:
    result = elements[-1]
    for i, element in reversed(list(enumerate(elements[:-1], start=1))):
        result = f"{{ m = {M_BOT}; e' = Seq (\"BINRED-{i}-{label}\", {element}, {result}) }}"
    result = f"{{ m = {M_BOT}; e' = Red (Pyast.flatten_binary_seqs {len(elements) - 1} \"{label}\", {result}) }}"
    return result


import sympy as sp

points = [(0, 0), (2000, 400), (10000, 1200)]

# Define the variable
x = sp.symbols('x')

# init polynomial
P = 0

# calculate lagrange polynomial
for i, (xi, yi) in enumerate(points):
    li = 1
    for j, (xj, yj) in enumerate(points):
        if i != j:
            li *= (x - xj) / (xi - xj)
    P += yi * li

# simplify and print
P = sp.simplify(P)
print(P)


import mysql.connector

from collections import namedtuple

# TODO: Write document for this lib

class MySQLWrapper:
    """
    Wrapper lib for mysql connection
    """
    conn = None
    cur = None
    conf = None

    def __init__(self, **kwargs):
        """
        Setup database configuration

        :param kwargs:
        """
        self.conf = kwargs

    def init_app(self, current_app):
        """
        Get database configuration from flask app instance and connect to the mysql database

        :param current_app: <object> Flask app instance
        :return:
        """

        self.conf = dict(self.conf or ())

        self.conf['user'] = current_app.config.get('MYSQL_DATABASE_USERNAME')
        self.conf['password'] = current_app.config.get('MYSQL_DATABASE_PASSWORD')
        self.conf['host'] = current_app.config.get('MYSQL_DATABASE_HOST', '127.0.0.1')
        self.conf['database'] = current_app.config.get('MYSQL_DATABASE_NAME')
        self.conf['port'] = current_app.config.get('MYSQL_DATABASE_PORT', '3306')

        self.connect()

    def connect(self):
        """
        Setup connection and database cursor

        :return:
        """

        self.conn = mysql.connector.connect(**self.conf)
        self.cur = self.conn.cursor()

    def end(self):
        """Close the connection"""
        self.cur.close()
        self.conn.close()

    def _select(self, table=None, fields=(), where=None, order=None, limit=None):
        """
        Form the sql query for execution

        :param table: <string> table name
        :param fields: <list> list of selecting fields e.g. ("field1", "field2")
        :param where: <list> query condition e.g. [ "id > 1 and id < %s", 5 ]
        :param order: <list> returning order [ "id ASC", "name DESC" ]
        :param limit: <limit> number of returning items and offset of the query [5, 10]
        :return: <object> Mysql cursor object
        """
        query = "SELECT %s FROM `%s`" % (",".join(fields), table)

        # handle where query
        if where and len(where) > 0:
            query += " WHERE %s" % where[0]

        # handle order query
        if order and len(order) > 0:
            query += " ORDER BY %s" % ", ".join(order)

        # Handle limit query
        if limit and len(limit) > 0:
            query += " LIMIT %s" % limit[0]

            if len(limit) > 1:
                query += ", %s" % limit[1]

        return self.query(query, where[1] if where and len(where) > 1 else None)

    def query(self, sql, params=None, multi=False):
        """
        Run raw query

        :param sql: <string> raw sql query
        :param params: <tuple> parameters
        :param multi: <boolean> indicator for running multiple queries or not
        :return: <object> Mysql cursor object
        """

        if not multi:
            self.cur.execute(sql, params, multi=multi)
        else:
            for result in self.conn.cmd_query_iter(sql):
                continue

        return self.cur

    def get_one(self, table=None, fields='*', where=None, order=None, limit=(0, 1)):
        """
        Get single result

        :param table: <string> table name
        :param fields: <list> list of selecting fields e.g. ("field1", "field2")
        :param where: <list> query condition e.g. [ "id > 1 and id < %s", 5 ]
        :param order: <list> returning order [ "id ASC", "name DESC" ]
        :param limit: <limit> number of returning items and offset of the query [5, 10]
        :return: <namedtuple> Row object-like with fields are query fields
        """

        cur = self._select(table, fields, where, order, limit)
        result = cur.fetchone()

        row = None
        if result:
            # Create named tuple by using the query fields
            Row = namedtuple("Row", [d[0] for d in cur.description])
            row = Row(*result)

        return row

    def get_all(self, table=None, fields='*', where=None, order=None, limit=None):
        """
        Get all results

        :param table: <string> table name
        :param fields: <list> list of selecting fields e.g. ("field1", "field2")
        :param where: <list> query condition e.g. [ "id > 1 and id < %s", 5 ]
        :param order: <list> returning order [ "id ASC", "name DESC" ]
        :param limit: <limit> number of returning items and offset of the query [5, 10]
        :return: <list> A list of Row object-like with fields are query fields
        """

        cur = self._select(table, fields, where, order, limit)
        result = cur.fetchall()

        rows = None
        if result:
            # Create named tuple by using the query fields
            Row = namedtuple("Row", [d[0] for d in cur.description])
            rows = [Row(*r) for r in result]

        return rows

    def commit(self):
        """Commit a transaction"""
        return self.conn.commit()


"""Synchronous client handler for Scryfall cards APIs."""

from collections.abc import Iterable
from typing import TYPE_CHECKING, cast, overload
from uuid import UUID

from aioscryfall.models.cards import ScryCard

from .base import BaseSyncHandler

if TYPE_CHECKING:
    from aioscryfall.api.cards import CardIdentifier, SortDirection, SortOrdering, UniqueMode

class CardsSyncHandler(BaseSyncHandler):
    """ScryfallSyncClient handler for cards APIs."""

    def search(
        self,
        query: str,
        *,
        unique: "UniqueMode | None" = None,
        order: "SortOrdering | None" = None,
        direction: "SortDirection | None" = None,
        include_extras: bool | None = None,
        include_multilingual: bool | None = None,
        include_variations: bool | None = None,
    ) -> Iterable[ScryCard]:
        """Search for cards."""
        return self._iterable_extract(
            lambda c: c.cards.search(
                query,
                unique=unique,
                order=order,
                direction=direction,
                include_extras=include_extras,
                include_multilingual=include_multilingual,
                include_variations=include_variations,
            )
        )

    @overload
    def named(self, *, exact: str, set_code: str | None = None) -> ScryCard:
        ...

    @overload
    def named(self, *, fuzzy: str, set_code: str | None = None) -> ScryCard:
        ...

    def named(
        self,
        *,
        exact: str | None = None,
        fuzzy: str | None = None,
        set_code: str | None = None,
    ) -> ScryCard:
        """Get a card by name and, optionally, set."""
        invalid_args_msg = "Exactly one of exact, fuzzy must be specified."
        if exact is None and fuzzy is None:
            raise ValueError(invalid_args_msg)
        if exact is not None:
            return self._result_extract(
                # cast is necessary because of https://github.com/python/mypy/issues/2608
                lambda c: c.cards.named(exact=cast(str, exact), set_code=set_code)
            )
        if fuzzy is not None:
            return self._result_extract(
                # cast is necessary because of https://github.com/python/mypy/issues/2608
                lambda c: c.cards.named(fuzzy=cast(str, fuzzy), set_code=set_code)
            )
        raise ValueError(invalid_args_msg)

    def autocomplete(self, query: str, *, include_extras: bool | None = None) -> list[str]:
        """Get autocomplete suggestions for a query."""
        return self._result_extract(
            lambda c: c.cards.autocomplete(query, include_extras=include_extras)
        )

    def random(self) -> ScryCard:
        """Get a random card."""
        return self._result_extract(lambda c: c.cards.random())

    def get_collection(self, identifiers: list["CardIdentifier"]) -> Iterable[ScryCard]:
        """Get a collection of cards by ID."""
        return self._iterable_extract(lambda c: c.cards.get_collection(identifiers))

    @overload
    def get_card(self, *, set_code: str, collector_number: str) -> ScryCard:
        ...

    @overload
    def get_card(self, *, multiverse_id: int) -> ScryCard:
        ...

    @overload
    def get_card(self, *, mtgo_id: int) -> ScryCard:
        ...

    @overload
    def get_card(self, *, arena_id: int) -> ScryCard:
        ...

    @overload
    def get_card(self, *, tcgplayer_id: int) -> ScryCard:
        ...

    @overload
    def get_card(self, *, cardmarket_id: int) -> ScryCard:
        ...

    @overload
    def get_card(self, *, scryfall_id: UUID) -> ScryCard:
        ...

    def get_card(
        self,
        *,
        set_code: str | None = None,
        collector_number: str | None = None,
        multiverse_id: int | None = None,
        mtgo_id: int | None = None,
        arena_id: int | None = None,
        tcgplayer_id: int | None = None,
        cardmarket_id: int | None = None,
        scryfall_id: UUID | None = None,
    ) -> ScryCard:
        """Get a single card."""
        has_identifier = (
            set_code is not None and collector_number is not None,
            multiverse_id is not None,
            mtgo_id is not None,
            arena_id is not None,
            tcgplayer_id is not None,
            cardmarket_id is not None,
            scryfall_id is not None,
        )
        invalid_args_msg = "Exactly one of (set_code and collector_number), multiverse_id, mtgo_id, arena_id, tcgplayer_id, cardmarket_id, scryfall_id must be specified."
        if len([x for x in has_identifier if x]) != 1:
            raise ValueError(invalid_args_msg)
        if set_code is not None and collector_number is not None:
            return self._result_extract(
                lambda c: c.cards.get_card(
                    set_code=cast(str, set_code), collector_number=cast(str, collector_number)
                )
            )
        if multiverse_id is not None:
            return self._result_extract(
                lambda c: c.cards.get_card(multiverse_id=cast(int, multiverse_id))
            )
        if mtgo_id is not None:
            return self._result_extract(lambda c: c.cards.get_card(mtgo_id=cast(int, mtgo_id)))
        if arena_id is not None:
            return self._result_extract(lambda c: c.cards.get_card(arena_id=cast(int, arena_id)))
        if tcgplayer_id is not None:
            return self._result_extract(
                lambda c: c.cards.get_card(tcgplayer_id=cast(int, tcgplayer_id))
            )
        if cardmarket_id is not None:
            return self._result_extract(
                lambda c: c.cards.get_card(cardmarket_id=cast(int, cardmarket_id))
            )
        if scryfall_id is not None:
            return self._result_extract(
                lambda c: c.cards.get_card(scryfall_id=cast(UUID, scryfall_id))
            )
        raise ValueError(invalid_args_msg)


import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

class LinearRegression(tf.keras.Model):
    def __init__(self):
        super(LinearRegression, self).__init__()
        self.W = tf.keras.layers.Dense(1, use_bias=True, kernel_initializer='random_uniform')

    def call(self, X):
        return self.W(X)

def train_step(X, Y):
    with tf.GradientTape() as tape:
        Y_hat = model(X)
        loss = loss_fn(Y, Y_hat)
    gradients = tape.gradient(loss, model.trainable_variables)
    optimizer.apply_gradients(zip(gradients, model.trainable_variables))
    train_loss(loss)
    train_accuracy(X, Y)

if __name__ == '__main__':
    # generate random dataset
    num_points = 1000
    points = []
    W, b = 0.1, 0.3
    for i in range(num_points):
        x = np.random.normal(0.0, 0.55)
        y = x * W + b + np.random.normal(0.0, 0.03)
        points.append([x, y])

    # generate x, y dataset
    x_data = [p[0] for p in points]
    y_data = [p[1] for p in points]

    x_data = tf.expand_dims(tf.convert_to_tensor(x_data), axis=1)
    y_data = tf.convert_to_tensor(y_data)

    # plot this line
    plt.scatter(x_data, y_data, c='r')
    plt.show()
    num_epoch = 50

    # define model
    model = LinearRegression()
    optimizer = tf.keras.optimizers.SGD(learning_rate=0.3)
    loss_fn = tf.keras.losses.MeanSquaredError()

    # define metrics
    train_loss = tf.keras.metrics.Mean(name='train_loss')
    train_accuracy = tf.keras.metrics.MeanSquaredError(name='train_accuracy')

    for epoch in range(num_epoch):
        train_step(x_data, y_data)

    print("Input Weight:")
    print("W = %.4f, b = %.4f" % (W, b))

    print("Final Weight:")
    W = model.layers[0].get_weights()[0]
    b = model.layers[0].get_weights()[1]
    print("W = %.4f, b = %.4f" % (W, b))

    # plot the fitted line
    plt.scatter(x_data, y_data, c='r')
    plt.plot(x_data, W * x_data + b, c='b')
    plt.show()


from typing import Union

from django.contrib.auth.models import User
from django.utils import timezone

from bookmarks.models import Bookmark, parse_tag_string
from bookmarks.services.tags import get_or_create_tags
from bookmarks.services import website_loader
from bookmarks.services import tasks

def create_bookmark(bookmark: Bookmark, tag_string: str, current_user: User):
    # If URL is already bookmarked, then update it
    existing_bookmark: Bookmark = Bookmark.objects.filter(owner=current_user, url=bookmark.url).first()

    if existing_bookmark is not None:
        _merge_bookmark_data(bookmark, existing_bookmark)
        return update_bookmark(existing_bookmark, tag_string, current_user)

    # Update website info
    _update_website_metadata(bookmark)
    # Set currently logged in user as owner
    bookmark.owner = current_user
    # Set dates
    bookmark.date_added = timezone.now()
    bookmark.date_modified = timezone.now()
    bookmark.save()
    # Update tag list
    _update_bookmark_tags(bookmark, tag_string, current_user)
    bookmark.save()
    # Create snapshot on web archive
    tasks.create_web_archive_snapshot(current_user, bookmark, False)
    # Load favicon
    tasks.load_favicon(current_user, bookmark)

    return bookmark

def update_bookmark(bookmark: Bookmark, tag_string, current_user: User):
    # Detect URL change
    original_bookmark = Bookmark.objects.get(id=bookmark.id)
    has_url_changed = original_bookmark.url != bookmark.url
    # Update tag list
    _update_bookmark_tags(bookmark, tag_string, current_user)
    # Update dates
    bookmark.date_modified = timezone.now()
    bookmark.save()
    # Update favicon
    tasks.load_favicon(current_user, bookmark)

    if has_url_changed:
        # Update web archive snapshot, if URL changed
        tasks.create_web_archive_snapshot(current_user, bookmark, True)
        # Only update website metadata if URL changed
        _update_website_metadata(bookmark)
        bookmark.save()

    return bookmark

def archive_bookmark(bookmark: Bookmark):
    bookmark.is_archived = True
    bookmark.date_modified = timezone.now()
    bookmark.save()
    return bookmark

def archive_bookmarks(bookmark_ids: [Union[int, str]], current_user: User):
    sanitized_bookmark_ids = _sanitize_id_list(bookmark_ids)
    bookmarks = Bookmark.objects.filter(owner=current_user, id__in=sanitized_bookmark_ids)

    bookmarks.update(is_archived=True, date_modified=timezone.now())

def unarchive_bookmark(bookmark: Bookmark):
    bookmark.is_archived = False
    bookmark.date_modified = timezone.now()
    bookmark.save()
    return bookmark

def unarchive_bookmarks(bookmark_ids: [Union[int, str]], current_user: User):
    sanitized_bookmark_ids = _sanitize_id_list(bookmark_ids)
    bookmarks = Bookmark.objects.filter(owner=current_user, id__in=sanitized_bookmark_ids)

    bookmarks.update(is_archived=False, date_modified=timezone.now())

def delete_bookmarks(bookmark_ids: [Union[int, str]], current_user: User):
    sanitized_bookmark_ids = _sanitize_id_list(bookmark_ids)
    bookmarks = Bookmark.objects.filter(owner=current_user, id__in=sanitized_bookmark_ids)

    bookmarks.delete()

def tag_bookmarks(bookmark_ids: [Union[int, str]], tag_string: str, current_user: User):
    sanitized_bookmark_ids = _sanitize_id_list(bookmark_ids)
    bookmarks = Bookmark.objects.filter(owner=current_user, id__in=sanitized_bookmark_ids)
    tag_names = parse_tag_string(tag_string)
    tags = get_or_create_tags(tag_names, current_user)

    for bookmark in bookmarks:
        bookmark.tags.add(*tags)
        bookmark.date_modified = timezone.now()

    Bookmark.objects.bulk_update(bookmarks, ['date_modified'])

def untag_bookmarks(bookmark_ids: [Union[int, str]], tag_string: str, current_user: User):
    sanitized_bookmark_ids = _sanitize_id_list(bookmark_ids)
    bookmarks = Bookmark.objects.filter(owner=current_user, id__in=sanitized_bookmark_ids)
    tag_names = parse_tag_string(tag_string)
    tags = get_or_create_tags(tag_names, current_user)

    for bookmark in bookmarks:
        bookmark.tags.remove(*tags)
        bookmark.date_modified = timezone.now()

    Bookmark.objects.bulk_update(bookmarks, ['date_modified'])

def mark_bookmarks_as_read(bookmark_ids: [Union[int, str]], current_user: User):
    sanitized_bookmark_ids = _sanitize_id_list(bookmark_ids)
    bookmarks = Bookmark.objects.filter(owner=current_user, id__in=sanitized_bookmark_ids)

    bookmarks.update(unread=False, date_modified=timezone.now())

def mark_bookmarks_as_unread(bookmark_ids: [Union[int, str]], current_user: User):
    sanitized_bookmark_ids = _sanitize_id_list(bookmark_ids)
    bookmarks = Bookmark.objects.filter(owner=current_user, id__in=sanitized_bookmark_ids)

    bookmarks.update(unread=True, date_modified=timezone.now())

def share_bookmarks(bookmark_ids: [Union[int, str]], current_user: User):
    sanitized_bookmark_ids = _sanitize_id_list(bookmark_ids)
    bookmarks = Bookmark.objects.filter(owner=current_user, id__in=sanitized_bookmark_ids)

    bookmarks.update(shared=True, date_modified=timezone.now())

def unshare_bookmarks(bookmark_ids: [Union[int, str]], current_user: User):
    sanitized_bookmark_ids = _sanitize_id_list(bookmark_ids)
    bookmarks = Bookmark.objects.filter(owner=current_user, id__in=sanitized_bookmark_ids)

    bookmarks.update(shared=False, date_modified=timezone.now())

def _merge_bookmark_data(from_bookmark: Bookmark, to_bookmark: Bookmark):
    to_bookmark.title = from_bookmark.title
    to_bookmark.description = from_bookmark.description
    to_bookmark.notes = from_bookmark.notes
    to_bookmark.unread = from_bookmark.unread
    to_bookmark.shared = from_bookmark.shared

def _update_website_metadata(bookmark: Bookmark):
    metadata = website_loader.load_website_metadata(bookmark.url)
    bookmark.website_title = metadata.title
    bookmark.website_description = metadata.description

def _update_bookmark_tags(bookmark: Bookmark, tag_string: str, user: User):
    tag_names = parse_tag_string(tag_string)
    tags = get_or_create_tags(tag_names, user)
    bookmark.tags.set(tags)

def _sanitize_id_list(bookmark_ids: [Union[int, str]]) -> [int]:
    # Convert string ids to int if necessary
    return [int(bm_id) if isinstance(bm_id, str) else bm_id for bm_id in bookmark_ids]


import os
import requests
from flask import Blueprint, make_response
from transport_service.logging import mainLogger

def _checkValhalla():
    url = os.environ['VALHALLA_URL']
    r = requests.get(url + '/status')
    mainLogger.debug("_checkValhalla(): Connected to %s", url)
    if len(r.json().keys()) > 0:
        raise Exception(r.json())

bp = Blueprint('misc', __name__)

@bp.route("/health", methods=['GET'])
def health():
    """**Flask GET rule**

    Perform basic health checks.
    ---
    get:
        summary: Get health status.
        tags:
            - Misc
        responses:
            200:
                description: An object with status information.
                content:
                    application/json:
                        schema:
                            type: object
                            properties:
                                status:
                                    type: string
                                    enum:
                                        - OK
                                        - FAILED
                                    description: A status of 'OK' or 'FAILED'.
                                details:
                                    type: object
                                    description: The reason of failure for each component, or 'OK' if not failed.
                                    properties:
                                        valhalla:
                                            type: string
                                            example: OK
    """
    mainLogger.info('Performing health checks...')
    msg = {'valhalla': 'OK'}
    status = True

    try:
        _checkValhalla()
    except Exception as e:
        msg['valhalla'] = str(e)
        status = False

    return make_response({'status': 'OK' if status else 'FAILED', 'details': msg}, 200)


# ## Import Libraries
# Loading all libraries to be used
import copy
import numpy as np
import re
import nltk
import numpy as np
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow.keras import layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.utils import pad_sequences
from keras.callbacks import ModelCheckpoint
# fix random seed for reproducibility
np.random.seed(7)
# from prettytable import PrettyTable
import warnings
warnings.filterwarnings('ignore')
nltk.download('stopwords')

candidate_Labels={"Politics":["Donald Trump","Politics & Policy","Hillary Clinton","Congress","Politics","Joe Biden","Bernie Sanders","Ted Cruz"],
                  "Business":["Business & Finance"],
                  "Opinion":["World","Debates"],
                  "Tech":["Technology","On Instagram","Television","Telecoms"  ],
                  "Science":["Space","Science & Health","Neuroscience","Climate Change"],
                  "Health":["Health Care","Infectious Disease"],
                  "Sports":["Sports","2016 Rio Olympics"],
                  "Entertainment":["Movies" ,"Comic Books","Music","Energy & Environment","Game of Thrones","Episode of the Week","Star Wars","Avengers: Age of Ultron","Game of Thrones, season 6, episode 10","Mad Men, season 7, episode 8","Mad Men, season 7, episode 11"],
                  "Education":["Books","Education"],
                  "Fashion":["Culture","Religion" ],
                  "Food":[],
                  "Travel":["Transportation"],
                  "Real Estate":["Small Business"],
                  "Legal":["Supreme Court ","Policy","LGBTQ","Marriage Equality"],
                  "Crime":["Criminal Justice","Gun Violence","Gender-Based Violence","Hate Crimes"]}

def readData(file):
    titles = []
    categories = []
    links=[]
    with open(file,'r') as tsv:
        count = 0;
        for line in tsv:
            a = line.strip().split('\t')[:6]
            # if a[2] in ['Business & Finance', 'Health Care', 'Science & Health', 'Politics & Policy', 'Criminal Justice']:
            title = a[0].lower()
            title = re.sub('\s\W', ' ', title)
            title = re.sub('\W\s', ' ', title)
            for k, v in candidate_Labels.items():
                if a[2] in v:
                    categories.append(k)
                    titles.append(title)
                    links.append(a[5])
    return titles[1:],categories[1:],links[1:]

def buildDictionary(textlist,minrepition=10):
    wordslist=[text.lower().split() for text in textlist]
    uniquewordscount=list(map(lambda x:x[0], filter(lambda x:x[1]>minrepition,zip(*np.unique(sum(wordslist,[]),return_counts=True)))))
    wordtonumber={w:i+2 for i,w in enumerate(uniquewordscount)}
    wordslist=[[wordtonumber.get(word,1) for word in words] for words in wordslist]
    return wordslist,wordtonumber

# Dataset: https://data.world/elenadata/vox-articles
titles,categories,links=readData('../Files/dsjVoxArticles.tsv')
wordslist,wordtonumber=buildDictionary(titles)
nb_classes=np.unique(categories).shape[0]
categoriestonumber={cat:i for i,cat in enumerate(np.unique(categories))}
categoriesonehot=list(map(lambda x:tf.one_hot(categoriestonumber[x], nb_classes),categories))
n_uniquewords=len(wordtonumber)
embedding_dim=64

print(len(wordslist),len(categoriesonehot))
X_train,X_test,y_train,y_test = train_test_split(wordslist,categoriesonehot,test_size = 0.2)
print("Shape of train data:", len(X_train))
print("Shape of Test data:", len(X_test))

max_title_length = 25
X_train = pad_sequences(X_train, maxlen=max_title_length)
X_test = pad_sequences(X_test, maxlen=max_title_length)

batchsize=64
train_ds=tf.data.Dataset.from_tensor_slices((X_train,y_train))
test_ds=tf.data.Dataset.from_tensor_slices((X_test,y_test))
train_ds = train_ds.batch(batchsize).cache().prefetch(buffer_size=tf.data.AUTOTUNE)
test_ds = test_ds.batch(batchsize).cache().prefetch(buffer_size=tf.data.AUTOTUNE)

model = tf.keras.Sequential([
  layers.Embedding(n_uniquewords, embedding_dim,input_length=max_title_length),
  layers.LSTM(1024,return_sequences=True),
  layers.Dropout(0.2),
  layers.LSTM(512,return_sequences=True),
  layers.Dropout(0.2),
  layers.LSTM(256,return_sequences=True),
  layers.Dropout(0.2),
  layers.LSTM(128,return_sequences=True),
  layers.Dropout(0.2),
  layers.GlobalAveragePooling1D(),
  layers.Dropout(0.2),
  layers.Dense(256,activation="relu"),
  layers.Dropout(0.2),
  layers.Dense(128,activation="relu"),
  layers.Dropout(0.2),
  layers.Dense(64,activation="relu"),
  layers.Dropout(0.2),
  layers.Dense(32, activation="relu"),
  layers.Dropout(0.2),
  layers.Dense(nb_classes,activation="softmax")
])

model.summary()

model.compile(loss=tf.keras.losses.CategoricalCrossentropy(),
              optimizer='adam',
              metrics=tf.keras.metrics.Accuracy())

epochs = 50
history = model.fit(
    train_ds,
    validation_data=test_ds,
    epochs=epochs)

model.save("Model.h5")
print(model.predict(X_train[:1]))


from sqlalchemy.sql.functions import user
from app.tests.utils.users import UserGenerator, create_users, get_auth_user_header
from app.tests.utils.algos import AlgoGenerator, create_algos_lite
from app.tests.utils.backtests import create_backtest_test
from app.tests.utils.quotes import get_intervals
from app.utils.time import unix_to_utc_datetime
from app.tests.utils.shared import IntegrationClear

# ADJUST THESE FOR STRESS GENERATION
NUM_USERS = 100
NUM_ALGOS_PER_USER = 100
BACKTESTS_PER_ALGO = 1 # there will be 10,000 algorithms with 100x100

def generate_backtests(
    min_, max_,
    user_to_algo,
):

    for user_id in user_to_algo:
        algos = user_to_algo[user_id]['algos']
        user_info = user_to_algo[user_id]['user_obj']
        create_backtest_test(algos['id'], user_info, min_, max_)

def generate_algos(users):
    mock_algos = AlgoGenerator().generate_algos(NUM_ALGOS_PER_USER)
    user_to_algo = {}
    for user in users:
        algos = create_algos_lite(mock_algos, user)
        user_to_algo[user['id']] = {
            'user_obj': user,
            'algos': algos,
        }
    return user_to_algo

def generate_users():
    mock_users = UserGenerator().generate_users(NUM_USERS)
    users = create_users(mock_users)
    return (users, mock_users)

def generate_stress_data():

    # CLEAR TABLES FIRST
    IntegrationClear.clear_all_tables()

    # GENERATE USERS
    users, mock_users = generate_users()

    # GENERATE ALGORITHMS
    user_to_algo = generate_algos(mock_users)

    min_, max_ = get_intervals(mock_users[0])
    min_, max_ = unix_to_utc_datetime(min_), unix_to_utc_datetime(max_)

    # GENERATE BACKTESTS WITH MAX INTERVALS
    generate_backtests(min_, max_, user_to_algo)

    # GENERATE COMPETITIONS

class TestStress:

    @classmethod
    def setup_class(cls):
        generate_stress_data()

    def test_placeholder(self):
        pass