from collections import namedtuple
from enum import Enum

from dagster import check
from dagster.core.definitions import Materialization, SolidHandle
from dagster.core.definitions.events import EventMetadataEntry
from dagster.core.serdes import whitelist_for_serdes
from dagster.core.types.runtime_type import RuntimeType
from dagster.utils import merge_dicts
from dagster.utils.error import SerializableErrorInfo

@whitelist_for_serdes
class StepOutputHandle(namedtuple('_StepOutputHandle', 'step_key output_name')):
    @staticmethod
    def from_step(step, output_name='result'):
        check.inst_param(step, 'step', ExecutionStep)

        return StepOutputHandle(step.key, output_name)

    def __new__(cls, step_key, output_name='result'):
        return super(StepOutputHandle, cls).__new__(
            cls,
            step_key=check.str_param(step_key, 'step_key'),
            output_name=check.str_param(output_name, 'output_name'),
        )

@whitelist_for_serdes
class StepInputData(namedtuple('_StepInputData', 'input_name type_check_data')):
    def __new__(cls, input_name, type_check_data):
        return super(StepInputData, cls).__new__(
            cls,
            input_name=check.str_param(input_name, 'input_name'),
            type_check_data=check.opt_inst_param(type_check_data, 'type_check_data', TypeCheckData),
        )

@whitelist_for_serdes
class TypeCheckData(namedtuple('_TypeCheckData', 'success label description metadata_entries')):
    def __new__(cls, success, label, description=None, metadata_entries=None):
        return super(TypeCheckData, cls).__new__(
            cls,
            success=check.bool_param(success, 'success'),
            label=check.str_param(label, 'label'),
            description=check.opt_str_param(description, 'description'),
            metadata_entries=check.opt_list_param(
                metadata_entries, metadata_entries, of_type=EventMetadataEntry
            ),
        )

@whitelist_for_serdes
class UserFailureData(namedtuple('_UserFailureData', 'label description metadata_entries')):
    def __new__(cls, label, description=None, metadata_entries=None):
        return super(UserFailureData, cls).__new__(
            cls,
            label=check.str_param(label, 'label'),
            description=check.opt_str_param(description, 'description'),
            metadata_entries=check.opt_list_param(
                metadata_entries, metadata_entries, of_type=EventMetadataEntry
            ),
        )

@whitelist_for_serdes
class StepOutputData(
    namedtuple('_StepOutputData', 'step_output_handle intermediate_materialization type_check_data')
):
    def __new__(cls, step_output_handle, intermediate_materialization=None, type_check_data=None):
        return super(StepOutputData, cls).__new__(
            cls,
            step_output_handle=check.inst_param(
                step_output_handle, 'step_output_handle', StepOutputHandle
            ),
            intermediate_materialization=check.opt_inst_param(
                intermediate_materialization, 'intermediate_materialization', Materialization
            ),
            type_check_data=check.opt_inst_param(type_check_data, 'type_check_data', TypeCheckData),
        )

    @property
    def output_name(self):
        return self.step_output_handle.output_name

@whitelist_for_serdes
class StepFailureData(namedtuple('_StepFailureData', 'error user_failure_data')):
    def __new__(cls, error, user_failure_data):
        return super(StepFailureData, cls).__new__(
            cls,
            error=check.opt_inst_param(error, 'error', SerializableErrorInfo),
            user_failure_data=check.opt_inst_param(
                user_failure_data, 'user_failure_data', UserFailureData
            ),
        )

@whitelist_for_serdes
class StepSuccessData(namedtuple('_StepSuccessData', 'duration_ms')):
    def __new__(cls, duration_ms):
        return super(StepSuccessData, cls).__new__(
            cls, duration_ms=check.float_param(duration_ms, 'duration_ms')
        )

class StepKind(Enum):
    COMPUTE = 'COMPUTE'

class StepInputSourceType(Enum):
    SINGLE_OUTPUT = 'SINGLE_OUTPUT'
    MULTIPLE_OUTPUTS = 'MULTIPLE_OUTPUTS'
    CONFIG = 'CONFIG'

class StepInput(
    namedtuple('_StepInput', 'name runtime_type source_type source_handles config_data')
):
    def __new__(cls, name, runtime_type, source_type, source_handles=None, config_data=None):
        return super(StepInput, cls).__new__(
            cls,
            name=check.str_param(name, 'name'),
            runtime_type=check.inst_param(runtime_type, 'runtime_type', RuntimeType),
            source_type=check.inst_param(source_type, 'source_type', StepInputSourceType),
            source_handles=check.opt_list_param(
                source_handles, 'source_handles', of_type=StepOutputHandle
            ),
            config_data=config_data,  # can be any type
        )

    @property
    def is_from_output(self):
        return (
            self.source_type == StepInputSourceType.SINGLE_OUTPUT
            or self.source_type == StepInputSourceType.MULTIPLE_OUTPUTS
        )

    @property
    def is_from_single_output(self):
        return self.source_type == StepInputSourceType.SINGLE_OUTPUT

    @property
    def is_from_multiple_outputs(self):
        return self.source_type == StepInputSourceType.MULTIPLE_OUTPUTS

    @property
    def dependency_keys(self):
        return {handle.step_key for handle in self.source_handles}

class StepOutput(namedtuple('_StepOutput', 'name runtime_type optional')):
    def __new__(cls, name, runtime_type, optional):
        return super(StepOutput, cls).__new__(
            cls,
            name=check.str_param(name, 'name'),
            runtime_type=check.inst_param(runtime_type, 'runtime_type', RuntimeType),
            optional=check.bool_param(optional, 'optional'),
        )

class ExecutionStep(
    namedtuple(
        '_ExecutionStep',
        (
            'pipeline_name key_suffix step_inputs step_input_dict step_outputs step_output_dict '
            'compute_fn kind solid_handle logging_tags metadata'
        ),
    )
):
    def __new__(
        cls,
        pipeline_name,
        key_suffix,
        step_inputs,
        step_outputs,
        compute_fn,
        kind,
        solid_handle,
        logging_tags=None,
        metadata=None,
    ):
        return super(ExecutionStep, cls).__new__(
            cls,
            pipeline_name=check.str_param(pipeline_name, 'pipeline_name'),
            key_suffix=check.str_param(key_suffix, 'key_suffix'),
            step_inputs=check.list_param(step_inputs, 'step_inputs', of_type=StepInput),
            step_input_dict={si.name: si for si in step_inputs},
            step_outputs=check.list_param(step_outputs, 'step_outputs', of_type=StepOutput),
            step_output_dict={so.name: so for so in step_outputs},
            compute_fn=check.callable_param(compute_fn, 'compute_fn'),
            kind=check.inst_param(kind, 'kind', StepKind),
            solid_handle=check.inst_param(solid_handle, 'solid_handle', SolidHandle),
            logging_tags=merge_dicts(
                {
                    'step_key': str(solid_handle) + '.' + key_suffix,
                    'pipeline': pipeline_name,
                    'solid': solid_handle.name,
                    'solid_definition': solid_handle.definition_name,
                },
                check.opt_dict_param(logging_tags, 'logging_tags'),
            ),
            metadata=check.opt_dict_param(metadata, 'metadata', key_type=str),
        )

    @property
    def key(self):
        return str(self.solid_handle) + '.' + self.key_suffix

    @property
    def solid_name(self):
        return self.solid_handle.name

    @property
    def solid_definition_name(self):
        return self.solid_handle.definition_name

    def has_step_output(self, name):
        check.str_param(name, 'name')
        return name in self.step_output_dict

    def step_output_named(self, name):
        check.str_param(name, 'name')
        return self.step_output_dict[name]

    def has_step_input(self, name):
        check.str_param(name, 'name')
        return name in self.step_input_dict

    def step_input_named(self, name):
        check.str_param(name, 'name')
        return self.step_input_dict[name]


from typing import Any, NoReturn, List
from numpy import ndarray
import socket
import select
import xml.etree.ElementTree as ElementTree
import logging
import errno

from base_classes import PostProcessorBase

class SocketServerPostProcessor(PostProcessorBase):
    """
    Outputs data to clients as a TCP server.

    Configuration info:

    - `Port`: The port number of the TCP server. This must follow the port usage rules specified by the FRC Game
    Manual. A port number in the range 5800-5810 is recommended.
    """
    port = int()

    sock = None

    async def setup(self, component_config_root: ElementTree.Element):
        self.port = int(component_config_root.find("Port").text)

        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.sock.bind(('0.0.0.0', self.port))
        self.sock.listen(5)
        logging.debug("Server listening on port " + str(self.port))

        self.read_list = [self.sock]
        self.write_list = []

    # noinspection PyMethodMayBeStatic
    async def cleanup(self):
        self.sock.close()

    async def postprocess(self, data: List[Any], frame: ndarray) -> NoReturn:
        def disconnect(s: socket.socket) -> NoReturn:
            if s in self.read_list:
                self.read_list.remove(s)
            if s in self.write_list:
                self.write_list.remove(s)
            if s in readable:
                readable.remove(s)
            if s in writable:
                writable.remove(s)
            s.close()

        try:
            readable, writable, errored = select.select(self.read_list, self.write_list, self.read_list, 0.02)
        except select.error as e:
            print(e)

        for s in errored:
            logging.warn("exceptional condition on " + str(s.getpeername()))
            disconnect(s)

        for s in readable:
            if s is self.sock:
                client_socket, address = self.sock.accept()
                logging.debug("Accepted connection from " + str(client_socket.getpeername()))
                client_socket.setblocking(0)
                self.read_list.append(client_socket)
                self.write_list.append(client_socket)
            else:
                data_in = s.recv(1024)
                if data_in:
                    if data_in.startswith(b"shutdown"):
                        raise KeyboardInterrupt

        for s in writable:
            if len(data) > 0:
                message = self.to_string(data[0])
                try:
                    s.send(bytes(message, "utf-8"))
                except socket.error as err:
                    if err.errno == errno.EPIPE:
                        logging.warn("client unexpectedly disconnected")
                        disconnect(s)

    @staticmethod
    def to_string(data: Any):
        return str(data.angle) + "\n"


import copy
import json
import logging
import os
import pickle
import warnings

import numpy as np
from typing import Any, List, Optional, Text, Dict, Tuple

import rasa.utils.io
from rasa.core.domain import Domain
from rasa.core.featurizers import (
    TrackerFeaturizer,
    FullDialogueTrackerFeaturizer,
    LabelTokenizerSingleStateFeaturizer,
    MaxHistoryTrackerFeaturizer,
)
from rasa.core.policies.policy import Policy
from rasa.core.constants import DEFAULT_POLICY_PRIORITY
from rasa.core.trackers import DialogueStateTracker
from rasa.utils import train_utils

import tensorflow as tf

# avoid warning println on contrib import - remove for tf 2
tf.contrib._warning = None
logger = logging.getLogger(__name__)

class EmbeddingPolicy(Policy):
    """Transformer Embedding Dialogue Policy (TEDP)

    Transformer version of the REDP used in our paper https://arxiv.org/abs/1811.11707
    """

    SUPPORTS_ONLINE_TRAINING = True

    # default properties (DOC MARKER - don't remove)
    defaults = {
        # nn architecture
        # a list of hidden layers sizes before user embed layer
        # number of hidden layers is equal to the length of this list
        "hidden_layers_sizes_pre_dial": [],
        # a list of hidden layers sizes before bot embed layer
        # number of hidden layers is equal to the length of this list
        "hidden_layers_sizes_bot": [],
        # number of units in transformer
        "transformer_size": 128,
        # number of transformer layers
        "num_transformer_layers": 1,
        # type of positional encoding in transformer
        "pos_encoding": "timing",  # string 'timing' or 'emb'
        # max sequence length if pos_encoding='emb'
        "max_seq_length": 256,
        # number of attention heads in transformer
        "num_heads": 4,
        # training parameters
        # initial and final batch sizes:
        # batch size will be linearly increased for each epoch
        "batch_size": [8, 32],
        # how to create batches
        "batch_strategy": "balanced",  # string 'sequence' or 'balanced'
        # number of epochs
        "epochs": 1,
        # set random seed to any int to get reproducible results
        "random_seed": None,
        # embedding parameters
        # dimension size of embedding vectors
        "embed_dim": 20,
        # the type of the similarity
        "num_neg": 20,
        # flag if minimize only maximum similarity over incorrect labels
        "similarity_type": "auto",  # string 'auto' or 'cosine' or 'inner'
        # the type of the loss function
        "loss_type": "softmax",  # string 'softmax' or 'margin'
        # how similar the algorithm should try
        # to make embedding vectors for correct labels
        "mu_pos": 0.8,  # should be 0.0 < ... < 1.0 for 'cosine'
        # maximum negative similarity for incorrect labels
        "mu_neg": -0.2,  # should be -1.0 < ... < 1.0 for 'cosine'
        # the number of incorrect labels, the algorithm will minimize
        # their similarity to the user input during training
        "use_max_sim_neg": True,  # flag which loss function to use
        # scale loss inverse proportionally to confidence of correct prediction
        "scale_loss": True,
        # regularization
        # the scale of L2 regularization
        "C2": 0.001,
        # the scale of how important is to minimize the maximum similarity
        # between embeddings of different labels
        "C_emb": 0.8,
        # dropout rate for dial nn
        "droprate_a": 0.1,
        # dropout rate for bot nn
        "droprate_b": 0.0,
        # visualization of accuracy
        # how often calculate validation accuracy
        "evaluate_every_num_epochs": 20,  # small values may hurt performance
        # how many examples to use for hold out validation set
        "evaluate_on_num_examples": 0,  # large values may hurt performance
    }
    # end default properties (DOC MARKER - don't remove)

    @staticmethod
    def _standard_featurizer(max_history: Optional[int] = None) -> "TrackerFeaturizer":
        if max_history is None:
            return FullDialogueTrackerFeaturizer(LabelTokenizerSingleStateFeaturizer())
        else:
            return MaxHistoryTrackerFeaturizer(
                LabelTokenizerSingleStateFeaturizer(), max_history=max_history
            )

    def __init__(
        self,
        featurizer: Optional["TrackerFeaturizer"] = None,
        priority: int = DEFAULT_POLICY_PRIORITY,
        graph: Optional["tf.Graph"] = None,
        session: Optional["tf.Session"] = None,
        user_placeholder: Optional["tf.Tensor"] = None,
        bot_placeholder: Optional["tf.Tensor"] = None,
        similarity_all: Optional["tf.Tensor"] = None,
        pred_confidence: Optional["tf.Tensor"] = None,
        similarity: Optional["tf.Tensor"] = None,
        dial_embed: Optional["tf.Tensor"] = None,
        bot_embed: Optional["tf.Tensor"] = None,
        all_bot_embed: Optional["tf.Tensor"] = None,
        attention_weights: Optional["tf.Tensor"] = None,
        max_history: Optional[int] = None,
        **kwargs: Any,
    ) -> None:
        """Declare instant variables with default values"""

        if not featurizer:
            featurizer = self._standard_featurizer(max_history)
        super().__init__(featurizer, priority)

        self._load_params(**kwargs)

        # encode all label_ids with numbers
        self._encoded_all_label_ids = None

        # tf related instances
        self.graph = graph
        self.session = session
        self.a_in = user_placeholder
        self.b_in = bot_placeholder
        self.sim_all = similarity_all
        self.pred_confidence = pred_confidence
        self.sim = similarity

        # persisted embeddings
        self.dial_embed = dial_embed
        self.bot_embed = bot_embed
        self.all_bot_embed = all_bot_embed

        self.attention_weights = attention_weights
        # internal tf instances
        self._iterator = None
        self._train_op = None
        self._is_training = None

    # init helpers
    def _load_nn_architecture_params(self, config: Dict[Text, Any]) -> None:
        self.hidden_layers_sizes = {
            "pre_dial": config["hidden_layers_sizes_pre_dial"],
            "bot": config["hidden_layers_sizes_bot"],
        }

        self.pos_encoding = config["pos_encoding"]
        self.max_seq_length = config["max_seq_length"]
        self.num_heads = config["num_heads"]

        self.transformer_size = config["transformer_size"]
        self.num_transformer_layers = config["num_transformer_layers"]

        self.batch_size = config["batch_size"]
        self.batch_strategy = config["batch_strategy"]

        self.epochs = config["epochs"]

        self.random_seed = config["random_seed"]

    def _load_embedding_params(self, config: Dict[Text, Any]) -> None:
        self.embed_dim = config["embed_dim"]
        self.num_neg = config["num_neg"]

        self.similarity_type = config["similarity_type"]
        self.loss_type = config["loss_type"]
        if self.similarity_type == "auto":
            if self.loss_type == "softmax":
                self.similarity_type = "inner"
            elif self.loss_type == "margin":
                self.similarity_type = "cosine"

        self.mu_pos = config["mu_pos"]
        self.mu_neg = config["mu_neg"]
        self.use_max_sim_neg = config["use_max_sim_neg"]

        self.scale_loss = config["scale_loss"]

    def _load_regularization_params(self, config: Dict[Text, Any]) -> None:
        self.C2 = config["C2"]
        self.C_emb = config["C_emb"]
        self.droprate = {"bot": config["droprate_b"], "dial": config["droprate_a"]}

    def _load_visual_params(self, config: Dict[Text, Any]) -> None:
        self.evaluate_every_num_epochs = config["evaluate_every_num_epochs"]
        if self.evaluate_every_num_epochs < 1:
            self.evaluate_every_num_epochs = self.epochs
        self.evaluate_on_num_examples = config["evaluate_on_num_examples"]

    def _load_params(self, **kwargs: Dict[Text, Any]) -> None:
        config = copy.deepcopy(self.defaults)
        config.update(kwargs)

        self._tf_config = train_utils.load_tf_config(config)
        self._load_nn_architecture_params(config)
        self._load_embedding_params(config)
        self._load_regularization_params(config)
        self._load_visual_params(config)

    # data helpers
    # noinspection PyPep8Naming
    @staticmethod
    def _label_ids_for_Y(data_Y: "np.ndarray") -> "np.ndarray":
        """Prepare Y data for training: extract label_ids."""

        return data_Y.argmax(axis=-1)

    # noinspection PyPep8Naming
    def _label_features_for_Y(self, label_ids: "np.ndarray") -> "np.ndarray":
        """Prepare Y data for training: features for label_ids."""

        if len(label_ids.shape) == 2:  # full dialogue featurizer is used
            return np.stack(
                [
                    np.stack(
                        [
                            self._encoded_all_label_ids[label_idx]
                            for label_idx in seq_label_ids
                        ]
                    )
                    for seq_label_ids in label_ids
                ]
            )
        else:  # max history featurizer is used
            return np.stack(
                [self._encoded_all_label_ids[label_idx] for label_idx in label_ids]
            )

    # noinspection PyPep8Naming
    def _create_session_data(
        self, data_X: "np.ndarray", data_Y: Optional["np.ndarray"] = None
    ) -> "train_utils.SessionDataType":
        """Combine all tf session related data into dict."""
        if data_Y is not None:
            # training time
            label_ids = self._label_ids_for_Y(data_Y)
            Y = self._label_features_for_Y(label_ids)
            # explicitly add last dimension to label_ids
            # to track correctly dynamic sequences
            label_ids = np.expand_dims(label_ids, -1)
        else:
            # prediction time
            label_ids = None
            Y = None

        return {
            "dialogue_features": [data_X],
            "bot_features": [Y],
            "action_ids": [label_ids],
        }

    def _create_tf_bot_embed(self, b_in: "tf.Tensor") -> "tf.Tensor":
        """Create embedding bot vector."""

        b = train_utils.create_tf_fnn(
            b_in,
            self.hidden_layers_sizes["bot"],
            self.droprate["bot"],
            self.C2,
            self._is_training,
            layer_name_suffix="bot",
        )
        return train_utils.create_tf_embed(
            b, self.embed_dim, self.C2, self.similarity_type, layer_name_suffix="bot"
        )

    def _create_tf_dial(self, a_in) -> Tuple["tf.Tensor", "tf.Tensor"]:
        """Create dialogue level embedding and mask."""

        # mask different length sequences
        # if there is at least one `-1` it should be masked
        mask = tf.sign(tf.reduce_max(self.a_in, -1) + 1)

        a = train_utils.create_tf_fnn(
            a_in,
            self.hidden_layers_sizes["pre_dial"],
            self.droprate["dial"],
            self.C2,
            self._is_training,
            layer_name_suffix="pre_dial",
        )

        self.attention_weights = {}
        hparams = train_utils.create_t2t_hparams(
            self.num_transformer_layers,
            self.transformer_size,
            self.num_heads,
            self.droprate["dial"],
            self.pos_encoding,
            self.max_seq_length,
            self._is_training,
        )

        a = train_utils.create_t2t_transformer_encoder(
            a, mask, self.attention_weights, hparams, self.C2, self._is_training
        )

        if isinstance(self.featurizer, MaxHistoryTrackerFeaturizer):
            # pick last label if max history featurizer is used
            a = a[:, -1:, :]
            mask = mask[:, -1:]

        dial_embed = train_utils.create_tf_embed(
            a, self.embed_dim, self.C2, self.similarity_type, layer_name_suffix="dial"
        )

        return dial_embed, mask

    def _build_tf_train_graph(self) -> Tuple["tf.Tensor", "tf.Tensor"]:
        """Bulid train graph using iterator."""
        # iterator returns a_in, b_in, action_ids
        self.a_in, self.b_in, _ = self._iterator.get_next()

        if isinstance(self.featurizer, MaxHistoryTrackerFeaturizer):
            # add time dimension if max history featurizer is used
            self.b_in = self.b_in[:, tf.newaxis, :]

        all_bot_raw = tf.constant(
            self._encoded_all_label_ids, dtype=tf.float32, name="all_bot_raw"
        )

        self.dial_embed, mask = self._create_tf_dial(self.a_in)

        self.bot_embed = self._create_tf_bot_embed(self.b_in)
        self.all_bot_embed = self._create_tf_bot_embed(all_bot_raw)

        return train_utils.calculate_loss_acc(
            self.dial_embed,
            self.bot_embed,
            self.b_in,
            self.all_bot_embed,
            all_bot_raw,
            self.num_neg,
            mask,
            self.loss_type,
            self.mu_pos,
            self.mu_neg,
            self.use_max_sim_neg,
            self.C_emb,
            self.scale_loss,
        )

    # prepare for prediction
    def _create_tf_placeholders(
        self, session_data: "train_utils.SessionDataType"
    ) -> None:
        """Create placeholders for prediction."""

        dialogue_len = None  # use dynamic time
        self.a_in = tf.placeholder(
            dtype=tf.float32,
            shape=(None, dialogue_len, session_data["dialogue_features"][0].shape[-1]),
            name="a",
        )
        self.b_in = tf.placeholder(
            dtype=tf.float32,
            shape=(None, dialogue_len, None, session_data["bot_features"][0].shape[-1]),
            name="b",
        )

    def _build_tf_pred_graph(
        self, session_data: "train_utils.SessionDataType"
    ) -> "tf.Tensor":
        """Rebuild tf graph for prediction."""

        self._create_tf_placeholders(session_data)

        self.dial_embed, mask = self._create_tf_dial(self.a_in)

        self.sim_all = train_utils.tf_raw_sim(
            self.dial_embed[:, :, tf.newaxis, :],
            self.all_bot_embed[tf.newaxis, tf.newaxis, :, :],
            mask,
        )

        self.bot_embed = self._create_tf_bot_embed(self.b_in)

        self.sim = train_utils.tf_raw_sim(
            self.dial_embed[:, :, tf.newaxis, :], self.bot_embed, mask
        )

        return train_utils.confidence_from_sim(self.sim_all, self.similarity_type)

    # training methods
    def train(
        self,
        training_trackers: List["DialogueStateTracker"],
        domain: "Domain",
        **kwargs: Any,
    ) -> None:
        """Train the policy on given training trackers."""

        logger.debug("Started training embedding policy.")

        # set numpy random seed
        np.random.seed(self.random_seed)

        # dealing with training data
        training_data = self.featurize_for_training(training_trackers, domain, **kwargs)

        # encode all label_ids with policies' featurizer
        state_featurizer = self.featurizer.state_featurizer
        self._encoded_all_label_ids = state_featurizer.create_encoded_all_actions(
            domain
        )

        # check if number of negatives is less than number of label_ids
        logger.debug(
            "Check if num_neg {} is smaller "
            "than number of label_ids {}, "
            "else set num_neg to the number of label_ids - 1"
            "".format(self.num_neg, domain.num_actions)
        )
        # noinspection PyAttributeOutsideInit
        self.num_neg = min(self.num_neg, domain.num_actions - 1)

        # extract actual training data to feed to tf session
        session_data = self._create_session_data(training_data.X, training_data.y)

        if self.evaluate_on_num_examples:
            session_data, eval_session_data = train_utils.train_val_split(
                session_data,
                self.evaluate_on_num_examples,
                self.random_seed,
                label_key="action_ids",
            )
        else:
            eval_session_data = None

        self.graph = tf.Graph()
        with self.graph.as_default():
            # set random seed in tf
            tf.set_random_seed(self.random_seed)

            # allows increasing batch size
            batch_size_in = tf.placeholder(tf.int64)

            (
                self._iterator,
                train_init_op,
                eval_init_op,
            ) = train_utils.create_iterator_init_datasets(
                session_data,
                eval_session_data,
                batch_size_in,
                self.batch_strategy,
                label_key="action_ids",
            )

            self._is_training = tf.placeholder_with_default(False, shape=())

            loss, acc = self._build_tf_train_graph()

            # define which optimizer to use
            self._train_op = tf.train.AdamOptimizer().minimize(loss)

            # train tensorflow graph
            self.session = tf.Session(config=self._tf_config)
            train_utils.train_tf_dataset(
                train_init_op,
                eval_init_op,
                batch_size_in,
                loss,
                acc,
                self._train_op,
                self.session,
                self._is_training,
                self.epochs,
                self.batch_size,
                self.evaluate_on_num_examples,
                self.evaluate_every_num_epochs,
            )

            # rebuild the graph for prediction
            self.pred_confidence = self._build_tf_pred_graph(session_data)

            self.attention_weights = train_utils.extract_attention(
                self.attention_weights
            )

    def continue_training(
        self,
        training_trackers: List["DialogueStateTracker"],
        domain: "Domain",
        **kwargs: Any,
    ) -> None:
        """Continue training an already trained policy."""

        batch_size = kwargs.get("batch_size", 5)
        epochs = kwargs.get("epochs", 50)

        with self.graph.as_default():
            for _ in range(epochs):
                training_data = self._training_data_for_continue_training(
                    batch_size, training_trackers, domain
                )

                session_data = self._create_session_data(
                    training_data.X, training_data.y
                )
                train_dataset = train_utils.create_tf_dataset(
                    session_data, batch_size, label_key="action_ids"
                )
                train_init_op = self._iterator.make_initializer(train_dataset)
                self.session.run(train_init_op)

                # fit to one extra example using updated trackers
                while True:
                    try:
                        self.session.run(
                            self._train_op, feed_dict={self._is_training: True}
                        )

                    except tf.errors.OutOfRangeError:
                        break

    def tf_feed_dict_for_prediction(
        self, tracker: "DialogueStateTracker", domain: "Domain"
    ) -> Dict["tf.Tensor", "np.ndarray"]:
        """Create feed dictionary for tf session."""

        # noinspection PyPep8Naming
        data_X = self.featurizer.create_X([tracker], domain)
        session_data = self._create_session_data(data_X)

        return {self.a_in: session_data["dialogue_features"][0]}

    def predict_action_probabilities(
        self, tracker: "DialogueStateTracker", domain: "Domain"
    ) -> List[float]:
        """Predict the next action the bot should take.

        Return the list of probabilities for the next actions.
        """

        if self.session is None:
            logger.error(
                "There is no trained tf.session: "
                "component is either not trained or "
                "didn't receive enough training data"
            )
            return [0.0] * domain.num_actions

        tf_feed_dict = self.tf_feed_dict_for_prediction(tracker, domain)

        confidence = self.session.run(self.pred_confidence, feed_dict=tf_feed_dict)

        return confidence[0, -1, :].tolist()

    def persist(self, path: Text) -> None:
        """Persists the policy to a storage."""

        if self.session is None:
            warnings.warn(
                "Method `persist(...)` was called "
                "without a trained model present. "
                "Nothing to persist then!"
            )
            return

        self.featurizer.persist(path)

        meta = {"priority": self.priority}

        meta_file = os.path.join(path, "embedding_policy.json")
        rasa.utils.io.dump_obj_as_json_to_file(meta_file, meta)

        file_name = "tensorflow_embedding.ckpt"
        checkpoint = os.path.join(path, file_name)
        rasa.utils.io.create_directory_for_file(checkpoint)

        with self.graph.as_default():
            train_utils.persist_tensor("user_placeholder", self.a_in, self.graph)
            train_utils.persist_tensor("bot_placeholder", self.b_in, self.graph)

            train_utils.persist_tensor("similarity_all", self.sim_all, self.graph)
            train_utils.persist_tensor(
                "pred_confidence", self.pred_confidence, self.graph
            )
            train_utils.persist_tensor("similarity", self.sim, self.graph)

            train_utils.persist_tensor("dial_embed", self.dial_embed, self.graph)
            train_utils.persist_tensor("bot_embed", self.bot_embed, self.graph)
            train_utils.persist_tensor("all_bot_embed", self.all_bot_embed, self.graph)

            train_utils.persist_tensor(
                "attention_weights", self.attention_weights, self.graph
            )

            saver = tf.train.Saver()
            saver.save(self.session, checkpoint)

        with open(os.path.join(path, file_name + ".tf_config.pkl"), "wb") as f:
            pickle.dump(self._tf_config, f)

    @classmethod
    def load(cls, path: Text) -> "EmbeddingPolicy":
        """Loads a policy from the storage.

        **Needs to load its featurizer**
        """

        if not os.path.exists(path):
            raise Exception(
                "Failed to load dialogue model. Path '{}' "
                "doesn't exist".format(os.path.abspath(path))
            )

        featurizer = TrackerFeaturizer.load(path)

        file_name = "tensorflow_embedding.ckpt"
        checkpoint = os.path.join(path, file_name)

        if not os.path.exists(checkpoint + ".meta"):
            return cls(featurizer=featurizer)

        meta_file = os.path.join(path, "embedding_policy.json")
        meta = json.loads(rasa.utils.io.read_file(meta_file))

        with open(os.path.join(path, file_name + ".tf_config.pkl"), "rb") as f:
            _tf_config = pickle.load(f)

        graph = tf.Graph()
        with graph.as_default():
            session = tf.Session(config=_tf_config)
            saver = tf.train.import_meta_graph(checkpoint + ".meta")

            saver.restore(session, checkpoint)

            a_in = train_utils.load_tensor("user_placeholder")
            b_in = train_utils.load_tensor("bot_placeholder")

            sim_all = train_utils.load_tensor("similarity_all")
            pred_confidence = train_utils.load_tensor("pred_confidence")
            sim = train_utils.load_tensor("similarity")

            dial_embed = train_utils.load_tensor("dial_embed")
            bot_embed = train_utils.load_tensor("bot_embed")
            all_bot_embed = train_utils.load_tensor("all_bot_embed")

            attention_weights = train_utils.load_tensor("attention_weights")

        return cls(
            featurizer=featurizer,
            priority=meta["priority"],
            graph=graph,
            session=session,
            user_placeholder=a_in,
            bot_placeholder=b_in,
            similarity_all=sim_all,
            pred_confidence=pred_confidence,
            similarity=sim,
            dial_embed=dial_embed,
            bot_embed=bot_embed,
            all_bot_embed=all_bot_embed,
            attention_weights=attention_weights,
        )


from flask import current_app, g, Flask, flash, jsonify, redirect, render_template, request, session, Response
import logging
import sqlite3
import json
import requests
from db import DB, KeyNotFound, BadRequest
import datetime

# Configure application
app = Flask(__name__)

app.config['JSON_SORT_KEYS'] = False

# Ensure templates are auto-reloaded
app.config["TEMPLATES_AUTO_RELOAD"] = True

# Needed to flash messages
app.secret_key = b'mEw6%7APK'

# path to database
DATABASE = 'splatDB.sqlite3'

# default path
@app.route('/')
def home():
    return render_template("home.html")

# hello world
@app.route('/hello')
def hello_world():
    data = {"message": "Hello, World!"}
    return jsonify(data)

# -----------------
# Create/Read Endpoints
# These JSON/REST api endpoints are used to add new records
# and return lookups based on Ids
# -------------------

# creates required table for application.
# note having a web endpoint for this is not a standard approach, but used for quick testing
@app.route('/create', methods=["GET"])
def create_tables():
    """
    Drops existing tables and creates new tables
    """
    db = DB(get_db_conn())
    return db.create_db('schema/create.sql')

@app.route('/artist', methods=["POST"])
def add_artist():
    """
    Loads a new appearance of song
    (and possibly a new song) into the database.
    """
    post_body = request.json
    if not post_body:
        logging.error("No post body")
        return Response(status=400)

    # get DB class with new connection
    db = DB(get_db_conn())

    try:
        db.add_artist(post_body)
    except BadRequest as e:
        raise InvalidUsage(e.message, status_code=e.error_code)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=201)

@app.route('/album', methods=["POST"])
def add_album():
    """
    Loads a new appearance of song
    (and possibly a new song) into the database.
    """
    post_body = request.json
    if not post_body:
        logging.error("No post body")
        return Response(status=400)

    # get DB class with new connection
    db = DB(get_db_conn())

    try:
        db.add_album(post_body)
    except BadRequest as e:
        raise InvalidUsage(e.message, status_code=e.error_code)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=201)

@app.route('/songs', methods=["POST"])
def add_song():
    """
    Loads a new appearance of song
    (and possibly a new song) into the database.
    """
    post_body = request.json
    if not post_body:
        logging.error("No post body")
        return Response(status=400)

    # get DB class with new connection
    db = DB(get_db_conn())

    try:
        db.add_song_ms2(post_body)
    except BadRequest as e:
        raise InvalidUsage(e.message, status_code=e.error_code)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=201)

@app.route('/playlists', methods=["POST"])
def add_playlist():
    """
    Adds a new playlist, with a list of ordered songs
    """
    post_body = request.json
    if not post_body:
        logging.error("No post body")
        return Response(status=400)

    # get DB class with new connection
    db = DB(get_db_conn())

    try:
        db.add_playlist(post_body)
    except BadRequest as e:
        raise InvalidUsage(e.message, status_code=e.error_code)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=201)

@app.route("/playcount", methods=["POST"])
def add_play():
    """
    Aad a play count detail
    Must have"play_count","song_id","date"
    May have a source as either a "playlist_id" OR "album_id" OR NEITHER (indicates a direct play)
    NOTE: Can have multiple calls for the same song, date, and source (playlist,album,or none)
    """
    post_body = request.json
    if not post_body:
        logging.error("No post body")
        return Response(status=400)

    # get DB class with new connection
    db = DB(get_db_conn())

    try:
        db.add_play(post_body)
    except BadRequest as e:
        raise InvalidUsage(e.message, status_code=e.error_code)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=201)

@app.route('/songs/<song_id>', methods=["GET"])
def find_song(song_id):
    """
    Returns a song's info
    (song_id, name, length, artist name, album name) based on song_id
    """
    # get DB class with new connection
    db = DB(get_db_conn())

    try:
        res = db.find_song(song_id)
        return jsonify(res)
    except KeyNotFound as e:
        print(e)
        raise InvalidUsage(e.message, status_code=404)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=400)

@app.route('/songs/by_album/<album_id>', methods=["GET"])
def find_songs_by_album(album_id):
    """
    Returns all an album's songs
    (song_id, name, length, artist name, album name) based on album_id
    """
    # get DB class with new connection
    db = DB(get_db_conn())

    try:
        res = db.find_songs_by_album(album_id)
        return jsonify(res)
    except KeyNotFound as e:
        print(e)
        raise InvalidUsage(e.message, status_code=404)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=400)

@app.route('/songs/by_artist/<artist_id>', methods=["GET"])
def find_songs_by_artist(artist_id):
    """
    Returns all an artists' songs
    (song_id, name, length, artist name, album name) based on artist_id
    """
    # get DB class with new connection
    db = DB(get_db_conn())

    try:
        res = db.find_songs_by_artist(artist_id)
        return jsonify(res)
    except KeyNotFound as e:
        print(e)
        raise InvalidUsage(e.message, status_code=404)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=400)

@app.route('/albums/<album_id>', methods=["GET"])
def find_album(album_id):
    """
    Returns a album's info
    (album_id, album_name, release_year).
    """
    # get DB class with new connection
    db = DB(get_db_conn())

    try:
        res = db.find_album(album_id)
        return jsonify(res)
    except KeyNotFound as e:
        print(e)
        raise InvalidUsage(e.message, status_code=404)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=400)

@app.route('/albums/by_artist/<artist_id>', methods=["GET"])
def find_album_by_artist(artist_id):
    """
    Returns a album's info
    (album_id, album_name, release_year).
    """
    # get DB class with new connection
    db = DB(get_db_conn())

    try:
        res = db.find_album_by_artist(artist_id)
        return jsonify(res)
    except KeyNotFound as e:
        print(e)
        raise InvalidUsage(e.message, status_code=404)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=400)

@app.route('/artists/<artist_id>', methods=["GET"])
def find_artist(artist_id):
    """
    Returns a artist's info
    (artist_id, artist_name, country).
    """
    # get DB class with new connection
    db = DB(get_db_conn())

    try:
        res = db.find_artist(artist_id)
        return jsonify(res)
    except KeyNotFound as e:
        print(e)
        raise InvalidUsage(e.message, status_code=404)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=400)

# -----------------
# Analytics Endpoints
# These JSON/REST api endpoints are used to run analysis
# over the dataset and calculate an aggregated answer
# -------------------

@app.route('/analytics/artists/avg_song_length/<artist_id>', methods=["GET"])
def avg_song_length(artist_id):
    """
    Returns the average length of an artist's songs (artist_id, avg_length)
    """
    # get DB class with new connection
    db = DB(get_db_conn())

    try:
        res = db.avg_song_length(artist_id)
        return jsonify(res)
    except KeyNotFound as e:
        print(e)
        raise InvalidUsage(e.message, status_code=404)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=400)

@app.route('/analytics/artists/cnt_singles/<artist_id>', methods=["GET"])
def cnt_singles(artist_id):
    """
    Returns the number of singles an artist has (artist_id, cnt_single)
    """
    # get DB class with new connection
    db = DB(get_db_conn())

    try:
        res = db.cnt_singles(artist_id)
        return jsonify(res)
    except KeyNotFound as e:
        print(e)
        raise InvalidUsage(e.message, status_code=404)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=400)

@app.route('/analytics/artists/top_length/<num_artists>', methods=["GET"])
def top_length(num_artists):
    """
    Returns top (n=num_artists) artists based on total length of songs
    (artist_id, total_length).
    """
    # get DB class with new connection
    db = DB(get_db_conn())

    try:
        res = db.top_length(num_artists)
        return jsonify(res)
    except KeyNotFound as e:
        print(e)
        raise InvalidUsage(e.message, status_code=404)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=400)

@app.route('/analytics/solo_albums', methods=["GET"])
def solo_albums():
    """
    Returns an array/list of album_ids where the album
    and all songs are by the same single artist_id
    """
    # get DB class with new connection
    db = DB(get_db_conn())
    try:
        res = db.solo_albums()
        return jsonify(res)
    except BadRequest as e:
        print(e)
        raise InvalidUsage(e.message, status_code=404)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=400)

# Convert a YYYY-MM-DD string to a datetime.date. Raises BadRequest if not in the right format
# or if not a valid date.
# From https://stackoverflow.com/questions/53460391/passing-a-date-as-a-url-parameter-to-a-flask-route
# Better function exists in 3.7+ adding this to support 3.6+
def to_date(date_string):
    try:
        return datetime.datetime.strptime(date_string, "%Y-%m-%d").date()
    except ValueError:
        raise BadRequest('{} is not valid date in the format YYYY-MM-DD'.format(date_string))

@app.route('/analytics/playcount/top_song/<date_string>', methods=["GET"])
def top_song(date_string):
    """
    Get the top song played on a given date
    The test data does not account for ties/ have ties. if you want to break them use song_id ascending.
    """
    # get DB class with new connection
    db = DB(get_db_conn())
    try:
        check_date = to_date(date_string)
        res = db.top_song(check_date)
        return jsonify(res)
    except BadRequest as e:
        raise InvalidUsage(e.message, status_code=400)
    except KeyNotFound as e:
        print(e)
        raise InvalidUsage(e.message, status_code=404)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=400)

@app.route('/analytics/playcount/top_source/<song_id>/<date_string>', methods=["GET"])
def top_source(song_id, date_string):
    """
    For a given song and date, return the source that contributed to the most plays
    This could be a given playlist_id, a given album_id, or None (a direct play)
    The test data does not account for ties.
    If you want to want to account for ties, give all sources that have the same (top) play_count
    """
    # get DB class with new connection
    db = DB(get_db_conn())
    try:
        check_date = to_date(date_string)
        res = db.top_source(song_id, check_date)
        return jsonify(res)
    except BadRequest as e:
        raise InvalidUsage(e.message, status_code=400)
    except KeyNotFound as e:
        print(e)
        raise InvalidUsage(e.message, status_code=404)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=400)

@app.route('/analytics/playcount/top_country/<date_string>', methods=["GET"])
def top_country(date_string):
    """
    This is an extra credit MS that for a given date , it gives the country
    with the most play
    """
    # get DB class with new connection
    db = DB(get_db_conn())
    try:
        check_date = to_date(date_string)
        res = db.top_country(check_date)
        return jsonify(res)
    except BadRequest as e:
        raise InvalidUsage(e.message, status_code=400)
    except KeyNotFound as e:
        print(e)
        raise InvalidUsage(e.message, status_code=404)
    except sqlite3.Error as e:
        print(e)
        raise InvalidUsage(str(e))
    return Response(status=400)

# -----------------
# Web APIs
# These simply wrap requests from the website/browser and
# invoke the underlying REST / JSON API.
# -------------------

# paste in a query
@app.route('/web/query', methods=["GET", "POST"])
def query():
    """
    runs pasted in query
    """

    data = None
    if request.method == "POST":
        qry = request.form.get("query")
        # Ensure query was submitted

        # get DB class with new connection
        db = DB(get_db_conn())

        # note DO NOT EVER DO THIS NORMALLY (run SQL from a client/web directly)
        # https://xkcd.com/327/
        try:
            res = db.run_query(str(qry))
        except sqlite3.Error as e:
            print(e)
            return render_template("error.html", errmsg=str(e), errcode=400)

        data = res
    return render_template("query.html", data=data)

# paste in a query
@app.route('/web/post_data', methods=["GET", "POST"])
def post_song_web():
    """
    runs simple post song json
    """

    data = None
    if request.method == "POST":
        parameter = request.form.get("path")
        if parameter is None or parameter.strip() == "":
            flash("Must set key")
            return render_template("post_data.html", data=data)

        get_url = "http://127.0.0.1:5000/%s" % parameter
        print("Making request to %s" % get_url)
        # grab the response

        j = json.loads(request.form.get("json_data").strip())
        print("Json from form: %s" % j)
        r = requests.post(get_url, json=j)
        if r.status_code >= 400:
            print("Error.  %s  Body: %s" % (r, r.content))
            return render_template("error.html", errmsg=r.json(), errcode=r.status_code)

        else:
            flash("Ran post command")
        return render_template("post_data.html", data=None)
    return render_template("post_data.html", data=None)

@app.route('/web/create', methods=["GET"])
def create_web():
    get_url = "http://127.0.0.1:5000/create"
    print("Making request to %s" % get_url)
    # grab the response
    r = requests.get(get_url)
    if r.status_code >= 400:
        print("Error.  %s  Body: %s" % (r, r.content))
        return render_template("error.html", errmsg=r.json(), errcode=r.status_code)

    else:
        flash("Ran create command")
        data = r.json()
    return render_template("home.html", data=data)

@app.route('/web/songs', methods=["GET", "POST"])
def song_landing():
    data = None
    if request.method == "POST":
        path = request.form.get("path")

        parameter = request.form.get("parameter")
        if parameter is None or parameter.strip() == "":
            flash("Must set key")
            return render_template("songs.html", data=data)

        get_url = "http://127.0.0.1:5000/songs/" + path + parameter
        print("Making request to %s" % get_url)
        # grab the response
        r = requests.get(get_url)
        if r.status_code >= 400:
            print("Error.  %s  Body: %s" % (r, r.content))
            return render_template("error.html", errmsg=r.json(), errcode=r.status_code)

        else:
            data = r.json()
    return render_template("songs.html", data=data)

@app.route('/web/artists', methods=["GET", "POST"])
def artists_landing():
    data = None
    if request.method == "POST":
        path = request.form.get("path")
        # Ensure path was submitted

        parameter = request.form.get("parameter")
        if parameter is None or parameter.strip() == "":
            flash("Must set key")
            return render_template("artists.html", data=data)

        get_url = "http://127.0.0.1:5000/artists/" + path + parameter
        # grab the response
        r = requests.get(get_url)
        if r.status_code >= 400:
            print("Error.  %s  Body: %s" % (r, r.content))
            return render_template("error.html", errmsg=r.json(), errcode=r.status_code)

        else:
            data = r.json()
    return render_template("artists.html", data=data)

@app.route('/web/albums', methods=["GET", "POST"])
def albums_landing():
    data = None
    if request.method == "POST":
        path = request.form.get("path")
        # Ensure path was submitted

        parameter = request.form.get("parameter")
        if parameter is None or parameter.strip() == "":
            flash("Must set key")
            return render_template("albums.html", data=data)

        get_url = "http://127.0.0.1:5000/albums/" + path + parameter
        # grab the response
        r = requests.get(get_url)
        if r.status_code >= 400:
            print("Error.  %s  Body: %s" % (r, r.content))
            return render_template("error.html", errmsg=r.json(), errcode=r.status_code)
        else:
            data = r.json()
    return render_template("albums.html", data=data)

@app.route('/web/analytics', methods=["GET", "POST"])
def analytics_landing():
    data = None
    if request.method == "POST":
        path = request.form.get("path")
        # Ensure path was submitted

        if path == "solo_albums":
            get_url = "http://127.0.0.1:5000/analytics/" + path
        elif path == "playcount/top_song/" or path == "playcount/top_country/":
            date = request.form.get("date")
            if date is None or date.strip() == "":
                flash("Must set key")
                return render_template("analytics.html", data=data)

            get_url = "http://127.0.0.1:5000/analytics/" + path + date
        elif path == "playcount/top_source/":
            parameter = request.form.get("parameter")
            if parameter is None or parameter.strip() == "":
                flash("Must set key")
                return render_template("analytics.html", data=data)
            parameter2 = request.form.get("parameter2")
            if parameter2 is None or parameter2.strip() == "":
                flash("Must set key")
                return render_template("analytics.html", data=data)
            get_url = "http://127.0.0.1:5000/analytics/" + path + parameter + '/' + parameter2
        else:
            parameter = request.form.get("parameter")
            if parameter is None or parameter.strip() == "":
                flash("Must set key")
                return render_template("analytics.html", data=data)

            get_url = "http://127.0.0.1:5000/analytics/" + path + parameter

        # grab the response
        print(get_url)
        r = requests.get(get_url)
        if r.status_code >= 400:
            print("Error.  %s  Body: %s" % (r, r.content))
            return render_template("error.html", errmsg=r.json(), errcode=r.status_code)

        else:
            data = r.json()
    return render_template("analytics.html", data=data)

# -----------------
# Utilities / Errors
# -------------------

# gets connection to database
def get_db_conn():
    db = getattr(g, '_database', None)
    if db is None:
        db = g._database = sqlite3.connect(DATABASE)

    return db

# Error Class for managing Errors
class InvalidUsage(Exception):
    status_code = 400

    def __init__(self, message, status_code=None, payload=None):
        Exception.__init__(self)
        self.message = message
        if status_code is not None:
            self.status_code = status_code
        self.payload = payload

    def to_dict(self):
        rv = dict(self.payload or ())
        rv['message'] = self.message
        return rv

@app.errorhandler(InvalidUsage)
def handle_invalid_usage(error):
    response = jsonify(error.to_dict())
    response.status_code = error.status_code
    return response

# called on close of response; closes db connection
@app.teardown_appcontext
def close_connection(exception):
    db = getattr(g, '_database', None)
    if db is not None:
        db.close()

# ########### post MS1 ############## #


# I think there can be something cool to come from this eventually, I'm bad at picking colors
import json
from json import JSONDecoder

class ColorSchemaDeserializationError(Exception):
    """ Exception deserializing color schema from JSON """
    pass

class ColorSchema:
    def __init__(self, name: str = None, serialized_json: str = None):
        if serialized_json:
            try:
                data = json.loads(serialized_json)
                self._name = data["colorSchema"]["identifier"]
                for each_rgb_tuple in data["colorSchema"]["rgb_values"]:
                    hex_code = "%02x%02x%02x" % (tuple(each_rgb_tuple))
                    self._color_table.append(hex_code)
            except JSONDecoder:
                raise ColorSchemaDeserializationError
            except IndexError:
                raise ColorSchemaDeserializationError
        else:
            self._name = name
            self._color_table = []

    def _get_color_for_index(self, index):
        if index < len(self._color_table):
            return self._color_table[index]
        else:
            corrected_index = index % len(self._color_table)
            return self._color_table[corrected_index]

    def hex_code(self, index: int) -> str:
        return self._get_color_for_index(index)

    def rgb(self, index: int) -> tuple:
        rgb = self._get_color_for_index(index)
        red = int(rgb[0:2], 16)
        green = int(rgb[2:4], 16)
        blue = int(rgb[4:6], 16)
        return red, green, blue

    def __str__(self) -> str:
        if self._name:
            return self._name
        else:
            return "DEFAULT"

    def serialize_json(self) -> str:
        output = {
            "colorSchema": {"identifier": str(self),
                            "rgb_values": []}
        }
        for x in range(len(self._color_table)):
            output["colorSchema"]["rgb_values"].append(self.rgb(x))
        return json.dumps(output)

class DefaultColorSchema(ColorSchema):
    def __init__(self):
        super().__init__(name="DEFAULT")
        colors = ["233142",
                  "facf5a",
                  "ff5959"
                  "4f9da6",
                  "022c43",
                  "ffd700",
                  "115173",
                  "053f5e",
                  "3a9679",
                  "fabc60",
                  "11144c",
                  "085f63",
                  "49beb7",
                  "facf5a",
                  "ff5959"]
        self._color_table.extend(colors)


# This figure requires the grackle chemistry and cooling library.
# Visit grackle.readthedocs.org for more information.

from matplotlib import pyplot

from utilities.testing import *

from pygrackle.grackle_wrapper import *
from pygrackle.fluid_container import FluidContainer

from utilities.api import \
     setup_fluid_container, \
     calculate_mean_molecular_weight, \
     calculate_hydrogen_number_density, \
     set_cosmology_units, \
     get_cooling_units

from utilities.primordial_equilibrium import \
     total_cooling, \
     nHI, nHII, nHeI, nHeII, nHeIII, ne

from utilities.physical_constants import \
     mass_hydrogen_cgs, \
     sec_per_Myr, \
     sec_per_Gyr, \
     cm_per_mpc

my_chem = chemistry_data()
my_chem.use_chemistry = 1
my_chem.with_radiative_cooling = 0
my_chem.primordial_chemistry = 1
my_chem.metal_cooling = 0
my_chem.UVbackground = 0
my_chem.include_metal_heating = 0
my_chem.comoving_coordinates = 0
my_chem.a_units = 1.0
my_chem.density_units = mass_hydrogen_cgs
my_chem.length_units = 1.0
my_chem.time_units = 1.0
my_chem.velocity_units = my_chem.length_units / my_chem.time_units
current_redshift = 0.0
fc = setup_fluid_container(my_chem, current_redshift=current_redshift,
                           converge=True, tolerance=1e-6, max_iterations=np.inf,
                           dt=(0.0001 * sec_per_Myr / my_chem.time_units))
calculate_temperature(fc)
a = 1.0 / (1.0 + current_redshift) / my_chem.a_units
calculate_cooling_time(fc, a)
t_sort = np.argsort(fc["temperature"])
t_cool = fc["cooling_time"][t_sort] * my_chem.time_units
my_T = fc["temperature"][t_sort]
my_nH = calculate_hydrogen_number_density(my_chem, fc).mean()
cooling_rate = fc["energy"][t_sort] / t_cool * fc["density"] * \
  my_chem.density_units / my_nH**2
eq_cooling = total_cooling(my_T, my_nH) / my_nH**2
eq_cooling_cen = total_cooling(my_T, my_nH, rates='cen') / my_nH**2

fontsize = 14
n_rows = 1
n_columns = 1

# blank space between edge of figure and active plot area
top_buffer = 0.03
bottom_buffer = 0.1
left_buffer = 0.12
right_buffer = 0.03

# blank space between plots
hor_buffer = 0.05
vert_buffer = 0.05

# calculate the height and width of each panel
panel_width = ((1.0 - left_buffer - right_buffer -
                ((n_columns-1)*hor_buffer)) / n_columns)
panel_height = ((1.0 - top_buffer - bottom_buffer -
                 ((n_rows-1)*vert_buffer)) / n_rows)

# create a figure (figsize is in inches
pyplot.figure()

### Cooling figure
axes = pyplot.axes((left_buffer, bottom_buffer,
                    panel_width, panel_height))
axes.loglog(my_T, eq_cooling, color='black', alpha=0.7,
            linestyle="--", linewidth=1.5)
axes.loglog(my_T, cooling_rate, color='black', alpha=0.7,
            linestyle="-", linewidth=1)
axes.loglog(my_T, eq_cooling_cen, color='black', alpha=0.7,
            linestyle=":", linewidth=1.5)
axes.xaxis.set_label_text('T [K]', fontsize=fontsize)
axes.yaxis.set_label_text('$\\Lambda$ / n${_{\\rm H}}^{2}$ [erg s$^{-1}$ cm$^{3}$]',
                          fontsize=fontsize)
axes.set_xlim(1e4, 1e9)
axes.set_ylim(1e-26, 2e-22)
tick_labels = axes.xaxis.get_ticklabels() + \
  axes.yaxis.get_ticklabels()
for tick_label in tick_labels:
    tick_label.set_size(fontsize)
pyplot.savefig('cooling.png')
pyplot.savefig('cooling.pdf')
pyplot.savefig('cooling.eps')

pyplot.clf()

### Ionization balance figure
axes = pyplot.axes((left_buffer, bottom_buffer,
                    panel_width, panel_height))

# Plot H ions
axes.loglog(my_T, (nHI(my_T, my_nH) /
                   (nHI(my_T, my_nH) +
                    nHII(my_T, my_nH))),
            color="#B82E00", alpha=0.7, linestyle="--", linewidth=1.5)
axes.loglog(my_T, (nHII(my_T, my_nH) /
                   (nHI(my_T, my_nH) +
                    nHII(my_T, my_nH))),
            color="#B88A00", alpha=0.7, linestyle="--", linewidth=1.5)

axes.loglog(my_T, (nHI(my_T, my_nH, rates='cen') /
                   (nHI(my_T, my_nH, rates='cen') +
                    nHII(my_T, my_nH, rates='cen'))),
            color="#B82E00", alpha=0.7, linestyle=":", linewidth=1.5)
axes.loglog(my_T, (nHII(my_T, my_nH, rates='cen') /
                   (nHI(my_T, my_nH, rates='cen') +
                    nHII(my_T, my_nH, rates='cen'))),
            color="#B88A00", alpha=0.7, linestyle=":", linewidth=1.5)

axes.loglog(my_T, (fc["HI"] / (fc["HI"] + fc["HII"])),
            label="HI", color="#B82E00", alpha=0.7, linestyle="-", linewidth=1.)
axes.loglog(my_T, (fc["HII"] / (fc["HI"] + fc["HII"])),
            label="HII", color="#B88A00", alpha=0.7, linestyle="-", linewidth=1.)

# Plot He ions
axes.loglog(my_T, (nHeI(my_T, my_nH) /
                   (nHeI(my_T, my_nH)   +
                    nHeII(my_T, my_nH)  +
                    nHeIII(my_T, my_nH))),
            color="#002EB8", alpha=0.7, linestyle="--", linewidth=1.5)
axes.loglog(my_T, (nHeII(my_T, my_nH) /
                   (nHeI(my_T, my_nH)   +
                    nHeII(my_T, my_nH)  +
                    nHeIII(my_T, my_nH))),
               color="#008AB8", alpha=0.7, linestyle="--", linewidth=1.5)
axes.loglog(my_T, (nHeIII(my_T, my_nH) /
                   (nHeI(my_T, my_nH)   +
                    nHeII(my_T, my_nH)  +
                    nHeIII(my_T, my_nH))),
               color="#00B88A", alpha=0.7, linestyle="--", linewidth=1.5)

axes.loglog(my_T, (nHeI(my_T, my_nH, rates='cen') /
                   (nHeI(my_T, my_nH, rates='cen')   +
                    nHeII(my_T, my_nH, rates='cen')  +
                    nHeIII(my_T, my_nH, rates='cen'))),
              color="#002EB8", alpha=0.7, linestyle=":", linewidth=1.5)
axes.loglog(my_T, (nHeII(my_T, my_nH, rates='cen') /
                   (nHeI(my_T, my_nH, rates='cen')   +
                    nHeII(my_T, my_nH, rates='cen')  +
                    nHeIII(my_T, my_nH, rates='cen'))),
              color="#008AB8", alpha=0.7, linestyle=":", linewidth=1.5)
axes.loglog(my_T, (nHeIII(my_T, my_nH, rates='cen') /
                   (nHeI(my_T, my_nH, rates='cen')   +
                    nHeII(my_T, my_nH, rates='cen')  +
                    nHeIII(my_T, my_nH, rates='cen'))),
              color="#00B88A", alpha=0.7, linestyle=":", linewidth=1.5)

axes.loglog(my_T, (fc["HeI"] / (fc["HeI"] + fc["HeII"] + fc["HeIII"])),
              label="HeI", color="#002EB8", alpha=0.7, linestyle="-", linewidth=1.)
axes.loglog(my_T, (fc["HeII"] / (fc["HeI"] + fc["HeII"] + fc["HeIII"])),
              label="HeII", color="#008AB8", alpha=0.7, linestyle="-", linewidth=1.)
axes.loglog(my_T, (fc["HeIII"] / (fc["HeI"] + fc["HeII"] + fc["HeIII"])),
              label="HeIII", color="#00B88A", alpha=0.7, linestyle="-", linewidth=1.)

axes.xaxis.set_label_text('T [K]', fontsize=fontsize)
axes.yaxis.set_label_text('fraction', fontsize=fontsize)
axes.set_xlim(1e4, 1e9)
axes.set_ylim(1e-10, 1)
tick_labels = axes.xaxis.get_ticklabels() + \
  axes.yaxis.get_ticklabels()
for tick_label in tick_labels:
    tick_label.set_size(fontsize)
axes.legend(loc='best', prop=dict(size=fontsize))
pyplot.savefig('fractions.png')
pyplot.savefig('fractions.pdf')
pyplot.savefig('fractions.eps')


"""
This file contains a function to get the coordinates from the GPS
"""
import serial
import time
import string
import pynmea2

#GPS gets coordinates
#UART communication protocols
def getCoordinates():
	port="/dev/ttyAMA0"
	ser=serial.Serial(port, baudrate=9600, timeout=0.5)
	dataout = pynmea2.NMEAStreamReader()
	newdata=ser.readline()

	if newdata[0:6] == "$GPRMC":
		newmsg=pynmea2.parse(newdata)
		lat=newmsg.latitude
		lng=newmsg.longitude
		gps = [lat,lng]
		return gps


# Importing the Pillow library
from PIL import Image, ImageDraw, ImageFont
import os

def text_wrapper(text, font, max_width):
    # Totally not stolen from eyong kevin https://gist.github.com/Eyongkevin/adbac2334f1355d8045111c264d80621
    list_of_lines = []
    if font.getlength(text) <= max_width:
        return text
    else:
        # split the line by spaces to get words
        words = text.split(' ')
        i = 0
        # append every word to a line while its width is shorter than the image width
        while i < len(words):
            line = ''
            while i < len(words) and font.getlength(line + words[i]) <= max_width:
                line = line + words[i] + " "
                i += 1
            if not line:
                line = words[i]
                i += 1
            list_of_lines.append(f"{line}")
        new_line = "\n"
        return new_line.join(list_of_lines)

def get_colors_from_colorcombo_image(colorcombo_path, colorcombo_outline_path):
    colorcombo_image = Image.open(colorcombo_path)
    colorcombo_outline = Image.open(colorcombo_outline_path)
    colorcombo_image.convert("RGBA")
    get_pixel_color = colorcombo_image.load()

    color_top = get_pixel_color[100, 20]
    color_top_font = get_pixel_color[500, 20]
    color_bottom = get_pixel_color[1000, 20]
    color_bottom_font = get_pixel_color[1500, 20]

    get_pixel_color = colorcombo_outline.load()
    color_outline = get_pixel_color[10, 10]
    color_outline = (color_outline[0], color_outline[1], color_outline[2], 190)

    return color_top, color_top_font, color_bottom, color_bottom_font, color_outline

def create_cover(thumbnail_author_path, video_thumbnail_path, author, title, colorcombo, sender, color_number, outline):
    # Cover size = 1280 x 1650
    colorcombo_path = f"resources/color_combinations/{colorcombo}"
    colorcombo_outline_path = f"resources/color_combinations/{colorcombo[:6]}_outline.png"

    colors_list = get_colors_from_colorcombo_image(colorcombo_path, colorcombo_outline_path)

    myfont = ImageFont.truetype("resources/Ubuntu-R.ttf", 50)
    image_cover = Image.new("RGBA", (1280, 1650), "white")
    image_top = Image.new("RGBA", (1280, 465), color=colors_list[0])
    image_icon = Image.open(thumbnail_author_path).convert("RGBA")
    image_thumbnail = Image.open(video_thumbnail_path).convert("RGBA")
    image_bottom = Image.new("RGBA", (1280, 465), color=colors_list[2])

    image_top_draw = ImageDraw.Draw(image_top)
    image_bottom_draw = ImageDraw.Draw(image_bottom)

    # FIXME: Implement text resizing based on height of the text.
    image_top_draw.text((100, 320), text_wrapper(author, myfont, 1000), font=myfont, fill=colors_list[1])
    image_bottom_draw.multiline_text((100, 100), text_wrapper(title, myfont, 1130), font=myfont, fill=colors_list[4])

    image_cover.paste(image_top, (0, 0))
    image_cover.paste(image_icon, (100, 100))
    image_cover.paste(image_thumbnail, (0, 465))
    image_cover.paste(image_bottom, (0, 720 + 465))

    # FIXME, add this outside. When implementing cover editor
    proper_senders = ["epub", "showcover", "thumbnail"]
    if sender in proper_senders:
        outline_rectangle = (colors_list[4][0], colors_list[4][1], colors_list[4][2], 190)
    else:
        if color_number < 4:
            outline_rectangle = (colors_list[color_number][0], colors_list[color_number][1], colors_list[color_number][2], 190)
        else:
            outline_rectangle = (colors_list[4][0], colors_list[4][1], colors_list[4][2], 190)

    if outline:
        layer_rectangle = Image.new("RGBA", image_cover.size, color=(0, 0, 0, 0))
        ImageDraw.Draw(layer_rectangle).rectangle((50, 50, 1230, 1600), outline=outline_rectangle, width=13)
        final_cover = Image.alpha_composite(image_cover, layer_rectangle)
    else:
        final_cover = image_cover

    # FIXME: Change from arbitray number "ifs" to more readeable form

    if sender == "epub":
        path = f"tmp/cover.png"
        final_cover.save(path, "PNG")
    elif sender == "thumbnail":
        final_cover.thumbnail((330,425))
        path = f"tmp/cover_thumbnail.png"
        final_cover.save(path, "PNG")
        return path
    else:
        # this should be used by cover.py for testing purposes only
        if color_number < 4:
            path = f"tmp/cover{colorcombo[:6]}{color_number}.png"
            final_cover.save(path, "PNG")
        elif color_number == 10:
            path = f"tmp/cover{colorcombo[:6]}{color_number}.png"
            final_cover.save(path, "PNG")
        elif color_number == 42:
            final_cover.thumbnail((256,330))
            path = f"tmp/{colorcombo[:6]}_thumbnail.png"
            final_cover.save(path, "PNG")
        elif color_number == 5:
            # This is for testing purposes only
            print("imhere")
            path = f"tmp/cover_thumbnail.png"
            final_cover.save(path, "PNG")
        return path

def generate_all(type_of_ouput):
    # Testing different color combinations and outlines of said combinations
    author_thumbnail_path = "resources/test_graphics/author_thumbnail_test.png"
    video_thumbnail_path = "resources/test_graphics/thumbnail_test.png"
    author = "This is a test of Author Name"
    title = "This is just a Title Test Case with some words when you fill out the Youtube URL field you will see " \
                "different result"
    path = "resources/color_combinations"
    color_combinations = [0, 1, 2, 3]
    for colorcombo in os.listdir(path):
        if "outline" in colorcombo:
                    pass
        else:
            if type_of_ouput == "thumbnails":
                    create_cover(author_thumbnail_path, video_thumbnail_path, author, title, colorcombo, "epub", 42, True)

            elif type_of_ouput == "all":
                    for color in color_combinations:
                            create_cover(author_thumbnail_path, video_thumbnail_path, author, title, colorcombo, "epub", color, True)

            elif type_of_ouput== "outline":
                    create_cover(author_thumbnail_path, video_thumbnail_path, author, title, colorcombo, "epub", 5, True)

def main():

    # create_cover(url, video_thumbnail_path, author, title, "combo4.png", "epub", 10)
    # generate_all_cover_options(url, video_thumbnail_path, author, title, "outline")
    # type of options
    # "thumbnails" - generates thumbnail versions of covers
    # "outline" - generates covers from all theme files using coresponding *_outline.png file
    # "all" - generates covers using all base colors of a cover theme to get covers. Outuput is number_of_colors*number_of_combination.png_files of cover files
    generate_all("all")

if __name__ == "__main__":
    main()


# Creates a basic BLE Advertiser
# Check the Advertiser.py class for more info

import sys
import time
from datetime import datetime, timedelta

from Advertiser import Advertiser

# Advertise
def main():
    dt = datetime.now()
    print("Started at: %s" % dt)

    # Create advertiser
    adv = Advertiser()

    # Initiate BLE stack
    if adv.setupBleStack('Pi Range Tester 2', interval=1000):

        # Start advertising
        adv.enableAdvertisements(en=True)

        # Advertise for n seconds
        time.sleep(600)

        # Disable advertisements
        adv.enableAdvertisements(en=False)

    # Stop
    print("Stopped at: %s" % datetime.now())

# Program start
if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        pass
    finally:
        print("CTRL-C Detected")
        print("Stopped at: %s" % datetime.now())
        sys.exit()


from django.utils.translation import ugettext_lazy as _

from utils.faker_factory import faker

from ..mails import BaseMailView

class ProjectsMemberRoleChangedMailView(BaseMailView):
    """
    """
    template_name = 'mails/projects/member_role_changed.html'

    mandatory_mail_args = [
        'name',
        'user_name',
        'roles',
        'public_url',
    ]
    subject = _('Your role has changed: You are now acting as %(roles)s in %(name)s')
    section = 'projects'

    def get_mock_data(self, optional=True):
        mock_data = {
            'name': '[Project Name]',
            'roles': ['ExO Head Coach', 'Observer'],
            'user_name': '[Name]',
            'disable_notification_url': None,
            'public_url': '/{}'.format(faker.uri_path()),
        }
        return mock_data


import matplotlib.pyplot as plt
import numpy as np
from sympy import *
x = Symbol('x')
#! INPUT
print("""
Bắt đầu chương trình
Đây là phương pháp lặp đơn
INPUT
f(x) với f(x)=0
khoảng phân li nghiệm (a,b)

Và thêm SaiSoE đã cho
INPUT
""")

def Phi(x):
# ** là mũ
    # return 1/(x-1.5)
    return -1/3*x**3

a = -2
b = 2
SaiSoE = 10
print(f"""
INPUT
{Phi(x)} , \tvới f(x)=0
khoảng phân li nghiệm (a,b) = ({a},{b})

Và thêm SaiSoE đã cho: SaiSoE = {SaiSoE}
INPUT
""")
#! INPUT
#! KIỂM TRA ĐIỀU KIỆN PP
print(f"""
#! Điều kiện của phương pháp lặp đơn là:
1. Hàm Phi(x) co <=> 0<q<1 (q=MAX_đạo hàm Phi(x))
""")
dao_ham_Phi= diff(Phi(x),x)
print(dao_ham_Phi)
x_ve_hinh=np.linspace(a,b,1000)
y_ve_hinh=[dao_ham_Phi.subs(x,i) for i in x_ve_hinh]
q=max(y_ve_hinh)
print(f"q = {q}\n Xét 0<q<1 => ",end=" ")
if(0<q and q<1):
    print(f"Đúng")
else:
    print(f"Sai")
plt.plot(x_ve_hinh,y_ve_hinh)
plt.show()

#! KIỂM TRA ĐIỀU KIỆN PP

print("Kết thúc chương trình")


from django.shortcuts import render,redirect
from django.core.paginator import Paginator
from siteSettings.models import SiteSettings
from django.contrib import messages
from users.models import User
from carts.models import *
from .models import *

def courses_page(request):
    if request.user.is_authenticated:
        if request.user.phone_authentication != True:
            return redirect('account:code_page')
        else:
            user_info = User.objects.filter(username=request.user.username).first()

            if user_info.block == True:
                messages.error(request, 'اکانت شما مسدود شده است لطفا با پشتیبانی تماس بگیرید')
                return redirect('contact:contactus_page')
    else: return redirect('account:login_page')

    posts = Courses.objects.filter(CourseStatus=True).all()
    paginator = Paginator(posts, 9)
    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)
    settings = SiteSettings.objects.last()
    title = settings.title + ' ' + '-' + ' ' + 'دوره ها'
    context = {
        'title': title,
        'posts': page_obj,
    }
    return render(request,'courses/courses_page/coursesPage.html',context)

def ShowCourse_page(request,id):
    if request.user.is_authenticated:
        if request.user.phone_authentication != True:
            return redirect('account:code_page')
        else:
            user_info = User.objects.filter(username=request.user.username).first()

            if user_info.block == True:
                messages.error(request, 'اکانت شما مسدود شده است لطفا با پشتیبانی تماس بگیرید')
                return redirect('contact:contactus_page')
    else: return redirect('account:login_page')

    cart = Course.objects.filter(course_id=id,is_paid=True).first()
    status = False
    if cart is not None:
        status = True

    videos = Videos.objects.filter(course_id=id).all()
    videos_count = Videos.objects.filter(course_id=id).count()
    Introduction_Video = IntroductionVideo.objects.filter(course_id=id).last()

    course = Courses.objects.filter(id=id).first()
    settings = SiteSettings.objects.last()
    title = settings.title + ' ' + '-' + ' ' + f'{course.CourseName}'
    context = {
        'title': title,
        'status': status,
        'videos': videos,
        'videos_count': videos_count,
        'keyWord': course.keyWord,
        'Introduction_Video': Introduction_Video,
        'course': course,
    }

    return render(request,'courses/showCourse_page/show_course.html',context)


import nltk.classify.util
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import movie_reviews

def extract_features(word_list):
    return dict([(word, True) for word in word_list])

if __name__=='__main__':
   # Load positive and negative reviews
   positive_fileids = movie_reviews.fileids('pos')
   negative_fileids = movie_reviews.fileids('neg')
   features_positive = [(extract_features(movie_reviews.words(fileids=[f])),
                         'Positive') for f in positive_fileids]
   features_negative = [(extract_features(movie_reviews.words(fileids=[f])),
                         'Negative') for f in negative_fileids]

   # Split the data into train and test (80/20)
   threshold_factor = 0.8
   # threshold_factor = 0.7
   # threshold_factor = 0.5
   threshold_positive = int(threshold_factor * len(features_positive))
   threshold_negative = int(threshold_factor * len(features_negative))

   features_train = features_positive[:threshold_positive] + features_negative[:threshold_negative]
   features_test = features_positive[threshold_positive:] + features_negative[threshold_negative:]
   print("\nNumber of training datapoints:", len(features_train))
   print("Number of test datapoints:", len(features_test))

   # Train a Naive Bayes classifier
   classifier = NaiveBayesClassifier.train(features_train)
   print("\nAccuracy of the classifier:", nltk.classify.util.accuracy(classifier, features_test))

   print("\nTop 10 most informative words:")
   for item in classifier.most_informative_features()[:10]:
        print(item[0])

    # Sample input reviews
   # input_reviews = [
   #     "It is an amazing movie",
   #     "This is a dull movie. I would never recommend it to anyone.",
   #     "The cinematography is pretty great in this movie",
   #     "The direction was terrible and the story was all over the place"
   # ]
   input_reviews = [
       "This was easy to put together. It is sturdy ad a perfect fit for my daughter's room. We have one drawer that sticks a little COMMA  but it works.",
       "I loved the look of this dresser in the store and decided to take the plunge. It was a major project to assemble (6-ish hours for one relatively handy person without power tools) COMMA  but the finished product looks great and stores a ton of clothes! The directions could definitely be a little clearer on assembling the middle divider pieces COMMA  which looks wrong even when done correctly and the dimples in the wood for orientation look like holes in the instructions. I couldn't get two of the four screws that connect the front face to the dresser top to go in (screws too short or holes not quite aligned) COMMA  but thankfully there were many other points of attachment and it's not at all obvious that they're missing. And mine came with metal (not plastic) cam locks COMMA  which is a good thing. Great buy!",
       "We were very disappointed to realize that the hemnes set in white is made of mostly particle board. We were under the impression that the all the hemnes line was made of solid wood. After further investigation it seems as though all the dressers are made of wood except the white ones. Not sure why this is and is very misleading",
       "I not only purchased the dresser but I bought the matching chest I'd drawers. The pieces took a while to put together but they are worth the time. Great product."
   ]
   print("\nPredictions:")
   for review in input_reviews:
       print("\nReview:", review)
       probdist = classifier.prob_classify(extract_features(review.split()))
       pred_sentiment = probdist.max()
       print("Predicted sentiment:", pred_sentiment)
       print("Probability:", round(probdist.prob(pred_sentiment), 2))


'''
main driver file. Responsible for handling user inputs and displaying current GameState
'''

import pygame as p
from PIL import Image
import ChessEngine

p.init()
WIDTH = HEIGHT = 512  # or you can keep it as 512
DIMENSION = 8  # Chess boards are 8x8
SQ_SIZE = HEIGHT // DIMENSION
IMAGES = dict.fromkeys(['wR', 'wN', 'wB', 'wQ', 'wK', 'wp', 'bR', 'bN', 'bB', 'bQ', 'bK', 'bp'])
SET_FPS = 15

def loadImages ():
    for key in list (IMAGES.keys()):  # Use a list instead of a view
        IMAGES[key] = p.transform.scale(p.image.load('/Users/joshmachado/Desktop/Studies/coding/Chess-Engine/images/{}.png'
                                                     .format(key)), (SQ_SIZE, SQ_SIZE))

def main():
    screen = p.display.set_mode((WIDTH, HEIGHT))
    clock = p.time.Clock()
    screen.fill(p.Color('White'))
    loadImages()
    gs = ChessEngine.GameState()
    validMoves = gs.getValidMoves()
    moveMade = False   # flag for when a move is made
    sqSelected = ()    # Empty tuple which will save the position of the last square selected by user
    playerClicks = []  # List of two tuples tracking the clicks eg: [(6,4),(4,4)]

    running = True
    while running:
        for e in p.event.get():
            if e.type == p.QUIT:
                running = False
            elif e.type ==p.MOUSEBUTTONDOWN:
                location = p.mouse.get_pos()
                col = location[0]//SQ_SIZE
                row = location[1]//SQ_SIZE

                if sqSelected == (row, col):  # If the same square is selected twice, it:
                    sqSelected = ()           # Deselects the square
                    playerClicks = []         # Resets playerClicks
                else:
                    sqSelected = (row, col)
                    playerClicks.append(sqSelected)

                if len(playerClicks)==2:    # This indicates that 2 clicks have been made (done after second click)
                    move = ChessEngine.Move(playerClicks[0],playerClicks[1], gs.board)

                    if move in validMoves:
                        print(move.getChessNotation())
                        gs.makeMove(move)
                        moveMade = True
                        sqSelected = ()
                        playerClicks =[]

                    else:
                        playerClicks = [sqSelected]
            # Key handles
            elif e.type == p.KEYDOWN:
                if e.key == p.K_z:      # Undo the move made
                    gs.undoMove()
                    moveMade = True
        if moveMade:
            validMoves = gs.getValidMoves()
            moveMade = False

        drawGameState(screen, gs)
        clock.tick(SET_FPS)
        p.display.flip()

def drawGameState(screen, gs):
    drawBoard(screen)
    drawPieces(screen, gs.board)

'''
drawBoard is going to draw just the board without the pieces.

*Note to self*
Top left square of the chess board is always white irrespective of which colour you're playing with
'''

def drawBoard(screen):
    colours = [p.Color('white'),p.Color('grey')]
    for r in range(DIMENSION):
        for c in range(DIMENSION):
            colour = colours[((r+c)%2)]
            p.draw.rect(screen, colour, p.Rect(c*SQ_SIZE, r*SQ_SIZE, SQ_SIZE, SQ_SIZE))

'''
drawPieces is going to draw the pieces on the board given the current GameState
'''
def drawPieces(screen, board):
    for r in range(DIMENSION):
        for c in range(DIMENSION):
            piece = board[r][c]
            if piece != '--':
                screen.blit(IMAGES[piece], p.Rect(c*SQ_SIZE, r*SQ_SIZE, SQ_SIZE, SQ_SIZE))

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jan 22 20:10:01 2018

@author: Suveen
"""

from urllib.request import urlopen
from bs4 import BeautifulSoup
import ssl
import re

# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

url = input('Enter - ')
html = urlopen(url, context=ctx).read()

# html.parser is the HTML parser included in the standard Python 3 library.
# information on other HTML parsers is here:
# http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser
soup = BeautifulSoup(html, "html.parser")

# Retrieve all of the anchor tags
tags = soup('span')
count = 0
com_sum = 0
numlist= list()
for tag in tags:
    # Look at the parts of a tag
    count= count+1
    str_tag= str(tag)
#    print('TAg : ',tag)
#    print('String', str_tag)
    stuff=re.findall('[0-9]+',str_tag)

    for k in range(len(stuff)):
        num = float(stuff[k])
        com_sum = com_sum+ num
#    print (num)
        numlist.append(num)
#print(numlist)

print('Count ',count)
print('Sum ', int(com_sum))


from flask import Flask,render_template,request
from pymongo import MongoClient
import subprocess as sp
import os

mongo_server_url="mongodb://127.0.0.1:27017"
client = MongoClient(mongo_server_url)

app = Flask("chat app")
#here i am going to defind database and collection name
db ="lw"
collection="flask"

#We have to specify that In which IP and port you want to run this web app
port=80
hostname="0.0.0.0"

@app.route('/')
def home():
    #Very Important steps
    file=open("names.yml" , "r+")
    file.truncate(0)
    file.close()
    file=open("mobiles.yml" , "r+")
    file.truncate(0)
    file.close()
    file=open("emails.yml" , "r+")
    file.truncate(0)
    file.close()
    return render_template("index.html")

@app.route('/sumbitted' , methods=['POST'])
def sumbitted():
    name=request.form.get("name")
    mobile=request.form.get("mobile")
    email=request.form.get("email")
    #inserting the data in mongodb server
    client[db][collection].insert({"name": name , "mobile": mobile , "email": email})
    #now i want to do some manupulation to store the name, mobile and email

    # file handling for "names.yml"
    f = open("names.yml" ,"a")
    var="moqaddas"
    f.write("name: ")
    f.write(name)
    f.close()

    # file handling for "mobiles.yml"
    f = open("mobiles.yml" ,"a")
    var="moqaddas"
    f.write("mobile: ")
    f.write(mobile)
    f.close()

    # file handling for "names.yml"
    f = open("emails.yml" ,"a")
    var="moqaddas"
    f.write("email: ")
    f.write(email)
    f.close()

    m=sp.getstatusoutput("ansible-playbook mail.yml  --vault-password-file password.txt")
    exit_code=m[0]
    output=m[1]
    if exit_code==0:

        return render_template("response.html")
    else: return render_template("err.html")

app.run(debug=True , port=port , host=hostname)


import pandas as pd
from sklearn.ensemble import RandomForestClassifier

def train_random_forest():
    # ****** Fit a random forest and extract predictions
    #
    forest = RandomForestClassifier(n_estimators = 100)

    # Fitting the forest may take a few minutes
    print("Fitting a random forest to labeled training data...")
    forest = forest.fit(train_centroids, train["sentiment"])
    result = forest.predict(test_centroids)

    # Write the test results
    output = pd.DataFrame(data={"id": test["id"], "sentiment":result})
    output.to_csv("BagOfCentroids.csv", index=False, quoting=3)
    print("Wrote BagOfCentroids.csv")


import pyaudio
import numpy as np
from core.tools.buffer import AudioBuffer

class PyAudio():
    def __init__(self,verbose=True):
        """
        instantiate pyaudio client and buffer for input stream recording to plot
        """
        self.input_stream = None # on start will beinput stream from mic via pyaudio

        self.client=pyaudio.PyAudio()

        self.verbose=verbose

    def stream_read(self,chunk):
        """
        return values for a single chunk
        """
        data = np.frombuffer(self.input_stream.read(chunk),dtype=np.int16)
        return data

    def stream_start(self,rate,chunk):
        """
        start audio input stream
        """
        if self.verbose:
            print(" -- stream started")
        self.input_stream=self.client.open(format=pyaudio.paInt16,channels=1,
                                rate=rate,input=True,
                                frames_per_buffer=chunk)

    def stream_stop(self):
        """
        close the stream but keep the PyAudio instance alive.
        """
        self.input_stream.stop_stream()
        self.input_stream.close()
        if self.verbose:
            print("Programatically closing stream")


#!/usr/bin/python3
import requests
import subprocess  # to execute bash commands
import sys

try:
    check_for_package = subprocess.Popen(("dpkg", "-s", "html2text"), stdout=subprocess.PIPE)
    output = subprocess.check_output(("grep", "Status"), stdin=check_for_package.stdout)
    check_for_package.wait()
    opstr = str(output, 'utf-8')
    print(opstr)
    if opstr == "Status: install ok installed\n":
        print("Package installed")

except Exception as e:
    print("installing html2text..............................")
    install_pkg = subprocess.check_call("sudo apt install html2text", shell=True)

try:
    tracking_number = str(sys.argv[1])

except(IndexError, ValueError):
    print("please enter a tracking number of a valid format")
    sys.exit(2)

request_url = "http://ipsweb.ptcmysore.gov.in/ipswebtracking/IPSWeb_item_events.aspx?itemid=" + tracking_number
# print(request_url)
r = requests.get(request_url)
print(r.status_code)

f = open("raw_html", "w+")
f.write(r.text)
f.close()

view_html = subprocess.Popen(["html2text", "raw_html"])
output = view_html.communicate()
view_html.wait()
print(output)


import click
import requests
from tabulate import tabulate

class github:
    def __init__(self,ghf):
        self.ghf=ghf

    # def repos(self,org):
    #     if self.ghf.debug: click.echo("org:"+org+" token:"+self.ghf.token)
    #     url='https://api.github.com/orgs/'+org+'/repos'
    #     headers=self.get_auth_header()
    #     data=''
    #     r = requests.get(url, headers=headers, data=data)
    #     fields = ['full_name']
    #     if len(self.ghf.fields) > 0:
    #         fields = self.ghf.fields
    #     if self.ghf.profile:
    #         self.profile_keys(r.json())
    #     else:
    #         self.print_object_array_with_fields(r.json(),fields)

    def repos(self,org):
        url='https://api.github.com/orgs/'+org+'/repos'
        fields = ['full_name']
        headers=self.get_auth_header()
        self.make_github_get_request(fields,url,headers)

    def make_github_get_request(self,default_fields,url,headers,data=''):
        if self.ghf.debug: click.echo("url:"+url+" token:"+self.ghf.token+" fields:"+",".join(default_fields))
        r = requests.get(url, headers=headers, data=data)
        fields = default_fields
        if len(self.ghf.fields) > 0:
            fields = self.ghf.fields
        table = []
        if self.ghf.profile:
            (fields,table) = self.get_profile_table(r.json())
        else:
            table = self.get_table_from_object_array_with_fields(fields,r.json())
        self.print_table(fields,table)

    def print_table(self, field_names, table):
        if self.ghf.export_csv:
            separator = self.ghf.csv_separator
            if self.ghf.print_header_row:
                click.echo(separator.join(field_names))
            for entry in table:
                click.echo(separator.join(entry))
        else:
            if self.ghf.print_header_row:
                click.echo(tabulate(table, field_names, tablefmt="simple"))
            else:
                click.echo(tabulate(table, tablefmt="simple"))

    def test_post(self,url,headers,data):
        r = requests.post(url, headers=headers, data=data)
        if r.status_code == 201:
            click.echo(r.json())
        elif r.status_code == 401:
            click.echo("Error:"+r.json()['message'])
        else:
            click.echo('status:'+str(r.status_code))
            click.echo(r.text)

    def test_get(self,url,headers,data):
        r = requests.get(url, headers=headers, data=data)
        if r.status_code == 201:
            click.echo(r.json())
        elif r.status_code == 401:
            click.echo("Error:"+r.json()['message'])
        else:
            click.echo('status:'+str(r.status_code))
            click.echo(r.text)

    def get_auth_header(self):
        headers={'Authorization' : 'token '+self.ghf.token}
        return headers

    def get_profile_table(self,json):
        outter_key_hash = {}
        inner_key_hash = {}
        if type(json) == type([]):
            for item in json:
                if type(item) == type(u''):
                    outter_key_hash[item] = 1 if item not in outter_key_hash else outter_key_hash[item] + 1
                if type(item) == type({}):
                    for inner_item in item:
                        if type(inner_item) == type(u''):
                            inner_key_hash[inner_item] = 1 if inner_item not in inner_key_hash else inner_key_hash[inner_item] + 1
        # elif type(json) == type({}):
#             None
        table = []
        for key, value in outter_key_hash.items():
            table.append(['level1',key,str(value)])
        for key, value in inner_key_hash.items():
            table.append(['level2',key,str(value)])
        field_names = ['level', 'name','count']
        table = sorted(table, key=lambda key: key[1])
        return (field_names,table)
#        click.echo(tabulate(table, field_names, tablefmt="simple"))

    def get_table_from_object_array_with_fields(self,fields,json):
        table = []
        if type(json) == type([]):
            for item in json:
                if type(item) == type({}):
                    row = []
                    for field in fields:
                        if field in item:
                            row.append(item[field])
                        else:
                            row.append('')
                    table.append(row)
        headers = fields
        return table
#        click.echo(tabulate(table, headers, tablefmt="simple"))

    # >>> js = ['name1', 'name2', {'iname1':11,'iname2':12}]
    # >>> for item in js:
    # ...     print type(item)
    # ...
    # <type 'str'>
    # <type 'str'>
    # <type 'dict'>