# Copyright 2021, Google LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utility methods for working with Keras in TensorFlow Federated.

There are several changes worth highlighting: (1) the usage of two-branch
networks; (2) the usage of label smoothing and regularization; (3) batch-based
voting for determing the branch.
"""

# TODO(b/193904908): add unit tests.

import collections
from typing import Callable, List, Optional, OrderedDict, Union
import warnings

import attr
import tensorflow as tf
import tensorflow_federated as tff

Loss = Union[tf.keras.losses.Loss, List[tf.keras.losses.Loss]]

def from_keras_model(
    keras_model: tf.keras.Model,
    loss: Loss,
    input_spec,
    loss_weights: Optional[List[float]] = None,
    metrics: Optional[Union[List[tf.keras.metrics.Metric],
                            List[Callable[[],
                                          tf.keras.metrics.Metric]]]] = None,
    from_logits: bool = True,
    uniform_reg: float = 0.,
    label_smoothing: float = 1.,
    batch_majority_voting: bool = False,
) -> tff.learning.Model:
  """Builds a `tff.learning.Model` from a `tf.keras.Model`.

  The `tff.learning.Model` returned by this function uses `keras_model` for
  its forward pass and autodifferentiation steps.

  Notice that since TFF couples the `tf.keras.Model` and `loss`,
  TFF needs a slightly different notion of "fully specified type" than
  pure Keras does. That is, the model `M` takes inputs of type `x` and
  produces predictions of type `p`; the loss function `L` takes inputs of type
  `<p, y>` (where `y` is the ground truth label type) and produces a scalar.
  Therefore in order to fully specify the type signatures for computations in
  which the generated `tff.learning.Model` will appear, TFF needs the type `y`
  in addition to the type `x`.

  Note: This function does not currently accept subclassed `tf.keras.Models`,
  as it makes assumptions about presence of certain attributes which are
  guaranteed to exist through the functional or Sequential API but are
  not necessarily present for subclassed models.

  Note: This function raises a UserWarning if the `tf.keras.Model` contains a
  BatchNormalization layer, as the batch mean and variance will be treated as
  non-trainable variables and won't be updated during the training (see
  b/186845846 for more information). Consider using Group Normalization instead.

  Args:
    keras_model: A `tf.keras.Model` object that is not compiled.
    loss: A single `tf.keras.losses.Loss` or a list of losses-per-output. If a
      single loss is provided, then all model output (as well as all prediction
      information) is passed to the loss; this includes situations of multiple
      model outputs and/or predictions. If multiple losses are provided as a
      list, then each loss is expected to correspond to a model output; the
      model will attempt to minimize the sum of all individual losses
      (optionally weighted using the `loss_weights` argument).
    input_spec: A structure of `tf.TensorSpec`s or `tff.Type` specifying the
      type of arguments the model expects. If `input_spec` is a `tff.Type`, its
      leaf nodes must be `TensorType`s. Note that `input_spec` must be a
      compound structure of two elements, specifying both the data fed into the
      model (x) to generate predictions as well as the expected type of the
      ground truth (y). If provided as a list, it must be in the order [x, y].
      If provided as a dictionary, the keys must explicitly be named `'{}'` and
      `'{}'`.
    loss_weights: (Optional) A list of Python floats used to weight the loss
      contribution of each model output (when providing a list of losses for the
      `loss` argument).
    metrics: (Optional) a list of `tf.keras.metrics.Metric` objects or a list of
      no-arg callables that each constructs a `tf.keras.metrics.Metric`.
    from_logits: Whether the outputs of the model are logits or softmax scores.
    uniform_reg: Weight for the label-smoothing regularization.
    label_smoothing: Strength of the label smoothing.
    batch_majority_voting: Whether to use batch majority voting during
      evaluation.

  Returns:
    A `tff.learning.Model` object.

  Raises:
    TypeError: If `keras_model` is not an instance of `tf.keras.Model`, if
      `loss` is not an instance of `tf.keras.losses.Loss` nor a list of
      instances of `tf.keras.losses.Loss`, if `input_spec` is a `tff.Type` but
      the leaf nodes are not `tff.TensorType`s, if `loss_weight` is provided but
      is not a list of floats, or if `metrics` is provided but is not a list of
      instances of `tf.keras.metrics.Metric`.
    ValueError: If `keras_model` was compiled, if `loss` is a list of unequal
      length to the number of outputs of `keras_model`, if `loss_weights` is
      specified but `loss` is not a list, if `input_spec` does not contain
      exactly two elements, or if `input_spec` is a dictionary and does not
      contain keys `'x'` and `'y'`.
  """.format(tff.learning.model.MODEL_ARG_NAME,
             tff.learning.model.MODEL_LABEL_NAME)
  # Validate `keras_model`
  tff.types.py_typecheck.check_type(keras_model, tf.keras.Model)  # pytype: disable=module-attr
  if keras_model._is_compiled:  # pylint: disable=protected-access
    raise ValueError('`keras_model` must not be compiled')

  # Validate and normalize `loss` and `loss_weights`
  if not isinstance(loss, list):
    tff.types.py_typecheck.check_type(loss, tf.keras.losses.Loss)  # pytype: disable=module-attr
    if loss_weights is not None:
      raise ValueError('`loss_weights` cannot be used if `loss` is not a list.')
    loss = [loss]
    loss_weights = [1.0]
  else:
    if len(loss) != len(keras_model.outputs):
      raise ValueError('If a loss list is provided, `keras_model` must have '
                       'equal number of outputs to the losses.\nloss: {}\nof '
                       'length: {}.\noutputs: {}\nof length: {}.'.format(
                           loss, len(loss), keras_model.outputs,
                           len(keras_model.outputs)))
    for loss_fn in loss:
      tff.types.py_typecheck.check_type(loss_fn, tf.keras.losses.Loss)  # pytype: disable=module-attr

    if loss_weights is None:
      loss_weights = [1.0] * len(loss)
    else:
      if len(loss) != len(loss_weights):
        raise ValueError(
            '`keras_model` must have equal number of losses and loss_weights.'
            '\nloss: {}\nof length: {}.'
            '\nloss_weights: {}\nof length: {}.'.format(loss, len(loss),
                                                        loss_weights,
                                                        len(loss_weights)))
      for loss_weight in loss_weights:
        tff.types.py_typecheck.check_type(loss_weight, float)  # pytype: disable=module-attr

  if len(input_spec) != 2:
    raise ValueError('The top-level structure in `input_spec` must contain '
                     'exactly two top-level elements, as it must specify type '
                     'information for both inputs to and predictions from the '
                     'model. You passed input spec {}.'.format(input_spec))
  if isinstance(input_spec, tff.types.Type):
    if not tff.types.is_structure_of_tensors(input_spec):
      raise TypeError(
          'Expected a `tff.Type` with all the leaf nodes being '
          '`tff.TensorType`s, found an input spec {}.'.format(input_spec))
    input_spec = tff.structure_from_tensor_type_tree(
        lambda tensor_type: tf.TensorSpec(tensor_type.shape, tensor_type.dtype),
        input_spec)
  else:
    tensor_spec = (tf.TensorSpec, tf.RaggedTensorSpec)
    tf.nest.map_structure(
        lambda s: tff.types.py_typecheck.check_type(  # pylint: disable=g-long-lambda  # pytype: disable=module-attr
            s, tensor_spec, 'input spec member'),
        input_spec)
  if isinstance(input_spec, collections.abc.Mapping):
    if tff.learning.model.MODEL_ARG_NAME not in input_spec:
      raise ValueError(
          'The `input_spec` is a collections.abc.Mapping (e.g., a dict), so it '
          'must contain an entry with key `\'{}\'`, representing the input(s) '
          'to the Keras model.'.format(tff.learning.model.MODEL_ARG_NAME))
    if tff.learning.model.MODEL_LABEL_NAME not in input_spec:
      raise ValueError(
          'The `input_spec` is a collections.abc.Mapping (e.g., a dict), so it '
          'must contain an entry with key `\'{}\'`, representing the label(s) '
          'to be used in the Keras loss(es).'.format(
              tff.learning.model.MODEL_LABEL_NAME))

  if metrics is None:
    metrics = []
  else:
    tff.types.py_typecheck.check_type(metrics, list)  # pytype: disable=module-attr

  for layer in keras_model.layers:
    if isinstance(layer, tf.keras.layers.BatchNormalization):
      warnings.warn(
          'Batch Normalization contains non-trainable variables that won\'t be '
          'updated during the training. Consider using Group Normalization '
          'instead.', UserWarning)
      break

  return _KerasModel(
      keras_model,
      input_spec=input_spec,
      loss_fns=loss,
      loss_weights=loss_weights,
      metrics=metrics,
      uniform_reg=uniform_reg,
      from_logits=from_logits,
      label_smoothing=label_smoothing,
      batch_majority_voting=batch_majority_voting)

@attr.s(frozen=True, slots=True, eq=False)
class BatchOutput():
  """A structure that holds the output of a `tff.learning.Model`.

  Note: All fields are optional (may be None).

  Attributes:
    loss: The scalar mean loss on the examples in the batch. If the model has
      multiple losses, it is the sum of all the individual losses.
    predictions: Tensor of predictions on the examples. The first dimension must
      be the same size (the size of the batch).
    num_examples: Number of examples seen in the batch.
    features: The specified faetures for k-means.
  """
  loss = attr.ib()
  predictions = attr.ib()
  num_examples = attr.ib()
  features = attr.ib()

def kld_no_reduce(logits, kld_fn, from_logits, labels, smoothing):
  """Compute the KL divergence with the smoothed label.

  Args:
    logits: The logits from the model.
    kld_fn: The tf function for kl divergence, should set reduce to False
    from_logits: Whether the logits are real logits. Sometimes the models only
      outputs the softmax scores.
    labels: The one-hot label.
    smoothing: Label smoothing, values from 0 to 1. 0 means no smoothing, 1
      means uniform over all classes.

  Returns:

  """
  uniform_p = tf.ones_like(logits) / tf.cast(tf.shape(logits)[-1], logits.dtype)
  if smoothing < 1.:
    labels_onehot = tf.one_hot(labels, tf.shape(logits)[-1], dtype=logits.dtype)
    uniform_p = uniform_p * smoothing + tf.cast(
        labels_onehot, dtype=logits.dtype) * (1 - smoothing)
  if from_logits:
    sm_score = tf.nn.softmax(logits, axis=-1)
  else:
    sm_score = logits
  ret = kld_fn(uniform_p, sm_score)
  return ret

class _KerasModel(tff.learning.Model):
  """Internal wrapper class for tf.keras.Model objects."""

  def __init__(
      self,
      keras_model: tf.keras.Model,
      input_spec,
      loss_fns: List[tf.keras.losses.Loss],
      loss_weights: List[float],
      metrics: Union[List[tf.keras.metrics.Metric],
                     List[Callable[[], tf.keras.metrics.Metric]]],
      from_logits: bool = True,
      label_smoothing: float = 1.,
      uniform_reg: float = 0.,
      batch_majority_voting: bool = False,
  ):

    self._keras_model = keras_model
    self._input_spec = input_spec
    self._loss_fns = loss_fns
    self._loss_weights = loss_weights
    self._metrics = metrics
    self._label_smoothing = label_smoothing
    self._uniform_reg = uniform_reg
    self._batch_majority_voting = batch_majority_voting

    self._reg_kld = tf.keras.losses.KLDivergence()
    self._kld_no_reduce = tf.keras.losses.KLDivergence(
        reduction=tf.keras.losses.Reduction.NONE)
    self._from_logits = from_logits

    self._head_acc_metric = tf.keras.metrics.Mean(name='head_sel_acc')
    self._gt_g0_ratio = tf.keras.metrics.Mean(name='gt_g0_ratio')

    self._metrics = []
    self._metric_constructors = []
    if metrics:
      has_keras_metric = False
      has_keras_metric_constructor = False

      for metric in metrics:
        if isinstance(metric, tf.keras.metrics.Metric):
          self._metrics.append(metric)
          has_keras_metric = True
        elif callable(metric):
          constructed_metric = metric()
          if not isinstance(constructed_metric, tf.keras.metrics.Metric):
            raise TypeError(
                f'Metric constructor {metric} is not a no-arg callable that '
                'creates a `tf.keras.metrics.Metric`.')
          self._metric_constructors.append(metric)
          self._metrics.append(constructed_metric)
          has_keras_metric_constructor = True
        else:
          raise TypeError(
              'Expected the input metric to be either a '
              '`tf.keras.metrics.Metric` or a no-arg callable that constructs '
              'a `tf.keras.metrics.Metric`, found a non-callable '
              f'{tff.types.py_typecheck.type_string(type(metric))}.')

      if has_keras_metric and has_keras_metric_constructor:
        raise TypeError(
            'Expected the input `metrics` to be either a list of '
            '`tf.keras.metrics.Metric` objects or a list of no-arg callables '
            'that each constructs a `tf.keras.metrics.Metric`, '
            f'found both types in the `metrics`: {metrics}.')

    # This is defined here so that it closes over the `loss_fn`.
    class _WeightedMeanLossMetric(tf.keras.metrics.Mean):
      """A `tf.keras.metrics.Metric` wrapper for the loss function."""

      def __init__(self, name='loss', dtype=tf.float32):
        super().__init__(name, dtype)
        self._loss_fns = loss_fns
        self._loss_weights = loss_weights

      def update_state(self, y_true, y_pred, sample_weight=None):  # pytype: disable=signature-mismatch
        if isinstance(y_pred, list):
          batch_size = tf.shape(y_pred[0])[0]
        else:
          batch_size = tf.shape(y_pred)[0]

        if len(self._loss_fns) == 1:
          batch_loss = self._loss_fns[0](y_true, y_pred)
        else:
          batch_loss = tf.zeros(())
          for i in range(len(self._loss_fns)):
            batch_loss += self._loss_weights[i] * self._loss_fns[i](y_true[i],
                                                                    y_pred[i])

        return super().update_state(batch_loss, batch_size)

    self._metrics.append(_WeightedMeanLossMetric())
    if not metrics or self._metric_constructors:
      self._metric_constructors.append(_WeightedMeanLossMetric)

  @property
  def trainable_variables(self):
    return self._keras_model.trainable_variables

  @property
  def non_trainable_variables(self):
    return self._keras_model.non_trainable_variables

  @property
  def local_variables(self):
    local_variables = []
    for metric in self.get_metrics():
      local_variables.extend(metric.variables)  # pytype: disable=attribute-error
    return local_variables

  def get_metrics(self):
    return self._metrics + [self._head_acc_metric, self._gt_g0_ratio]

  @property
  def input_spec(self):
    return self._input_spec

  @tf.function
  def predict_on_batch(self, x, training=True):
    return self._keras_model(x, training=training)

  def _forward_training(
      self,
      y_true,
      pred_1,
      pred_2,
      groups,
      main_head,
  ):
    # During training, we select the branch by first running inference on
    # training data and collect the votes.
    assert main_head is not None

    loss_fn = self._loss_fns[0]

    pred_3 = tf.cond(
        tf.math.equal(main_head, 0), lambda: pred_1, lambda: pred_2)

    # Compute the label smoothing regularizations for the branches.
    reg1 = kld_no_reduce(
        pred_1,
        self._kld_no_reduce,
        from_logits=True,
        labels=y_true,
        smoothing=self._label_smoothing)
    reg2 = kld_no_reduce(
        pred_2,
        self._kld_no_reduce,
        from_logits=True,
        labels=y_true,
        smoothing=self._label_smoothing)

    reg3 = tf.cond(
        tf.math.equal(main_head, 0), lambda: tf.reduce_mean(reg2),
        lambda: tf.reduce_mean(reg1))

    # Compute the head selection accuracy.
    head_correct = tf.cast(
        tf.equal(1. - tf.cast(main_head, tf.float32), groups), tf.float32)

    self._head_acc_metric.update_state(head_correct)

    loss3 = loss_fn(y_true=y_true, y_pred=pred_3)

    batch_loss = loss3 + self._uniform_reg * reg3
    predictions = pred_3
    return predictions, batch_loss

  def _forward_eval(self, y_true, pred_1, pred_2, groups, kmeans_centers,
                    features, dist_scalar):
    assert kmeans_centers is not None
    loss_fn = self._loss_fns[0]

    # We use k-means to select the heads during evaluation.
    n_centers = tf.shape(kmeans_centers)[0]
    features = tf.expand_dims(features, axis=1)
    kmeans_centers = tf.expand_dims(kmeans_centers, axis=0)
    dists = tf.norm(features - kmeans_centers, axis=-1)
    dists = tf.concat([dists[:, :1] * dist_scalar, dists[:, 1:]], axis=1)

    sel_idxes = tf.expand_dims(tf.argmin(dists, axis=1), axis=1)
    sel_head1 = tf.cast(
        tf.math.less(sel_idxes, tf.cast(n_centers // 2, dtype=sel_idxes.dtype)),
        dtype=tf.float32)

    if self._batch_majority_voting:
      # TODO(b/193904908): the splitting should be done in dataset
      # creation and preprocessing.
      # We assume each client to come from one distribution, and hence we
      # split the clients in the original dataset in evaluation. The original
      # clients have data from distribution 1 & 2 (marked by groups), and two
      # "virtual" clients are created from a single client: client 1 which
      # has only data from distribution 1, and client 2 has only data from
      # distribution 2. The majority voting are done for client 1 & 2
      # separately.
      gt_1 = tf.cast(
          tf.math.equal(groups, tf.constant(0, dtype=groups.dtype)), tf.float32)
      gt_1 = tf.expand_dims(gt_1, axis=1)

      # Votes for client 1.
      votes_1_1 = tf.reduce_sum(sel_head1 * gt_1)
      votes_1_2 = tf.reduce_sum((1.0 - sel_head1) * gt_1)
      if tf.math.greater(votes_1_1, votes_1_2):
        group_1 = tf.ones_like(sel_head1) * gt_1
      else:
        group_1 = tf.zeros_like(sel_head1)

      # Votes for group 2.
      votes_2_1 = tf.reduce_sum(sel_head1 * (1. - gt_1))
      votes_2_2 = tf.reduce_sum((1.0 - sel_head1) * (1. - gt_1))
      if tf.math.greater(votes_2_1, votes_2_2):
        group_2 = tf.ones_like(sel_head1) * (1 - gt_1)
      else:
        group_2 = tf.zeros_like(sel_head1)
      sel_head1 = group_1 + group_2

    head_correct = tf.cast(
        tf.math.equal(tf.cast(1. - sel_head1[:, 0], groups.dtype), groups),
        tf.float32)
    predictions = sel_head1 * pred_1 + (1. - sel_head1) * pred_2

    self._head_acc_metric.update_state(head_correct)
    batch_loss = loss_fn(y_true=y_true, y_pred=predictions)
    return predictions, batch_loss

  def _forward_pass(self,
                    batch_input,
                    training=True,
                    get_head_scores=False,
                    main_branch=None,
                    kmeans_centers=None,
                    dist_scalar=1.):
    if hasattr(batch_input, '_asdict'):
      batch_input = batch_input._asdict()
    if isinstance(batch_input, collections.abc.Mapping):
      inputs = batch_input.get('x')
    else:
      inputs = batch_input[0]
    if inputs is None:
      raise KeyError('Received a batch_input that is missing required key `x`. '
                     'Instead have keys {}'.format(list(batch_input.keys())))

    pred_1, pred_2, features = self.predict_on_batch(inputs, training)

    if isinstance(batch_input, collections.abc.Mapping):
      y_true = batch_input.get('y')
    else:
      y_true = batch_input[1]

    if get_head_scores:
      return BatchOutput(
          loss=None,
          predictions=None,
          num_examples=tf.shape(features)[0],
          features=features)

    if training:
      predictions, batch_loss = self._forward_training(y_true, pred_1, pred_2,
                                                       inputs[1], main_branch)
    else:
      predictions, batch_loss = self._forward_eval(y_true, pred_1, pred_2,
                                                   inputs[1], kmeans_centers,
                                                   features, dist_scalar)

    for metric in self.get_metrics()[:-2]:
      metric.update_state(y_true=y_true, y_pred=predictions)  # pytype: disable=attribute-error
    self._gt_g0_ratio.update_state(inputs[1])

    return tff.learning.model.BatchOutput(
        loss=batch_loss,
        predictions=predictions,
        num_examples=tf.shape(tf.nest.flatten(inputs)[0])[0],
    )

  @tf.function
  def forward_pass(self,
                   batch_input,
                   training=True,
                   get_head_scores=False,
                   main_branch=None,
                   kmeans_centers=None,
                   dist_scalar=1.):
    return self._forward_pass(
        batch_input,
        training=training,
        get_head_scores=get_head_scores,
        main_branch=main_branch,
        kmeans_centers=kmeans_centers,
        dist_scalar=dist_scalar)

  @tf.function
  def report_local_unfinalized_metrics(
      self) -> OrderedDict[str, List[tf.Tensor]]:
    """Creates an `OrderedDict` of metric names to unfinalized values.

    Returns:
      An `OrderedDict` of metric names to lists of unfinalized metric values.
      For a Keras metric, its unfinalized values are the tensor values of its
      variables tracked during local training. The returned `OrderedDict` has
      the same keys (metric names) as the `OrderedDict` returned by the method
      `metric_finalizers()`, and can be used as input to the finalizers to get
      the finalized metric values. This method and the `metric_finalizers()`
      method can be used to construct a cross-client metrics aggregator when
      defining the federated training processes or evaluation computations.
    """
    outputs = collections.OrderedDict()
    for metric in self.get_metrics():
      outputs[metric.name] = [v.read_value() for v in metric.variables]
    return outputs

  def metric_finalizers(
      self
  ) -> OrderedDict[str, tff.learning.metrics.finalizer.KerasMetricFinalizer]:
    """Creates an `OrderedDict` of metric names to finalizers.

    Returns:
      An `OrderedDict` of metric names to finalizers. A finalizer of a Keras
      metric is a `tf.function` decorated callable that takes in this metric's
      unfinalized values (created by `report_local_unfinalized_metrics`), and
      returns the metric value computed by `tf.keras.metrics.Metric.result()`.
      This method and the `report_local_unfinalized_metrics` method can be used
      to construct a cross-client metrics aggregator when defining the federated
      training processes or evaluation computations.
    """
    finalizers = collections.OrderedDict()
    for metric in self.get_metrics():
      finalizers[metric.name] = (
          tff.learning.metrics.finalizer.create_keras_metric_finalizer(metric))  # pytype: disable=attribute-error
    return finalizers

  @tf.function
  def reset_metrics(self):
    """Resets metrics variables to initial value."""
    raise NotImplementedError(
        'The `reset_metrics` method isn\'t implemented for your custom '
        '`tff.learning.Model`. Please implement it before using this method. '
        'You can leave this method unimplemented if you won\'t use this method.'
    )


from dotenv import load_dotenv
load_dotenv()

from ciphers import encrypt_aes, decrypt_aes, decrypt_json
import base64

from db import connect, migrate, connection, insert_key, get_key, delete_key

if connect() is False:
    print('Database failed to connect')
else:
    migrate()

from flask import Flask, render_template, request, jsonify
app = Flask(__name__)

@app.route('/', methods=['GET'])
def show_form():
    return render_template("index.html")

@app.route('/key', methods=['GET'])
def create_keys():
    from ciphers import create_rsa_key_pair
    private_key, public_key = create_rsa_key_pair()

    # Store private key
    key_id = insert_key(private_key)

    return jsonify(
        id=key_id,
        public_key=public_key
    )

@app.route('/decrypt', methods=['POST'])
def decrypt_message():
    json = request.get_json()

    # Retrieve private key
    private_key = get_key(json['id'])

    # Decrypt payload
    message = decrypt_json(private_key, json['message'])

    # Delete private key
    delete_key(json['id'])

    return jsonify(
        message=message.decode("utf-8"),
    )

if __name__ == "__main__":
    app.run(debug=True)


from HAC import HACClustering
import numpy as np
from arff import Arff

filepath = 'iris.arff'
af = Arff()
af.load_arff(filepath)

alldata = af.data
split = int(.75 * len(alldata))
shuffle = False
if shuffle:
    np.random.shuffle(alldata)
trainingdata = alldata[:split]
testdata = alldata[:,:]
hac = HACClustering(k=2, link_type='single', print_k=[2, 3, 4, 5, 6, 7])
hac.fit(testdata)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Oct  2 22:09:40 2021

@author: richard
"""
import sys, os
import json
import datetime as dt
import pprint

import pytest

import rspace_client.tests.base_test as base
from rspace_client.inv import inv, template_builder, sample_builder2
from rspace_client.inv import quantity_unit as qu

class InventoryApiTest(base.BaseApiTest):
    def setUp(self):
        """
        Skips tests if aPI client credentials are missing
        """
        self.assertClientCredentials()
        self.invapi = inv.InventoryClient(self.rspace_url, self.rspace_apikey)

    def test_create_sample(self):
        sample_name = base.random_string(5)
        sample_tags = base.random_string(4)
        ef1 = inv.ExtraField("f1", inv.ExtraFieldType.TEXT, "hello")
        ef2 = inv.ExtraField("f2", inv.ExtraFieldType.NUMBER, 123)

        minTemp = inv.StorageTemperature(1, inv.TemperatureUnit.KELVIN)
        maxTemp = inv.StorageTemperature(4, inv.TemperatureUnit.KELVIN)
        expiry_date = dt.date(2024, 12, 25)
        amount = inv.Quantity(5, qu.QuantityUnit.of("ml"))
        sample = self.invapi.create_sample(
            name=sample_name,
            tags=sample_tags,
            extra_fields=[ef1, ef2],
            storage_temperature_min=minTemp,
            storage_temperature_max=maxTemp,
            expiry_date=expiry_date,
            total_quantity=amount,
            subsample_count=12,
            attachments=[open(base.get_any_datafile(), "rb")],
        )
        self.assertEqual(sample_name, sample["name"])
        self.assertEqual(2, len(sample["extraFields"]))
        self.assertEqual(4, sample["storageTempMax"]["numericValue"])
        self.assertEqual(1, sample["storageTempMin"]["numericValue"])
        self.assertEqual(12, sample["subSamplesCount"])
        self.assertEqual(expiry_date.isoformat(), sample["expiryDate"])
        self.assertEqual(1, len(sample["attachments"]))
        s_ob = inv.Sample(sample)

    def test_set_image_sample(self):
        sample = self.invapi.create_sample(base.random_string(5))
        file = base.get_datafile("AntibodySample150.png")
        with open(file, "rb") as f:
            updated_sample = self.invapi.set_image(sample, f)

        links = updated_sample["_links"]
        for l in links:
            if l["rel"] == "image":
                self.assertFalse(l["link"].endswith("-1"))

    def test_set_image_container(self):
        sample = self.invapi.create_list_container(base.random_string(5))
        file = base.get_datafile("freezer.jpg")
        with open(file, "rb") as f:
            updated_c = self.invapi.set_image(sample, f)

        links = updated_c["_links"]
        for l in links:
            if l["rel"] == "image":
                self.assertFalse(l["link"].endswith("-1"))

    def test_create_sample_name_only(self):
        sample = self.invapi.create_sample(base.random_string(5))
        self.assertIsNotNone(sample)

    def test_create_bulk_sample(self):
        s1 = inv.SamplePost("a")
        s2 = inv.SamplePost("b")
        result = self.invapi.bulk_create_sample(s1, s2)
        self.assertTrue(result.is_ok())
        self.assertEqual(2, len(result.data["results"]))

    def test_get_single_sample(self):
        sample = self.invapi.create_sample(base.random_string(5))
        sample2 = self.invapi.get_sample_by_id(sample["id"])
        sample3 = self.invapi.get_sample_by_id(sample["globalId"])
        self.assertEqual(sample["name"], sample2["name"])
        self.assertEqual(sample["name"], sample3["name"])

    def test_upload_file(self):
        sample = self.invapi.create_sample(base.random_string())
        data_file = base.get_any_datafile()
        with open(data_file, "rb") as f:
            resp = self.invapi.upload_attachment(sample["globalId"], f)
            self.assertIsNotNone(resp["id"])
            self.assertTrue(resp["globalId"][0:2] == "IF")

    def test_rename_item(self):
        sample = self.invapi.create_sample(base.random_string(5))
        new_name = base.random_string()
        updated = self.invapi.rename(sample["globalId"], new_name)
        self.assertEqual(new_name, updated["name"])

    def test_list_samples(self):
        samples = self.invapi.list_samples(inv.Pagination(sort_order="desc"))

        self.assertEqual(0, samples["pageNumber"])
        self.assertEqual(10, len(samples["samples"]))

    def test_add_note_to_subsample(self):
        note = " a note about a subsample " + base.random_string()
        sample = self.invapi.create_sample("with_note", subsample_count=1)
        ss = sample["subSamples"][0]
        updated = self.invapi.add_note_to_subsample(ss, note)
        self.assertEqual(1, len(updated["notes"]))
        self.assertEqual(note, updated["notes"][0]["content"])

    def test_paginated_samples(self):
        pag = inv.Pagination(page_number=1, page_size=1, sort_order="desc")
        samples = self.invapi.list_samples(pag)
        self.assertEqual(1, samples["pageNumber"])
        self.assertEqual(1, len(samples["samples"]))

    def test_paginated_containers(self):
        pag = inv.Pagination(page_number=0, page_size=1)
        name = base.random_string()
        c = self.invapi.create_list_container(name)
        c = self.invapi.set_as_top_level_container(c)
        containers = self.invapi.list_top_level_containers(pag)
        self.assertEqual(0, containers["pageNumber"])
        self.assertEqual(1, len(containers["containers"]))

    def test_paginated_subsamples(self):
        pag = inv.Pagination(page_number=0, page_size=1)
        ss = self.invapi.list_subsamples(pag)
        self.assertEqual(0, ss["pageNumber"])
        self.assertEqual(1, len(ss["subSamples"]))

    def test_stream_containers(self):
        pag = inv.Pagination(
            page_number=0, page_size=1, order_by="creationDate", sort_order="desc"
        )
        name_gen = base.random_string_gen()
        c1 = self.invapi.create_list_container(next(name_gen))
        c2 = self.invapi.create_list_container(next(name_gen))
        self.invapi.set_as_top_level_container(c1)
        self.invapi.set_as_top_level_container(c2)

        gen = self.invapi.stream_top_level_containers(pag)
        # get 2 items
        c2_l = next(gen)
        c1_l = next(gen)
        self.assertEqual(c1["id"], c1_l["id"])
        self.assertEqual(c2["id"], c2_l["id"])

    def test_stream_samples(self):
        onePerPage = inv.Pagination(page_size=1)
        gen = self.invapi.stream_samples(onePerPage)
        # get 2 items
        s1 = next(gen)
        s2 = next(gen)
        self.assertNotEqual(s1["id"], s2["id"])

        unknown_user = base.random_string(15)
        sample_filter = inv.SearchFilter(owned_by=unknown_user)
        gen = self.invapi.stream_samples(onePerPage, sample_filter)
        result_count = 0
        for sample in gen:
            result_count += 1
        self.assertEqual(0, result_count)

    def test_create_sample_withextra_fields(self):
        ef1 = inv.ExtraField(name="ef1", content="createdf1 content")
        sample = self.invapi.create_sample(base.random_string(), extra_fields=[ef1])
        updatedS = self.invapi.get_sample_by_id(sample["globalId"])
        self.assertEqual(1, len(updatedS["extraFields"]))

    def test_bulkcreate_sample_withextra_fields(self):
        ## TODO Fix
        ef1 = inv.ExtraField(
            name="ef1", fieldType=inv.ExtraFieldType.TEXT, content="createdf1 content"
        )
        sample_post = inv.SamplePost(base.random_string(), extra_fields=[ef1])
        sample = self.invapi.bulk_create_sample(sample_post).success_results()[0][
            "record"
        ]
        self.assertEqual(1, len(sample["extraFields"]))
        ## this works: {

    def test_add_extra_fields(self):
        sample = self.invapi.create_sample(base.random_string())
        ef1 = inv.ExtraField(name="ef1", content="ef1 content")
        ef2 = inv.ExtraField(name="ef2", content="ef2 content")
        self.assertEqual("ExtraField ('ef2', 'text','ef2 content')", repr(ef2))
        updatedS = self.invapi.add_extra_fields(sample["globalId"], ef1, ef2)
        self.assertEqual(2, len(updatedS["extraFields"]))

        container = self.invapi.create_list_container("any list container")
        updatedC = self.invapi.add_extra_fields(container, ef2)
        self.assertEqual(1, len(updatedC["extraFields"]))

    def test_search_sample(self):
        name = base.random_string()
        tags = base.random_string()
        sample = self.invapi.create_sample(name, tags=tags)
        results = self.invapi.search(query=name)
        ## sample and its subsample
        self.assertEqual(2, results["totalHits"])
        results_from_tag = self.invapi.search(query=tags)
        self.assertEqual(1, results_from_tag["totalHits"])

        results2 = self.invapi.search(query=name, result_type=inv.ResultType.SAMPLE)
        self.assertEqual(1, results2["totalHits"])
        results3 = self.invapi.search(query=name, result_type=inv.ResultType.SUBSAMPLE)
        self.assertEqual(1, results3["totalHits"])
        results4 = self.invapi.search(query=name, result_type=inv.ResultType.CONTAINER)
        self.assertEqual(0, results4["totalHits"])

    def test_equal_split(self):
        name = base.random_string()
        sample = self.invapi.create_sample(name)
        ss = sample["subSamples"][0]
        splits = self.invapi.split_subsample(ss, num_new_subsamples=3)
        self.assertEqual(3, len(splits))
        self.assertAlmostEqual(
            750, sum([x["quantity"]["numericValue"] for x in splits])
        )

    def test_partial_split(self):
        name = base.random_string()
        sample_total = inv.Quantity(5, qu.QuantityUnit.of("ml"))
        ## create 1 sample with 1 ss of 5ml
        sample = self.invapi.create_sample(name, total_quantity=sample_total)
        ss = sample["subSamples"][0]
        bulk_result = self.invapi.split_subsample(
            ss, num_new_subsamples=3, quantity_per_subsample=1.2
        )

        self.assertTrue(bulk_result.is_ok())
        results = bulk_result.data["results"]

        ## should result in original with (5 - (3 *1.2) = 1.4, new ones with 1.2
        ss_to_amount = {}
        for updated_ss in results:
            ss_to_amount[updated_ss["record"]["id"]] = updated_ss["record"]["quantity"][
                "numericValue"
            ]

        self.assertAlmostEqual(1.4, ss_to_amount[ss["id"]])
        ss_to_amount.pop(ss["id"])
        for v in ss_to_amount.values():
            self.assertAlmostEqual(1.2, v)

    def test_partial_split_rejects_impossible_quantity(self):
        ## create 1 sample with 1 ss of 5ml
        ss = {
            "id": 1234,
            "globalId": "SS1234",
            "quantity": {"unitId": 3, "numericValue": 5.0},
        }
        self.assertRaises(ValueError, self.invapi.split_subsample, ss, 10, 0.51)

    def test_duplicate(self):
        name = base.random_string()

        container = self.invapi.create_list_container("c_to_dup")
        container_dup = self.invapi.duplicate(container, f"{name}-newfromtest")
        self.assertNotEqual(container["id"], container_dup["id"])
        self.assertEqual(f"{name}-newfromtest", container_dup["name"])

        sample = self.invapi.create_sample(name)
        sample_dup = self.invapi.duplicate(sample)
        self.assertNotEqual(sample["id"], sample_dup["id"])

        ss = sample["subSamples"][0]
        ss_dup = self.invapi.duplicate(ss)
        self.assertNotEqual(ss["id"], ss_dup["id"])

    def test_get_benches(self):
        benches = self.invapi.get_workbenches()
        self.assertEqual(1, len(benches))
        bench_ob = inv.Container.of(benches[0])
        self.assertTrue(bench_ob.is_workbench())

    def test_bulk_create_top_level_containers(self):
        list_post = inv.ListContainerPost("list1", description="xxx")
        grid_post = inv.GridContainerPost("grid1", 2, 4, description="xxx")
        results = self.invapi.bulk_create_container(list_post, grid_post)
        self.assertTrue(results.is_ok())
        self.assertEqual(2, len(results.data["results"]))

    def test_bulk_create_containers_in_bench(self):
        list_post = inv.ListContainerPost(
            "list1", description="xxx", location=inv.BenchTargetLocation()
        )
        grid_post = inv.GridContainerPost(
            "grid1", 2, 4, description="xxx", location=inv.BenchTargetLocation()
        )
        results = self.invapi.bulk_create_container(list_post, grid_post)
        self.assertTrue(results.is_ok())
        self.assertEqual(2, len(results.data["results"]))

        ids = [
            x["record"]["parentContainers"][0]["globalId"]
            for x in results.success_results()
        ]
        self.assertTrue(all(inv.Id(x).is_bench() for x in ids))

    def test_bulk_create_containers_in_list(self):
        target_c = self.invapi.create_list_container("top-level")

        grid_posts = [
            inv.GridContainerPost(
                f"grid-{i}",
                2,
                4,
                description="grid",
                location=inv.ListContainerTargetLocation(target_c["id"]),
            )
            for i in range(3)
        ]
        list_posts = [
            inv.ListContainerPost(
                f"list-{i}",
                description="list",
                location=inv.ListContainerTargetLocation(target_c["id"]),
            )
            for i in range(3)
        ]
        list_posts.extend(grid_posts)
        results = self.invapi.bulk_create_container(*list_posts)
        self.assertTrue(results.is_ok())
        self.assertEqual(6, len(results.data["results"]))
        ids = [
            x["record"]["parentContainers"][0]["globalId"]
            for x in results.success_results()
        ]
        self.assertTrue(all(parent_id == target_c["globalId"] for parent_id in ids))

    def test_bulk_create_containers_in_grid(self):
        num_items = 5
        grid = self.invapi.create_grid_container(
            "grid1", 1, num_items, description="xxx"
        )
        ## Create 5 list containers and add them to cells in the grid.
        list_posts = [
            inv.ListContainerPost(
                f"c-{i}", location=inv.GridContainerTargetLocation(grid["id"], i + 1, 1)
            )
            for i in range(num_items)
        ]

        results = self.invapi.bulk_create_container(*list_posts)
        self.assertTrue(results.is_ok())
        self.assertEqual(5, len(results.data["results"]))
        for list_c in results.data["results"]:
            self.assertEqual(grid["id"], list_c["record"]["parentContainers"][0]["id"])

    def test_container_must_store_samples_or_containers(self):
        self.assertRaises(
            ValueError,
            self.invapi.create_list_container,
            "n",
            "",
            "",
            [],
            False,
            False,
            "t",
        )

    def test_grid_container_must_store_samples_or_containers(self):
        self.assertRaises(
            ValueError,
            self.invapi.create_grid_container,
            "n",
            1,
            2,
            "",
            "",
            [],
            False,
            False,
            "t",
        )

    def test_create_list_container(self):
        name = base.random_string()
        ct = self.invapi.create_list_container(
            name, tags="ab,cd,ef", can_store_samples=False
        )
        ## default is top-level
        self.assertTrue(ct["cType"] == "LIST")
        self.assertEqual(0, len(ct["parentContainers"]))
        self.assertFalse(ct["canStoreSamples"])

        ## create in Bench
        ct_in_wb = self.invapi.create_list_container(
            name, tags="ab,cd,ef", location=inv.BenchTargetLocation()
        )
        self.assertEqual("BE", ct_in_wb["parentContainers"][0]["globalId"][0:2])

        ## create in parent list container
        ct_sub_container = self.invapi.create_list_container(
            name, tags="ab,cd,ef", location=inv.TargetLocation(ct["id"])
        )
        self.assertEqual(
            ct["globalId"], ct_sub_container["parentContainers"][0]["globalId"]
        )

    def _createImageContainerPost(self):
        image_file = base.get_datafile("freezer.jpg")
        loci = [(180, 300), (740, 350), (520, 300)]
        image_post = inv.ImageContainerPost("freezer", image_file, loci)
        return image_post

    def test_create_image_container_from_reader(self):
        image_file = base.get_datafile("freezer.jpg")
        loci = [(180, 300), (740, 350), (520, 300)]
        with open(image_file, "rb") as b:
            image_post = inv.ImageContainerPost("freezer", b, loci)
        c = self.invapi.create_image_container(image_post)
        self.assertIsNotNone(c["id"])
        self.assertEqual(3, len(c["locations"]))

    def test_create_image_container(self):
        image_post = self._createImageContainerPost()
        c = self.invapi.create_image_container(image_post)
        self.assertIsNotNone(c["id"])
        self.assertEqual(3, len(c["locations"]))

        image_c_object = inv.ImageContainer(c)
        self.assertTrue(image_c_object.is_image())
        self.assertEqual(3, image_c_object.capacity())
        self.assertEqual(0, image_c_object.used_locations())
        self.assertEqual(3, image_c_object.free_locations())

    def test_add_new_image_locations_to_image_container(self):
        image_post = self._createImageContainerPost()
        image_c = self.invapi.create_image_container(image_post)
        initial_location_count = len(image_c["locations"])
        loci = [(80, 800), (740, 450), (520, 400)]
        updated = self.invapi.add_locations_to_image_container(image_c, *loci)
        self.assertEqual(initial_location_count + 3, len(updated["locations"]))

    def test_delete_new_image_locations_from_image_container(self):
        image_post = self._createImageContainerPost()
        image_c = self.invapi.create_image_container(image_post)
        image_c_object = inv.ImageContainer(image_c)
        self.assertEqual(3, image_c_object.capacity())
        to_delete = [x["id"] for x in image_c["locations"]]
        updated = self.invapi.delete_locations_from_image_container(image_c, *to_delete)
        image_c_object = inv.ImageContainer(updated)
        self.assertEqual(0, image_c_object.capacity())

    def test_create_container_in_image_container(self):
        image_post = self._createImageContainerPost()
        image_c = self.invapi.create_image_container(image_post)
        first_location = image_c["locations"][0]["id"]
        sub_container = self.invapi.create_list_container(
            "list",
            location=inv.ImageContainerTargetLocation(image_c["id"], first_location),
        )

        self.assertEqual(image_c["id"], sub_container["parentContainers"][0]["id"])

    def test_add_items_to_image_container(self):
        image_post = self._createImageContainerPost()
        image_c = self.invapi.create_image_container(image_post)
        loci = [l["id"] for l in image_c["locations"]]
        sample = self.invapi.create_sample("in-image", subsample_count=3)
        result = self.invapi.add_items_to_image_container(
            image_c, sample["subSamples"], loci
        )
        self.assertTrue(result.is_ok())
        self.assertEqual(3, len(result.success_results()))
        ## reload:
        image_c = inv.ImageContainer(self.invapi.get_container_by_id(image_c))
        self.assertEqual(0, image_c.free_locations())
        self.assertEqual(3, image_c.capacity())

    def test_move_container_to_list_container(self):
        name = base.random_string() + "_to_move"
        toMove = self.invapi.create_list_container(name)
        name_target = base.random_string() + "_target"
        target = self.invapi.create_list_container(name_target)
        moved = self.invapi.add_items_to_list_container(
            target["id"], toMove["globalId"]
        )
        self.assertTrue(moved.is_ok())

    def test_move_subsamples_to_list_container(self):
        name = base.random_string() + "_to_move"
        toMove = self.invapi.create_sample(name, subsample_count=2)
        name_target = base.random_string() + "_target"
        target = self.invapi.create_list_container(name_target)

        ## get the 2 subsample ids ad move to container
        subsample_ids = [ss["globalId"] for ss in toMove["subSamples"]]
        moved = self.invapi.add_items_to_list_container(target["id"], *subsample_ids)
        self.assertTrue(moved.is_ok())

    def test_move_single_item_to_grid(self):
        grid_c = self.invapi.create_grid_container("gridX", 3, 2)
        sample = self.invapi.create_sample(name="multiS", subsample_count=1)
        self.invapi.add_items_to_grid_container(
            grid_c, inv.ByRow(2, 1, 2, 3, sample["subSamples"][0]["globalId"])
        )

        ## now reload the container, which should show subsamples
        updated_container_json = self.invapi.get_container_by_id(grid_c["id"])
        container = inv.Container.of(updated_container_json)
        self.assertEqual(6, container.capacity())
        self.assertEqual(5, container.free())
        self.assertEqual(1, container.in_use())
        self.assertTrue((2, 1) in container.used_locations())

    def test_move_container_and_subsample_to_grid(self):
        grid_c = self.invapi.create_grid_container("gridX", 3, 2)
        sample = self.invapi.create_sample(name="multiS", subsample_count=1)
        other_container = self.invapi.create_list_container("toMove")
        ids_to_move = [sample["subSamples"][0]["globalId"], other_container["globalId"]]
        result = self.invapi.add_items_to_grid_container(
            grid_c, inv.ByRow(1, 1, 3, 2, *ids_to_move)
        )
        self.assertTrue(result.is_ok())

    def test_cannot_move_too_many_items_b4_request(self):
        rows = 2
        cols = 2
        grid_c = self.invapi.create_grid_container("gridX", rows, cols)
        sample = self.invapi.create_sample(
            name="toomany", subsample_count=rows * cols + 1
        )

        ss_ids = [x["globalId"] for x in sample["subSamples"]]
        ## using GridContainer object, we can check capacity before requesting
        self.assertRaises(
            ValueError,
            self.invapi.add_items_to_grid_container,
            inv.GridContainer(grid_c),
            inv.ByRow(1, 1, 2, 2, *ss_ids),
        )

    def test_place_by_location(self):
        grid_c = self.invapi.create_grid_container("gridExact", 2, 5)
        sample = self.invapi.create_sample(
            name="placing_in_container", subsample_count=5
        )
        ids_to_move = [x["globalId"] for x in sample["subSamples"]]
        ## place items in the top row
        locations = [inv.GridLocation(x + 1, 1) for x in range(5)]
        result = self.invapi.add_items_to_grid_container(
            grid_c, inv.ByLocation(locations, *ids_to_move)
        )
        self.assertTrue(result.is_ok())
        ## get container
        updated_container_json = self.invapi.get_container_by_id(grid_c["id"])
        container = inv.Container.of(updated_container_json)
        self.assertEqual(10, container.capacity())
        self.assertEqual(5, container.in_use())

    def test_cannot_move_too_many_items_with_request(self):
        grid_c = self.invapi.create_grid_container("gridX", 1, 2)
        sample = self.invapi.create_sample(name="toomany", subsample_count=4)

        ss_ids = [x["globalId"] for x in sample["subSamples"]]
        ## using just id, we try to make request anyway
        resp = self.invapi.add_items_to_grid_container(
            grid_c["id"], inv.ByRow(1, 1, 2, 1, *ss_ids[0:2])
        )
        self.assertTrue(resp.is_ok())
        ## overwrite, what happens
        resp2 = self.invapi.add_items_to_grid_container(
            grid_c["id"], inv.ByRow(1, 1, 2, 1, *ss_ids[2:4])
        )

        self.assertFalse(resp2.is_ok())
        self.assertTrue(resp2.is_failed())

    def test_bulk_move_to_grid(self):
        grid_c = self.invapi.create_grid_container("gridX", 7, 3)
        sample = self.invapi.create_sample(name="multiS", subsample_count=10)
        ss_ids = [x["globalId"] for x in sample["subSamples"]]
        rc = self.invapi.add_items_to_grid_container(
            grid_c, inv.ByColumn(2, 1, 3, 7, *ss_ids)
        )
        ## get list of updated subsamples
        self.assertEqual(10, len(rc.data["results"]))

        ## now reload the container, which should show subsamples
        updated_container_json = self.invapi.get_container_by_id(grid_c["id"])
        container = inv.Container.of(updated_container_json)
        self.assertEqual(21, container.capacity())
        self.assertEqual(11, container.free())
        self.assertEqual(10, container.in_use())

    def test_calculate_grid_start(self):
        total_cols = 5
        total_rows = 4
        self.assertEqual(
            7,
            inv._calculate_start_index(
                3, 2, total_cols, total_rows, inv.FillingStrategy.BY_ROW
            ),
        )
        self.assertEqual(
            9,
            inv._calculate_start_index(
                3, 2, total_cols, total_rows, inv.FillingStrategy.BY_COLUMN
            ),
        )
        self.assertEqual(
            0,
            inv._calculate_start_index(
                1, 1, total_cols, total_rows, inv.FillingStrategy.BY_COLUMN
            ),
        )
        self.assertEqual(
            19,
            inv._calculate_start_index(
                5, 4, total_cols, total_rows, inv.FillingStrategy.BY_COLUMN
            ),
        )
        self.assertEqual(
            19,
            inv._calculate_start_index(
                5, 4, total_cols, total_rows, inv.FillingStrategy.BY_ROW
            ),
        )

    def test_calculate_grid_start_validation(self):
        self.assertRaises(
            ValueError,
            inv._calculate_start_index,
            -1,
            2,
            5,
            4,
            inv.FillingStrategy.BY_ROW,
        )
        self.assertRaises(
            ValueError,
            inv._calculate_start_index,
            -2,
            -1,
            5,
            4,
            inv.FillingStrategy.BY_ROW,
        )

        self.assertRaises(
            ValueError,
            inv._calculate_start_index,
            6,
            2,
            5,
            4,
            inv.FillingStrategy.BY_ROW,
        )

        self.assertRaises(
            ValueError,
            inv._calculate_start_index,
            5,
            5,
            5,
            4,
            inv.FillingStrategy.BY_ROW,
        )

    def test_barcode(self):
        barcode_bytes = self.invapi.barcode("SA14567")
        self.assertEqual(99, len(barcode_bytes))

        qr_bytes = self.invapi.barcode(
            "SA12345", outfile="out10.png", barcode_type=inv.Barcode.QR
        )
        self.assertEqual(293, len(qr_bytes))

    def test_delete_samples(self):
        new_sample = self.invapi.create_sample("to_delete")
        total_samples = self.invapi.list_samples()

        total_samples_count = total_samples["totalHits"]
        total_deleted = self.invapi.list_samples(
            sample_filter=inv.SearchFilter(
                deleted_item_filter=inv.DeletedItemFilter.DELETED_ONLY
            )
        )["totalHits"]
        self.invapi.delete_sample(new_sample["id"])
        total_deleted2 = self.invapi.list_samples(
            sample_filter=inv.SearchFilter(
                deleted_item_filter=inv.DeletedItemFilter.DELETED_ONLY
            )
        )["totalHits"]
        self.assertEqual(total_deleted2, total_deleted + 1)

    def test_create_sample_template(self):
        tb = template_builder.TemplateBuilder("toTest", "ml")
        t_json = tb.text("Notes").number("pH", 7).build()
        st = self.invapi.create_sample_template(t_json)
        self.assertTrue("id" in st)
        self.assertEqual("toTest", st["name"])
        self.assertEqual(2, len(st["fields"]))

    def test_delete_restore_sample_template(self):
        t_json = (
            template_builder.TemplateBuilder("toTest", "ml")
            .text("Notes")
            .number("pH", 7)
            .build()
        )
        st = self.invapi.create_sample_template(t_json)

        self.invapi.delete_sample_template(st["id"])
        restored = self.invapi.restore_sample_template(st["id"])
        self.assertTrue("id" in restored)
        self.assertEqual(2, len(restored["fields"]))

    def test_post_retrieve_sample_template_icon(self):
        t_json = (
            template_builder.TemplateBuilder("WithIcon", "ml")
            .text("Notes")
            .number("pH", 7)
            .build()
        )
        st = self.invapi.create_sample_template(t_json)
        icon_file = base.get_datafile("antibodySample150.png")
        with open(icon_file, "rb") as icon:
            updated_template = self.invapi.set_sample_template_icon(st["id"], icon)
            self.assertTrue(updated_template["iconId"] > 0)
        outfile = "downloaded.png"
        try:
            self.invapi.get_sample_template_icon(
                st["id"], updated_template["iconId"], outfile
            )
            self.assertEqual(1600, os.path.getsize(outfile))
        finally:
            os.remove(os.path.join(os.getcwd(), outfile))

    def test_list_sample_templates(self):
        results = self.invapi.list_sample_templates()
        self.assertTrue(results["totalHits"] > 0)
        self.assertTrue(all([a["template"] for a in results["templates"]]))

        ## search for non-existent user
        sf = inv.SearchFilter(owned_by="XXXX1123")
        results = self.invapi.list_sample_templates(search_filter=sf)
        self.assertEqual(0, results["totalHits"])

    def test_create_sample_from_template(self):

        ## create a new template with different fields
        builder = template_builder.TemplateBuilder("MyEnzyme", "ml")
        st_json = (
            builder.string("comment")
            .number("pH")
            .radio("type", ["Commercial", "Academic"], "Commercial")
            .choice("supplier", ["NEB", "BM", "Sigma"])
            .date("manufacture date")
            .time("manufacture time")
            .uri("website")
            .attachment("Safety Data", "Cosh form pdf")
            .build()
        )
        st = self.invapi.create_sample_template(st_json)

        ## create a template specific object to validate and store field information
        ForSampleCreation = sample_builder2.FieldBuilderGenerator().generate_class(st)
        sample = ForSampleCreation()

        website = "https://mysample.supplier.com"
        ## can set in any order
        sample.supplier = ["NEB"]
        sample.comment = "Some comment"
        sample.ph = 4.7
        sample.manufacture_time = dt.time(11, 20)
        sample.type = "Commercial"
        sample.website = website
        sample.manufacture_date = dt.date(2022, 1, 21)
        sample.safety_data = "A description of this specific PDF"

        fields_to_post = sample.to_field_post()

        created_sample = self.invapi.create_sample(
            name="From MyEnzyme", sample_template_id=st["id"], fields=fields_to_post
        )

        self.assertIsNotNone(created_sample["id"])
        self.assertEqual("Some comment", created_sample["fields"][0]["content"])
        self.assertEqual(4.7, float(created_sample["fields"][1]["content"]))
        self.assertEqual(
            "Commercial", created_sample["fields"][2]["selectedOptions"][0]
        )
        self.assertEqual("NEB", created_sample["fields"][3]["selectedOptions"][0])
        self.assertEqual("2022-01-21", created_sample["fields"][4]["content"])
        self.assertEqual("11:20", created_sample["fields"][5]["content"])
        self.assertEqual(website, created_sample["fields"][6]["content"])

    @pytest.mark.skip(reason="requires test user to be in a group with another user")
    def test_transfer_sample_owner(self):
        sample = self.invapi.create_sample(base.random_string(5))
        updated = self.invapi.transfer_sample_owner(sample["id"], "user2b")
        self.assertEqual("user2b", updated["owner"]["username"])

    @pytest.mark.skip(reason="requires test user to be in a group with another user")
    def test_transfer_sample_template_owner(self):
        st_json = template_builder.TemplateBuilder("MyEnzyme", "ml").build()
        st = self.invapi.create_sample_template(st_json)
        updated = self.invapi.transfer_sample_owner(st["id"], "user2b")
        self.assertEqual("user2b", updated["owner"]["username"])

    def test_rename_template(self):
        st_json = template_builder.TemplateBuilder("MyEnzyme", "ml").build()
        st = self.invapi.create_sample_template(st_json)
        new_name = "MyEnzyme2"
        updated = self.invapi.rename(st["globalId"], new_name)
        self.assertEqual(new_name, updated["name"])

    def test_attach_file_to_attachment_field(self):
        st_json = (
            template_builder.TemplateBuilder("MyEnzyme", "ml")
            .attachment("Safety Data", "Coshh")
            .build()
        )
        st = self.invapi.create_sample_template(st_json)
        created_sample = self.invapi.create_sample(
            name="FromAttachment", sample_template_id=st["id"]
        )
        ## upload a file separately, using the field ID.
        data_file = base.get_any_datafile()
        with open(data_file, "rb") as f:
            self.invapi.upload_attachment(created_sample["fields"][0]["globalId"], f)


import os
import sys
from datetime import datetime

from openvino.tools.benchmark.benchmark import Benchmark
from openvino.tools.benchmark.parameters import parse_args
from openvino.tools.benchmark.utils.constants import MULTI_DEVICE_NAME, HETERO_DEVICE_NAME, CPU_DEVICE_NAME, \
    GPU_DEVICE_NAME, MYRIAD_DEVICE_NAME, GNA_DEVICE_NAME, BLOB_EXTENSION
from openvino.tools.benchmark.utils.inputs_filling import set_inputs
from openvino.tools.benchmark.utils.logging import logger
from openvino.tools.benchmark.utils.progress_bar import ProgressBar
from openvino.tools.benchmark.utils.utils import next_step, config_network_inputs, get_number_iterations, \
    process_help_inference_string, print_perf_counters, dump_exec_graph, get_duration_in_milliseconds, \
    get_command_line_arguments, parse_nstreams_value_per_device, parse_devices, update_shapes, \
    adjust_shapes_batch, load_config, dump_config
from openvino.tools.benchmark.utils.statistics_report import StatisticsReport, averageCntReport, detailedCntReport

def main():
    # ------------------------------ 1. Parsing and validating input arguments -------------------------------------
    next_step()
    run(parse_args())

def run(args):
    statistics = None
    try:
        if args.number_streams is None:
                logger.warn(" -nstreams default value is determined automatically for a device. "
                            "Although the automatic selection usually provides a reasonable performance, "
                            "but it still may be non-optimal for some cases, for more information look at README. ")

        command_line_arguments = get_command_line_arguments(sys.argv)
        if args.report_type:
          statistics = StatisticsReport(StatisticsReport.Config(args.report_type, args.report_folder))
          statistics.add_parameters(StatisticsReport.Category.COMMAND_LINE_PARAMETERS, command_line_arguments)

        def is_flag_set_in_command_line(flag):
            return any(x.strip('-') == flag for x, y in command_line_arguments)

        device_name = args.target_device

        devices = parse_devices(device_name)
        device_number_streams = parse_nstreams_value_per_device(devices, args.number_streams)

        config = {}
        if args.load_config:
            load_config(args.load_config, config)

        is_network_compiled = False
        _, ext = os.path.splitext(args.path_to_model)

        if ext == BLOB_EXTENSION:
            is_network_compiled = True
            print("Network is compiled")

        # ------------------------------ 2. Loading Inference Engine ---------------------------------------------------
        next_step(step_id=2)

        benchmark = Benchmark(args.target_device, args.number_infer_requests,
                              args.number_iterations, args.time, args.api_type)

        ## CPU (MKLDNN) extensions
        if CPU_DEVICE_NAME in device_name and args.path_to_extension:
            benchmark.add_extension(path_to_extension=args.path_to_extension)

        ## GPU (clDNN) Extensions
        if GPU_DEVICE_NAME in device_name and args.path_to_cldnn_config:
            if GPU_DEVICE_NAME not in config.keys():
                config[GPU_DEVICE_NAME] = {}
            config[GPU_DEVICE_NAME]['CONFIG_FILE'] = args.path_to_cldnn_config

        if GPU_DEVICE_NAME in config.keys() and 'CONFIG_FILE' in config[GPU_DEVICE_NAME].keys():
            cldnn_config = config[GPU_DEVICE_NAME]['CONFIG_FILE']
            benchmark.add_extension(path_to_cldnn_config=cldnn_config)

        version = benchmark.get_version_info()

        logger.info(version)

        # --------------------- 3. Setting device configuration --------------------------------------------------------
        next_step()

        perf_counts = False
        for device in devices:
            if device not in config.keys():
                config[device] = {}
            ## Set performance counter
            if is_flag_set_in_command_line('pc'):
                ## set to user defined value
                config[device]['PERF_COUNT'] = 'YES' if args.perf_counts else 'NO'
            elif 'PERF_COUNT' in config[device].keys() and config[device]['PERF_COUNT'] == 'YES':
                logger.warn("Performance counters for {} device is turned on. ".format(device) +
                            "To print results use -pc option.")
            elif args.report_type in [ averageCntReport, detailedCntReport ]:
                logger.warn("Turn on performance counters for {} device ".format(device) +
                            "since report type is {}.".format(args.report_type))
                config[device]['PERF_COUNT'] = 'YES'
            elif args.exec_graph_path is not None:
                logger.warn("Turn on performance counters for {} device ".format(device) +
                            "due to execution graph dumping.")
                config[device]['PERF_COUNT'] = 'YES'
            else:
                ## set to default value
                config[device]['PERF_COUNT'] = 'YES' if args.perf_counts else 'NO'
            perf_counts = True if config[device]['PERF_COUNT'] == 'YES' else perf_counts

            def set_throughput_streams():
                key = device + "_THROUGHPUT_STREAMS"
                if device in device_number_streams.keys():
                    ## set to user defined value
                    supported_config_keys = benchmark.ie.get_metric(device, 'SUPPORTED_CONFIG_KEYS')
                    if key not in supported_config_keys:
                        raise Exception("Device {} doesn't support config key '{}'! ".format(device, key) +
                                        "Please specify -nstreams for correct devices in format  <dev1>:<nstreams1>,<dev2>:<nstreams2>")
                    config[device][key] = device_number_streams[device]
                elif key not in config[device].keys() and args.api_type == "async":
                    logger.warn("-nstreams default value is determined automatically for {} device. ".format(device) +
                                "Although the automatic selection usually provides a reasonable performance,"
                                "but it still may be non-optimal for some cases, for more information look at README.")
                    config[device][key] = device + "_THROUGHPUT_AUTO"
                if key in config[device].keys():
                    device_number_streams[device] = config[device][key]

            if device == CPU_DEVICE_NAME: # CPU supports few special performance-oriented keys
                # limit threading for CPU portion of inference
                if args.number_threads and is_flag_set_in_command_line("nthreads"):
                    config[device]['CPU_THREADS_NUM'] = str(args.number_threads)

                if is_flag_set_in_command_line("enforcebf16") or is_flag_set_in_command_line("enforce_bfloat16"):
                    config[device]['ENFORCE_BF16'] = 'YES' if args.enforce_bfloat16 else 'NO'

                if is_flag_set_in_command_line('pin'):
                    ## set to user defined value
                    config[device]['CPU_BIND_THREAD'] = args.infer_threads_pinning
                elif 'CPU_BIND_THREAD' not in config[device].keys():
                    if MULTI_DEVICE_NAME in device_name and GPU_DEVICE_NAME in device_name:
                        logger.warn("Turn off threads pinning for {}".format(device) +
                                    "device since multi-scenario with GPU device is used.")
                        config[device]['CPU_BIND_THREAD'] = 'NO'
                    else:
                        ## set to default value
                        config[device]['CPU_BIND_THREAD'] = args.infer_threads_pinning

                ## for CPU execution, more throughput-oriented execution via streams
                set_throughput_streams()
            elif device == GPU_DEVICE_NAME:
                ## for GPU execution, more throughput-oriented execution via streams
                set_throughput_streams()

                if MULTI_DEVICE_NAME in device_name and CPU_DEVICE_NAME in device_name:
                    logger.warn("Turn on GPU trottling. Multi-device execution with the CPU + GPU performs best with GPU trottling hint, " +
                                "which releases another CPU thread (that is otherwise used by the GPU driver for active polling)")
                    config[device]['CLDNN_PLUGIN_THROTTLE'] = '1'
            elif device == MYRIAD_DEVICE_NAME:
                config[device]['LOG_LEVEL'] = 'LOG_INFO'
            elif device == GNA_DEVICE_NAME:
                if is_flag_set_in_command_line('qb'):
                    if args.qb == 8:
                        config[device]['GNA_PRECISION'] = 'I8'
                    else:
                        config[device]['GNA_PRECISION'] = 'I16'
                if args.number_threads and is_flag_set_in_command_line("nthreads"):
                    config[device]['GNA_LIB_N_THREADS'] = str(args.number_threads)
        perf_counts = perf_counts

        benchmark.set_config(config)
        batch_size = args.batch_size
        if not is_network_compiled:
            # --------------------- 4. Read the Intermediate Representation of the network -----------------------------
            next_step()

            start_time = datetime.utcnow()
            ie_network = benchmark.read_network(args.path_to_model)
            duration_ms = "{:.2f}".format((datetime.utcnow() - start_time).total_seconds() * 1000)
            logger.info("Read network took {} ms".format(duration_ms))
            if statistics:
                statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                          [
                                              ('read network time (ms)', duration_ms)
                                          ])

            # --------------------- 5. Resizing network to match image sizes and given batch ---------------------------
            next_step()

            shapes = {k: v.input_data.shape.copy() for k, v in ie_network.input_info.items()}
            reshape = False
            if args.shape:
                reshape |= update_shapes(shapes, args.shape, ie_network.input_info)
            if args.batch_size and args.batch_size != ie_network.batch_size:
                reshape |= adjust_shapes_batch(shapes, args.batch_size, ie_network.input_info)

            if reshape:
                start_time = datetime.utcnow()
                logger.info(
                    'Reshaping network: {}'.format(', '.join("'{}': {}".format(k, v) for k, v in shapes.items())))
                ie_network.reshape(shapes)
                duration_ms = "{:.2f}".format((datetime.utcnow() - start_time).total_seconds() * 1000)
                logger.info("Reshape network took {} ms".format(duration_ms))
                if statistics:
                    statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                              [
                                                  ('reshape network time (ms)', duration_ms)
                                              ])

            batch_size = ie_network.batch_size
            logger.info('Network batch size: {}'.format(ie_network.batch_size))

            # --------------------- 6. Configuring input of the model --------------------------------------------------
            next_step()

            config_network_inputs(ie_network)

            # --------------------- 7. Loading the model to the device -------------------------------------------------
            next_step()

            start_time = datetime.utcnow()
            exe_network = benchmark.load_network(ie_network)
            duration_ms = "{:.2f}".format((datetime.utcnow() - start_time).total_seconds() * 1000)
            logger.info("Load network took {} ms".format(duration_ms))
            if statistics:
                statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                          [
                                              ('load network time (ms)', duration_ms)
                                          ])
        else:
            next_step()
            print("Skipping the step for compiled network")
            next_step()
            print("Skipping the step for compiled network")
            next_step()
            print("Skipping the step for compiled network")

            # --------------------- 7. Loading the model to the device -------------------------------------------------
            next_step()

            start_time = datetime.utcnow()
            exe_network = benchmark.import_network(args.path_to_model)
            duration_ms = "{:.2f}".format((datetime.utcnow() - start_time).total_seconds() * 1000)
            logger.info("Import network took {} ms".format(duration_ms))
            if statistics:
                statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                          [
                                              ('import network time (ms)', duration_ms)
                                          ])
            if batch_size == 0:
                batch_size = 1

        # --------------------- 8. Setting optimal runtime parameters --------------------------------------------------
        next_step()

        # Update number of streams
        for device in device_number_streams.keys():
            key = device + '_THROUGHPUT_STREAMS'
            device_number_streams[device] = benchmark.ie.get_config(device, key)

        # Number of requests
        infer_requests = exe_network.requests

        # Iteration limit
        benchmark.niter = get_number_iterations(benchmark.niter, benchmark.nireq, args.api_type)

        # ------------------------------------ 9. Creating infer requests and filling input blobs ----------------------
        next_step()

        paths_to_input = list()
        if args.paths_to_input:
            for path in args.paths_to_input:
                paths_to_input.append(os.path.abspath(*path) if args.paths_to_input else None)
        set_inputs(paths_to_input, batch_size, exe_network.input_info, infer_requests)

        if statistics:
            statistics.add_parameters(StatisticsReport.Category.RUNTIME_CONFIG,
                                      [
                                          ('topology', ie_network.name),
                                          ('target device', device_name),
                                          ('API', args.api_type),
                                          ('precision', "UNSPECIFIED"),
                                          ('batch size', str(batch_size)),
                                          ('number of iterations', str(benchmark.niter) if benchmark.niter else "0"),
                                          ('number of parallel infer requests', str(benchmark.nireq)),
                                          ('duration (ms)', str(get_duration_in_milliseconds(benchmark.duration_seconds))),
                                       ])

            for nstreams in device_number_streams.items():
                statistics.add_parameters(StatisticsReport.Category.RUNTIME_CONFIG,
                                         [
                                            ("number of {} streams".format(nstreams[0]), str(nstreams[1])),
                                         ])

        # ------------------------------------ 10. Measuring performance -----------------------------------------------

        output_string = process_help_inference_string(benchmark)

        next_step(additional_info=output_string)
        progress_bar_total_count = 10000
        if benchmark.niter and not benchmark.duration_seconds:
            progress_bar_total_count = benchmark.niter

        progress_bar = ProgressBar(progress_bar_total_count, args.stream_output, args.progress) if args.progress else None

        duration_ms =  "{:.2f}".format(benchmark.first_infer(exe_network))
        logger.info("First inference took {} ms".format(duration_ms))
        if statistics:
            statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                    [
                                        ('first inference time (ms)', duration_ms)
                                    ])
        fps, latency_ms, total_duration_sec, iteration = benchmark.infer(exe_network, batch_size, progress_bar)

        # ------------------------------------ 11. Dumping statistics report -------------------------------------------
        next_step()

        if args.dump_config:
            dump_config(args.dump_config, config)
            logger.info("Inference Engine configuration settings were dumped to {}".format(args.dump_config))

        if args.exec_graph_path:
            dump_exec_graph(exe_network, args.exec_graph_path)

        if perf_counts:
            perfs_count_list = []
            for ni in range(int(benchmark.nireq)):
                perfs_count_list.append(exe_network.requests[ni].get_perf_counts())
            if args.perf_counts:
                print_perf_counters(perfs_count_list)
            if statistics:
              statistics.dump_performance_counters(perfs_count_list)

        if statistics:
            statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                      [
                                          ('total execution time (ms)', '{:.2f}'.format(get_duration_in_milliseconds(total_duration_sec))),
                                          ('total number of iterations', str(iteration)),
                                      ])
            if MULTI_DEVICE_NAME not in device_name:
                statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                          [
                                              ('latency (ms)', '{:.2f}'.format(latency_ms)),
                                          ])

            statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                      [
                                          ('throughput', '{:.2f}'.format(fps)),
                                      ])

        if statistics:
          statistics.dump()

        print('Count:      {} iterations'.format(iteration))
        print('Duration:   {:.2f} ms'.format(get_duration_in_milliseconds(total_duration_sec)))
        if MULTI_DEVICE_NAME not in device_name:
            print('Latency:    {:.2f} ms'.format(latency_ms))
        print('Throughput: {:.2f} FPS'.format(fps))

        del exe_network

        next_step.step_id = 0
    except Exception as e:
        logger.exception(e)

        if statistics:
            statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                      [
                                          ('error', str(e)),
                                      ])
            statistics.dump()
        sys.exit(1)


from sklearn import model_selection, preprocessing, linear_model, naive_bayes, metrics, svm
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn import decomposition, ensemble
import pandas as pd, xgboost, numpy as np, textblob, string
import statistics

predicted_vector = []
classes_csv = open('/home/arshad/Documents/FYP99/FYP/UserSpecs2PseudoCode/PC_Interface/pc_corpus/classes.csv').read()
classes = {}

for k, line in enumerate(classes_csv.split("\n")):
    try:
        if line is not '':
            content = line.split(',')
        classes[content[0]] = (content[1])
    except:
        print("Unable to locate classes_csv")

def predict(classifier, feature_vector_train, label, feature_vector_pred, header):
    clf = classifier
    clf.fit(feature_vector_train, label)
    r = clf.predict(feature_vector_pred)
    lbl = encoder.inverse_transform(r)
    # predicted_vector.append(r)
    for lb in lbl:
        intent = classes[lb]
        print(intent)
    predicted_vector.append(str(lbl))
    print("-" * 30)
    print(header + "\n" + str(r) + "-" + str(lbl))
    print("=" * 50)

def train_model(classifier, feature_vector_train, label, feature_vector_valid, is_neural_net=False):
    # fit the training dataset on the classifier
    classifier.fit(feature_vector_train, label)

    # predict the labels on validation dataset
    predictions = classifier.predict(feature_vector_valid)

    if is_neural_net:
        predictions = predictions.argmax(axis=-1)

    return metrics.accuracy_score(predictions, valid_y)

# load the dataset
# data = open('pc_corpus/corpus').read()
data = open('pc_corpus/user_entered').read()
labels, texts = [], []

# load the dataset (testing 1by1)
data_testing = open('pc_corpus/pred')
# data_testing = open('pc_corpus/users_entered_lines')
texts_testing = [line for line in data_testing.readlines() if line.strip()]

# load the dataset (filling 1by1)
data_filling = open('pc_corpus/test_filling').read()
labels_filling, texts_filling = [], []

for i, line in enumerate(data.split("\n")):
    content = line.split()
    try:
        labels.append(content[0])
        texts.append(" ".join(content[1:]))
    except:
        print("End of the list")

for i, line in enumerate(data_filling.split("\n")):
    content = line.split()
    try:
        labels_filling.append(content[0])
        texts_filling.append(" ".join(content[1:]))
    except:
        print("End of the list")

# create a dataframe using texts and lables
trainDF = pd.DataFrame()
trainDF['text'] = texts
trainDF['label'] = labels

# create a dataframe using texts and lables (testing 1by1)
trainDF_testing = pd.DataFrame()
trainDF_testing['text'] = texts_testing
print(trainDF_testing['text'])

# create a dataframe using texts and lables (filling)
trainDF_filling = pd.DataFrame()
trainDF_filling['text'] = texts_filling
trainDF_filling['label'] = labels_filling

# split the dataset into training and validation datasets
train_x, valid_x, train_y, valid_y = model_selection.train_test_split(trainDF['text'], trainDF['label'], test_size=0.2)

# combine the filling and corpus
frames_y = [valid_y, trainDF_filling['label']]
frames_x = [valid_x, trainDF_filling['text']]
valid_y_filled = pd.concat(frames_y)
valid_x_filled = pd.concat(frames_x)
# print(valid_x)
print("=" * 50)
# print(valid_x_filled)

f = open("results.txt", "w+")
f.write('*' * 120 + '\n')

# label encode the target variable
encoder = preprocessing.LabelEncoder()
train_y = encoder.fit_transform(train_y)

f.write('=' * 120 + '\n')
valid_y = encoder.fit_transform(valid_y_filled)

for n in range(48):
    x = encoder.inverse_transform(n)
    f.write(str(n) + " : " + str(x) + "\n")

# create a count vectorizer object
count_vect = CountVectorizer(analyzer='word', token_pattern=r'\w{1,}')
count_vect.fit(pd.concat([trainDF['text'], trainDF_testing['text'], trainDF_filling['text']]))

# transform the training and validation data using count vectorizer object
xtrain_count = count_vect.transform(train_x)
xvalid_count = count_vect.transform(valid_x_filled)
xvalid_count_t = count_vect.transform(trainDF_testing['text'])
# print(xvalid_count_t)

# word level tf-idf
tfidf_vect = TfidfVectorizer(analyzer='word', token_pattern=r'\w{1,}', max_features=5000)
tfidf_vect.fit(pd.concat([trainDF['text'], trainDF_testing['text'], trainDF_filling['text']]))
xtrain_tfidf = tfidf_vect.transform(train_x)
xvalid_tfidf = tfidf_vect.transform(valid_x_filled)
xvalid_tfidf_t = tfidf_vect.transform(trainDF_testing['text'])

# ngram level tf-idf
tfidf_vect_ngram = TfidfVectorizer(analyzer='word', token_pattern=r'\w{1,}', ngram_range=(2, 3), max_features=5000)
tfidf_vect_ngram.fit(pd.concat([trainDF['text'], trainDF_testing['text'], trainDF_filling['text']]))
xtrain_tfidf_ngram = tfidf_vect_ngram.transform(train_x)
xvalid_tfidf_ngram = tfidf_vect_ngram.transform(valid_x_filled)
xvalid_tfidf_ngram_t = tfidf_vect_ngram.transform(trainDF_testing['text'])

# characters level tf-idf
tfidf_vect_ngram_chars = TfidfVectorizer(analyzer='char', token_pattern=r'\w{1,}', ngram_range=(2, 3),
                                         max_features=5000)
tfidf_vect_ngram_chars.fit(pd.concat([trainDF['text'], trainDF_testing['text'], trainDF_filling['text']]))
xtrain_tfidf_ngram_chars = tfidf_vect_ngram_chars.transform(train_x)
xvalid_tfidf_ngram_chars = tfidf_vect_ngram_chars.transform(valid_x_filled)
xvalid_tfidf_ngram_chars_t = tfidf_vect_ngram_chars.transform(trainDF_testing['text'])

predict(svm.SVC(gamma=0.001, C=100.), xtrain_count, train_y, xvalid_count_t, "SVM, N-Gram Vectors")
predict(naive_bayes.MultinomialNB(), xtrain_count, train_y, xvalid_count_t, "NB, Count Vectors")
predict(naive_bayes.MultinomialNB(), xtrain_tfidf, train_y, xvalid_tfidf_t, "NB, WordLevel TF-IDF")
predict(naive_bayes.MultinomialNB(), xtrain_tfidf_ngram, train_y, xvalid_tfidf_ngram_t, "NB, N-Gram Vectors")
predict(naive_bayes.MultinomialNB(), xtrain_tfidf_ngram_chars, train_y, xvalid_tfidf_ngram_chars_t, "NB, CharLevel "
                                                                                "Vectors")

predict(linear_model.LogisticRegression(), xtrain_count, train_y, xvalid_count_t, "LR, Count Vectors")
predict(linear_model.LogisticRegression(), xtrain_tfidf, train_y, xvalid_tfidf_t, "LR, WordLevel TF-IDF")
predict(linear_model.LogisticRegression(), xtrain_tfidf_ngram, train_y, xvalid_tfidf_ngram_t, "LR, N-Gram Vectors")
predict(linear_model.LogisticRegression(), xtrain_tfidf_ngram_chars, train_y, xvalid_tfidf_ngram_chars_t, "LR, "
                                                                                "CharLevel "
                                                                                "Vectors")
predict(svm.SVC(), xtrain_tfidf_ngram, train_y, xvalid_tfidf_ngram_t, "SVM, N-Gram Vectors")
predict(ensemble.RandomForestClassifier(), xtrain_count, train_y, xvalid_count_t, "RF, Count Vectors")
predict(ensemble.RandomForestClassifier(), xtrain_tfidf, train_y, xvalid_tfidf_t, "RF, WordLevel TF-IDF")

# Naive Bayes on Count Vectors
accuracy = train_model(naive_bayes.MultinomialNB(), xtrain_count, train_y, xvalid_count)
print("NB, Count Vectors: ", accuracy)

# Naive Bayes on Word Level TF IDF Vectors
accuracy = train_model(naive_bayes.MultinomialNB(), xtrain_tfidf, train_y, xvalid_tfidf)
print("NB, WordLevel TF-IDF: ", accuracy)

# Naive Bayes on Ngram Level TF IDF Vectors
accuracy = train_model(naive_bayes.MultinomialNB(), xtrain_tfidf_ngram, train_y, xvalid_tfidf_ngram)
print("NB, N-Gram Vectors: ", accuracy)

# Naive Bayes on Character Level TF IDF Vectors
accuracy = train_model(naive_bayes.MultinomialNB(), xtrain_tfidf_ngram_chars, train_y, xvalid_tfidf_ngram_chars)
print("NB, CharLevel Vectors: ", accuracy)

# Linear Classifier on Count Vectors
accuracy = train_model(linear_model.LogisticRegression(), xtrain_count, train_y, xvalid_count)
print("LR, Count Vectors: ", accuracy)

# Linear Classifier on Word Level TF IDF Vectors
accuracy = train_model(linear_model.LogisticRegression(), xtrain_tfidf, train_y, xvalid_tfidf)
print("LR, WordLevel TF-IDF: ", accuracy)

# Linear Classifier on Ngram Level TF IDF Vectors
accuracy = train_model(linear_model.LogisticRegression(), xtrain_tfidf_ngram, train_y, xvalid_tfidf_ngram)
print("LR, N-Gram Vectors: ", accuracy)

# Linear Classifier on Character Level TF IDF Vectors
accuracy = train_model(linear_model.LogisticRegression(), xtrain_tfidf_ngram_chars, train_y, xvalid_tfidf_ngram_chars)
print("LR, CharLevel Vectors: ", accuracy)

# SVM on Ngram Level TF IDF Vectors
accuracy = train_model(svm.SVC(), xtrain_tfidf_ngram, train_y, xvalid_tfidf_ngram)
print("SVM, N-Gram Vectors: ", accuracy)

# SVM on Ngram Level TF IDF Vectors (param changed)
accuracy = train_model(svm.SVC(gamma=0.001, C=100.), xtrain_count, train_y, xvalid_count)
print("SVM, N-Gram Vectors Mod: ", accuracy)

# RF on Count Vectors
accuracy = train_model(ensemble.RandomForestClassifier(), xtrain_count, train_y, xvalid_count)
print("RF, Count Vectors: ", accuracy)

# RF on Word Level TF IDF Vectors
accuracy = train_model(ensemble.RandomForestClassifier(), xtrain_tfidf, train_y, xvalid_tfidf)
print("RF, WordLevel TF-IDF: ", accuracy)

# print(predicted_vector)

# for i in predicted_vector:
#     print(encoder.inverse_transform())

try:
    res = statistics.mode(predicted_vector)
except statistics.StatisticsError:
    print('equal no of res found')
    res = predicted_vector[6]

# print(res.split())
for result in res.split():
    result = result.replace(']', '').replace('[','').replace('\'', '')
    i = classes[result]
    print(i)


'''
IBEHS 1P10 Mini Milestone 11 Main File

Team Number: 76

Name: Owen Johnstone

Date: Feb. 25, 2020
'''
import matplotlib.patches as patches
import math
import matplotlib.pyplot as plt

""" GIVEN Functions """
## This function creates 6" x 6" graphics window
## The functional workspace is created within this window
def functional_workspace():
    fig, ax = plt.subplots(1,1,figsize=(6,6))
    rect = patches.Rectangle((-15,-15),30,30,angle=0.0,fill = False, ls = '--', color = 'r',linewidth=1.5)
    ax.add_patch(rect)
    plt.xlim((-45,45))
    plt.ylim((-45,45))
    ax.set(title='Forefinger-Thumb Path',
            xlabel = 'X (mm) ',
            ylabel = 'Y (mm) ')

##Student ID: 400242374
def forefinger_position(degree_of_increments):
    forefinger_length= 36
    position_list= []

    for angle in range(0, 91, int(degree_of_increments)):
        xcoordinates= -(forefinger_length*math.cos(math.radians(angle)))
        ycoordinates= 36 - forefinger_length*math.sin(math.radians(angle))
        position_list.append([xcoordinates, ycoordinates])
    return position_list

##Student ID: 400268572
def thumb_position(num_inc):
    thumb_length = 42
    coordinate_list = []

    for angle in range(0,91,int(num_inc)):
        x_coordinate = 42 - thumb_length * math.sin(math.radians(angle))
        y_coordinate = -thumb_length * math.cos(math.radians(angle))
        coordinate_list.append([x_coordinate, y_coordinate])

    return coordinate_list

##Student ID: 400242374
def write_lists(file_name, list1, list2):
    length_of_list1= float(len(list1))
    length_of_list2= float(len(list2))
    if length_of_list1 == length_of_list2:
        with open(file_name, "w") as file:
            for i in range(len(list1)):
                file.write(str(list1[i])+"\t"+str(list2[i])+"\n")
    else:
        print("The lengths of the two lists are not equal.")

##Student ID: 400268572
def append_array(text_file, data_array):
    with open(text_file, 'a') as txt:
        for item in data_array:
            txt.write(str(item[0]) + '\t' + str(item[1]) + '\n')

##Student ID: 400242374
def plot_forefinger():
    number_of_increments=int(input("Enter a value for the number of increments."))
    while 90 % number_of_increments != 0:
        print("The number you enetered is not go into 90 evenly. Try another number.")
        number_of_increments= int(input("Enter a value for the number of increments."))

    functional_workspace()
    data_list= forefinger_position(number_of_increments)
    for element in range(len(data_list)):
        x_cd= data_list[element][0]
        y_cd= data_list[element][1]
        plt.scatter(x_cd,y_cd)
    plt.show()

##Student ID: 400268572
def plot_thumb():
    num_inc = int(input('Enter the number of degrees to increment by: '))

    while 90 % num_inc != 0:
        print('Please enter a number of degrees that evenly divides into 90.')
        num_inc = int(input('Enter the number of degrees to increment by: '))

    functional_workspace()
    data_list = thumb_position(num_inc)

    for i in range(len(data_list)):
        x_cd = data_list[i][0]
        y_cd = data_list[i][1]

        plt.scatter(x_cd, y_cd, c='r')
    plt.show()

##Main Function
def main():
    inc_num = int(input('Please enter the number of degrees to increment by: '))

    while 90 % inc_num != 0:
        print('Please enter a number of degrees that evenly divides into 90.')
        inc_num = int(input('Enter the number of degrees to increment by: '))

    functional_workspace()
    thumb_pos  = thumb_position(inc_num)
    finger_pos = forefinger_position(inc_num)

    thumb_x_list = []
    forefinger_x_list = []

    thumb_y_list = []
    forefinger_y_list = []

    for i in range(len(thumb_pos)):
        thumb_x = thumb_pos[i][0]
        thumb_x_list.append(round(thumb_x, 2))

        thumb_y = thumb_pos[i][1]
        thumb_y_list.append(round(thumb_y, 2))

        finger_x = finger_pos[i][0]
        forefinger_x_list.append(round(finger_x, 2))

        finger_y = finger_pos[i][1]
        forefinger_y_list.append(round(finger_y, 2))

        plt.scatter(thumb_x, thumb_y)
        plt.scatter(finger_x, finger_y)

    plt.show()

    x_coordinates = thumb_x_list + forefinger_x_list
    y_coordinates = thumb_y_list + forefinger_y_list

    write_lists('Objective3.txt', x_coordinates, y_coordinates)
    append_array('Objective4.txt', finger_pos)
    append_array('Objective4.txt', thumb_pos)

main()