import os
from collections import abc
from collections import OrderedDict
import inspect
import json

import tensorflow as tf
import numpy as np
from tensorflow.python.keras import callbacks as cbks
from tensorflow.python.keras.engine import training_utils as keras_train_utils
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle as sklearn_shuffle

LOGS_PATH = os.path.join(os.getcwd(), '.deep_tempering_logs')

with tf.compat.v1.keras.backend.get_session().graph.as_default():
  _IS_TRAINING_PLACEHOLDER = tf.compat.v1.placeholder_with_default(True, shape=())

def get_training_phase_placeholder():
  return _IS_TRAINING_PLACEHOLDER

class HyperParamState:
  def __init__(self, default_values=None):
    self._attrs = {}
    self.default_values = default_values
  def get_hparam(self, name, default_value=None):
    # during creating of the optimal model default (such
    # as dropout etc) could be passed.
    if (isinstance(self.default_values, abc.Mapping)
        and name in self.default_values):
      return self.default_values[name]

    if name in self._attrs:
      raise ValueError('Hyper Params with name ', hp_name, 'already exists.')

    if default_value is None:
      hp = tf.compat.v1.placeholder(tf.float32, shape=(), name=name)
    else:
      hp = tf.compat.v1.placeholder_with_default(default_value,
                                                 shape=(),
                                                 name=name)
    self._attrs[name] = hp
    return hp

  def _get_hparam(self, name):
    if name in self._attrs:
      return self._attrs[name]

class ScheduledHyperParamSpace:
  def __init__(self, ensemble_model, hparams_dict):
    self.ensemble_model = ensemble_model
    if all(isinstance(s, str) for s in hparams_dict):
      # not scheduled hyperparams, start scheduling at step 1
      hparams_dict = {1: hparams_dict}

    self.hparams_spaces = {
        step: HyperParamSpace(ensemble_model, hparams_dict[step])
        for step in hparams_dict
    }

  def get_current_hyperparams_space(self):
    curr_step = self._get_current_scheduled_step()
    return self.hparams_spaces[curr_step]

  def _get_current_scheduled_step(self):
    """Returns starting step for which current hyperparams are valid."""
    scheduled_steps = sorted(list(self.hparams_spaces.keys()))
    glob_step = getattr(
        self.ensemble_model, 'global_step', min(scheduled_steps))
    curr_step = scheduled_steps[0]

    for i in range(1, len(scheduled_steps)):
      if glob_step >= scheduled_steps[i]:
        curr_step = scheduled_steps[i]

    return curr_step

class HyperParamSpace:
  """Represents the hyper-parameter state of all replicas.

  Serves as container for placeholders and actual values that
  are fed and updated during training.
  """
  def __init__(self, ensemble_model, hparams_dict):
    """Creates a new `HyperParamState` instance.
    ```python
    hparams_dict = {
        'learning_rate': np.linspace(0.001, 0.01, 6),
        'dropout_rate': np.linspace(0., 0.6, 6)
    }
    hps = HyperParamSpace(hparams_dict)
    hps.hpspace
    # {0: {'learning_rate': 0.001, 'dropout_rate': 0.0},
    #  1: {'learning_rate': 0.0055000000000000005, 'dropout_rate': 0.3},
    #  2: {'learning_rate': 0.01, 'dropout_rate': 0.6}}
    ```
    """
    self.ensemble_model = ensemble_model

    hparams_dict = dict((k, list(v)) for k, v in hparams_dict.items())
    n_replicas = len(hparams_dict[hparams_dict.__iter__().__next__()])
    self._hyperparameter_names = sorted(list(hparams_dict.keys()))
    self.n_replicas = n_replicas
    self.hpspace = {
        i: {k: v[i] for k, v in hparams_dict.items()}
        for i in range(n_replicas)
    }

  def set_hyperparams(self, value, hp_name, replica_id):
    self.hpspace[replica_id][hp_name] = value

  @property
  def hyperparameters_names(self):
    return self._hyperparameter_names

  def swap_between(self, replica_i, replica_j, hyperparam_name):
    """Swaps `hyperparam_name` between `replica_i` and `replica_j`."""
    hp_i = self.hpspace[replica_i][hyperparam_name]
    hp_j = self.hpspace[replica_j][hyperparam_name]
    self.hpspace[replica_j][hyperparam_name] = hp_i
    self.hpspace[replica_i][hyperparam_name] = hp_j

  def copy_hyperparams(self, src_replica, dst_replica, hyperparam_names=None):
    """Copies hyperparams in hyperparam_names from src to dst replica."""
    hyperparam_names = hyperparam_names or self.hyperparameters_names

    for hpname in hyperparam_names:
      self.hpspace[dst_replica][hpname] = self.hpspace[src_replica][hpname]

  def perturb_hyperparams(self, replica_id, hyperparam_name, dist_fn):
    """Perturbs `hyperparam_name` of `replica_id` by the value generated by `dist_fn()`."""
    self.hpspace[replica_id][hyperparam_name] += dist_fn()
    # self.hpspace[replica_id][hyperparam_name] *= dist_fn()

  def get_ordered_hparams(self, name):
    """Returns list of tuples of adjacent `(replica_id, hp_value)`."""
    hparams = [(i, self.hpspace[i][name]) for i in range(self.n_replicas)]
    hparams.sort(key=lambda x: x[1])
    return hparams

  def prepare_feed_tensors_and_values(self, training=True):
    # TODO: replace `training` with ModeKeys instance check

    assert training in {True, False}

    n_replicas = len(self.hpspace)
    hpnames = list(self.hpspace[0].keys())

    current_hp_dict = {
        name: dict(self.get_ordered_hparams(name))
        for name in hpnames
    }

    hpstates = {
        i: self.ensemble_model._train_attrs[i]['hp_state']
        for i in range(n_replicas)
    }
    feed_dict = {}

    for i in range(n_replicas):
      for hpname in hpnames:
        if not training and 'dropout_rate' in hpname:
          value = 0.
        else:
          value = current_hp_dict[hpname][i]
        placeholder = hpstates[i]._get_hparam(hpname)

        assert placeholder is not None
        feed_dict[placeholder] = value

    feed_dict[get_training_phase_placeholder()] = training

    return feed_dict

def call_metric_function(metric_fn,
                         y_true,
                         y_pred=None,
                         weights=None,
                         mask=None):
  return keras_train_utils.call_metric_function(metric_fn,
                                                y_true,
                                                y_pred,
                                                weights,
                                                mask)

def _infer_init_args_kwargs(cls_):
  """Attempts to extract args/kwargs that class `cls_` has been init'ed with."""
  extracted_signature = OrderedDict()
  signature = inspect.signature(cls_.__init__)
  args_dict = OrderedDict(
      [(k, v.default) for k, v in signature.parameters.items()])
  for arg in args_dict:
    default_value = None if args_dict[arg] == inspect._empty else args_dict[arg]
    extracted_value = None
    try:
      extracted_value = cls_.__dict__[arg]
    except KeyError:
      try:
        extracted_value = cls_.__dict__['_' + arg]
      except KeyError:
        pass
    extracted_signature[arg] = extracted_value or default_value
  return extracted_signature

def infer_shape_from_numpy_array(ary):
  if len(ary.shape) == 1:
    return (None,)
  return (None,) + ary.shape[1:]

def create_training_target(shape, dtype=None):
  dtype = dtype or tf.int32

  if shape[0] is None:
    shape = shape[1:]

  return tf.keras.layers.Input(shape, dtype=dtype)

class MetricsAggregator(keras_train_utils.Aggregator):
  """Aggregator that calculates loss and metrics info.
  Attributes:
    use_steps: Whether the loop is using `step` or `batch_size`.
    num_samples: Total number of samples: `batch_size*num_batches`.
    steps: Total number of steps, ie number of times to iterate over a dataset
      to cover all samples.
  """

  def __init__(self, n_replicas, num_samples=None, steps=None):
    super(MetricsAggregator, self).__init__(
        use_steps=False,
        num_samples=num_samples,
        steps=steps,
        batch_size=None)
    self.n_replicas = n_replicas

  def create(self, batch_outs):
    self.results = [0.] * len(batch_outs)

  def aggregate(self, batch_outs, batch_start=None, batch_end=None):
    # Losses.
    for i in range(self.n_replicas):
      self.results[i] += batch_outs[i] * (batch_end - batch_start)

    # Metrics (always stateful, just grab current values.)
    self.results[self.n_replicas:] = batch_outs[self.n_replicas:]

  def finalize(self):
    if not self.results:
      raise ValueError('Empty training data.')

    for i in range(self.n_replicas):
      self.results[i] /= self.num_samples

def prepare_data_iterables(x,
                           y=None,
                           validation_split=0.0,
                           validation_data=None,
                           exchange_split=0.0,
                           exchange_data=None,
                           batch_size=32,
                           epochs=1,
                           shuffle=True,
                           random_state=0):

  # during testing DataIterable is passed
  if isinstance(x, DataIterable):
    return x

  # predict mode
  if y is None:
    return DataIterable(x, y, batch_size, epochs, shuffle)

  train_data, validation_data, exchange_data = (
      _train_validation_exchange_data((x, y), validation_data=validation_data,
          exchange_data=exchange_data, validation_split=validation_split,
          exchange_split=exchange_split))

  train_iterable = DataIterable(train_data[0], train_data[1],
                                batch_size=batch_size, epochs=epochs,
                                shuffle=shuffle)
  if validation_data is not None:
    validation_iterable = DataIterable(validation_data[0], validation_data[1],
                                       batch_size=batch_size, epochs=epochs,
                                       shuffle=shuffle)
  else:
    validation_iterable = None

  # exchange data is passed as is

  return (train_iterable, validation_iterable, exchange_data)

  # elif  0.0 < validation_split < 1:
  #   x_train, x_test, y_train, y_test = train_test_split(x, y,
  #       test_size=validation_split, random_state=random_state)
  #   train_dataset = DataIterable(x_train, y_train, batch_size, epochs, shuffle)
  #   test_dataset = DataIterable(x_test, y_test, batch_size, epochs, shuffle)
  #   return [train_dataset, test_dataset]
  # else:
  #   raise ValueError('Cannot parition data.')

def _validate_dataset_shapes(*args):
  shapes = []
  for arg in args:
    if isinstance(arg, (list, tuple)):
      for item in arg:
        shapes.append(item.shape)
    else:
      shapes.append(arg.shape)

  if len(set([s[0] for s in shapes])) != 1:
    raise ValueError('First dimension of inputs and targets must be equal')

class DataIterable:
  """Batch-wise iterable for numpy data."""
  def __init__(self, x, y=None, batch_size=32, epochs=1, shuffle=False):
    x = np.asarray(x)

    if y is not None:
      y = np.asarray(y)
      assert x.shape[0] == y.shape[0]
    self.data = {
        'x': x,
        'y': y
    }
    self.batch_size = batch_size
    self.epochs = epochs
    self.shuffle = shuffle

  def __iter__(self):
    return _NumpyIterator(self.data,
                          self.batch_size,
                          self.epochs,
                          self.shuffle)

  def __len__(self):
    return self.data['x'].shape[0]

class _NumpyIterator:
  def __init__(self, data_dict, batch_size, epochs, shuffle):
    self.data_dict = data_dict
    self.batch_size = batch_size or 128
    self.epochs = epochs
    self.shuffle = shuffle
    self.begin = 0
    self.end = min(self.batch_size, data_dict['x'].shape[0])
    self.epoch_num = 0

  def __next__(self):

    y = self.data_dict['y']
    if self.begin == 0 and self.shuffle and y is not None:
      self.data_dict = arrays_datadict_shuffle(self.data_dict)
      y = self.data_dict['y']

    x = self.data_dict['x']
    if self.begin >= x.shape[0]:
      self.epoch_num += 1
      if self.epoch_num >= self.epochs:
        raise StopIteration()
      else:
        self.begin = 0
        self.end = min(self.batch_size, x.shape[0])

    batch_x = x[self.begin: self.end]
    if y is not None:
      batch_y = y[self.begin: self.end]

    self.begin = self.end
    self.end += self.batch_size
    if y is not None:
      return batch_x, batch_y
    else:
      return batch_x

class _GraphModeIterator:
  def __init__(self, next_elem):
    self.next_elem = next_elem

  def __next__(self):
    try:
      sess = tf.compat.v1.keras.backend.get_session()
      evaled = sess.run(self.next_elem)
    except tf.compat.v1.errors.OutOfRangeError:
      raise StopIteration()
    return evaled['x'], evaled['y']

class GraphModeDataIterable:
  # TODO: Extend support for eager iteration
  # Implement Wrapper for other than numpy arrays data types.

  def __init__(self, x, y, batch_size=32, epochs=1, shuffle=True, shuffle_buf_size=1024):

    self.batch_size = min(batch_size or 128, y.shape[0])
    self.epochs = epochs
    self.shuffle = shuffle
    self.shuffle_buf_size = shuffle_buf_size
    self.__len = x.shape[0]
    _validate_dataset_shapes(*[x, y])
    d = tf.data.Dataset.from_tensor_slices({
        'x': x,
        'y': y
    })
    d = d.repeat(self.epochs)
    if self.shuffle:
      d = d.shuffle(self.shuffle_buf_size)
    d = d.batch(self.batch_size)

    self._iterator = d.make_initializable_iterator()
    self._next = self._iterator.get_next()

  def __iter__(self):
    # When invoking validation in training loop, avoid creating iterator and
    # list of feed values for the same validation dataset multiple times (which
    # essentially would call `iterator.get_next()` that slows down execution
    # and leads to OOM errors eventually.
    if not tf.executing_eagerly():
      sess = tf.compat.v1.keras.backend.get_session()
      sess.run(self._iterator.initializer)
      return _GraphModeIterator(self._next)
    else:
      raise NotImplementedError()

  def __len__(self):
    return self.__len

def arrays_datadict_shuffle(datadict):
  """Shuffles all values in `dict` in unison."""
  indices = np.arange(datadict['x'].shape[0])
  np.random.shuffle(indices)

  return {
      k: np.take(v, indices=indices, axis=0) if v is not None else v
      for k, v in datadict.items()
  }

def _train_validation_exchange_data(train_data,
                                    validation_data=None,
                                    exchange_data=None,
                                    validation_split=0.0,
                                    exchange_split=0.0,
                                    random_state=0,
                                    verbose=0):
  """Extracts exchange data.
  Extraction of the exchanges dataset is done in the following order:

  * If exchange data is passes explicitly, then use this data.
  * Else If `exchange_split` is non-zero sample randomly out of train
  * Else If validation data is passed explicitly, then use this data.
    data.
  * Else raise `ValueError
  """
  # validate train data

  assert all(isinstance(a, np.ndarray) for a in train_data)

  x_train, y_train = train_data

  # extract validation data
  if validation_data is not None:
    validation_data = validation_data
  elif validation_split > 0.0:
    x_train, x_validation, y_train, y_validation = train_test_split(
        x_train, y_train, random_state=0, test_size=validation_split)
    validation_data = (x_validation, y_validation)

  # extract exchange data
  if exchange_data is not None:
    # exchange data is passed explicitly
    exchange_data = exchange_data
  elif exchange_split > 0:
    # exchange data is not passed explicitly, but `exchange_split > 0`
    x_train, x_exchange, y_train, y_exchange = train_test_split(
        x_train, y_train, random_state=random_state, test_size=exchange_split)
    # exchange data is not passed and exchange_split == 0, using validation_data
    exchange_data = (x_exchange, y_exchange)
  elif validation_data is not None:
    exchange_data = validation_data

  # Maybe show as warning?
  # if exchange_data is None:
  #   err_msg = ("Unable to extract exchange dataset. Pass exchange dataset "
  #              "explicitly as argument to `fit()`, or specify "
  #              "`exchange_split` argument to greater than 0, or "
  #              "pass validation_data or `validation_split > 0`.")
  #   raise ValueError(err_msg)

  return (x_train, y_train), validation_data, exchange_data

def gpu_device_name(replica_id):
  """Returns a device name on which is replica is going to executed."""
  if tf.__version__ < '2.1.0':
    gpus = tf.config.experimental.list_physical_devices('GPU')
  else:
    gpus = tf.config.list_physical_devices('GPU')

  gpus_names = [g.name.replace('/physical_device:', '') for g in gpus]
  if not gpus_names:
    return '/cpu:0'

  return gpus_names[replica_id % len(gpus_names)]

def min_or_max_for_metric(metric_name):
  """Decides whether optimal value for `metric_name` is being maximized or minimized"""
  min_metrics = ['loss', 'error']
  max_metrics = ['acc', 'accuracy', 'precision', 'recall', 'auc']

  if any(m in metric_name for m in min_metrics):
    return 'min'
  elif any(m in metric_name for m in max_metrics):
    return 'max'
  else:
    return 'min'

def load_optimal_model(model_builder, hyperparams=None, path=None):
  """Loads optimal model stored by instance of `MonitorOptimalModelCallback`.

  Args:
    model_builder: A function that creates a not compiled keras' model.
    hyperparams: (Optional) A dictionary of hyperparameters (such as dropout)
      you wish to instantiate your model with. If nothing is specified the
      model is created with hyperparameters of the optimal model at that time.
    path: (Optional) A path where the model is stored (in case the model's
      weights were not stored at the default place).
  """
  path = path or LOGS_PATH
  model_weights_path = os.path.join(path, 'optimal_model.h5')
  if hyperparams is None:
    with open(os.path.join(path, 'hyperparams.json')) as fo:
      hyperparams = json.load(fo)
  hp = HyperParamState(default_values=hyperparams)
  model = model_builder(hp)
  model.load_weights(model_weights_path)
  return model


from flask import Flask, render_template
import main_script
import threading

app = Flask(__name__)

@app.route('/record', methods=['POST'])
def main():
    # definitions = []
    # thread = threading.Thread(target=main_script.main, args=[])
    # thread.start()
    definitions = main_script.main()
    return render_template('recording.html', definitions=definitions)

@app.route('/')
def hello_word():
    return render_template('index.html')


from django.db import models
from tinymce import HTMLField

# Create your models here.

class About(models.Model):
    title = models.CharField(max_length=250, verbose_name='Заголовок')
    text = HTMLField(verbose_name='Текст')

    class Meta:
        verbose_name = 'О компании'
        verbose_name_plural = 'О компании'

    def __str__(self):
        return '%s' % self.title


# coding: utf-8
from ptrlib.util.encoding import *
from ptrlib.console.color import Color
from abc import ABCMeta, abstractmethod
import re
import sys
import threading
import time
from logging import getLogger

logger = getLogger(__name__)

class Tube(metaclass=ABCMeta):
    def __init__(self):
        self.buf = b''

    @abstractmethod
    def _settimeout(self, timeout):
        pass

    @abstractmethod
    def _recv(self, size, timeout):
        """Receive raw data

        Receive raw data of maximum `size` bytes length through the socket.

        Args:
            size    (int): The data size to receive
            timeout (int): Timeout (in second)

        Returns:
            bytes: The received data
        """
        pass

    def unget(self, data):
        if isinstance(data, str):
            data = str2bytes(data)
        self.buf = data + self.buf

    def recv(self, size=4096, timeout=None):
        """Receive raw data with buffering

        Receive raw data of maximum `size` bytes length through the socket.

        Args:
            size    (int): The data size to receive
            timeout (int): Timeout (in second)

        Returns:
            bytes: The received data
        """
        self._settimeout(timeout)
        if not self.buf:
            data = self._recv(size, timeout=-1)
            if data is not None:
                self.buf += data

        data, self.buf = self.buf[:size], self.buf[size:]
        return data

    def recvonce(self, size, timeout=None):
        """Receive raw data with buffering

        Receive raw data of size `size` bytes length through the socket.

        Args:
            size    (int): The data size to receive
            timeout (int): Timeout (in second)

        Returns:
            bytes: The received data
        """
        self._settimeout(timeout)
        data = b''
        while len(data) < size:
            data += self.recv(size - len(data))

        if len(data) > size:
            self.unget(data[size:])
        return data[:size]

    def recvuntil(self, delim, size=4096, timeout=None):
        """Receive raw data until `delim` comes

        Args:
            size (int)   : The data size to receive at once
            delim (bytes): The delimiter bytes
            timeout (int): Timeout (in second)

        Returns:
            bytes: The received data
        """
        if isinstance(delim, str):
            delim = str2bytes(delim)
        data = b''

        self._settimeout(timeout)
        while data.find(delim) == -1:
            data += self.recv(size, -1)

        pos = data.find(delim) + len(delim)
        self.unget(data[pos:])
        return data[:pos]

    def recvline(self, size=4096, timeout=None, drop=True):
        line = self.recvuntil(b'\n', size, timeout)
        if drop:
            return line.rstrip()
        return line

    def recvlineafter(self, delim, size=4096, timeout=None, drop=True):
        self.recvuntil(delim, size, timeout)
        return self.recvline(size, timeout, drop)

    def recvregex(self, regex, size=4096, discard=True, timeout=None):
        """Receive until a pattern comes

        Receive data until a specified regex pattern matches.

        Args:
            regex (bytes): Regex
            size (int)   : Size to read at once
            timeout (int): Timeout (in second)

        Returns:
            tuple: If the given regex has multiple patterns to find,
                   it returns all matches. Otherwise, it returns the
                   match string. If discard is false, it also returns
                   all data received so far along with the matches.
        """
        if not isinstance(regex, bytes):
            regex = str2bytes(regex)

        p = re.compile(regex)
        data = b''

        self._settimeout(timeout)
        while p.search(data) is None:
            data += self.recv(size, timeout=-1)

        r = p.search(data)
        pos = r.end()
        self.unget(data[pos:])

        group = r.group()
        groups = r.groups()
        if groups:
            if discard:
                return groups
            else:
                return groups, data[:pos]
        else:
            if discard:
                return group
            else:
                return group, data[:pos]

    @abstractmethod
    def send(self, data):
        pass

    @abstractmethod
    def _socket(self):
        pass

    def sendline(self, data, timeout=None):
        """Send a line

        Send a line of data.

        Args:
            data (bytes) : Data to send
            timeout (int): Timeout (in second)
        """
        if isinstance(data, str):
            data = str2bytes(data)
        self.send(data + b'\n')

    def sendafter(self, delim, data, timeout=None):
        """Send raw data after a deliminater

        Send raw data after `delim` is received.

        Args:
            delim (bytes): The deliminater
            data (bytes) : Data to send
            timeout (int): Timeout (in second)

        Returns:
            bytes: Received bytes before `delim` comes.
        """
        if isinstance(data, str):
            data = str2bytes(data)
        recv_data = self.recvuntil(delim, timeout=timeout)
        self.send(data)
        return recv_data

    def sendlineafter(self, delim, data, timeout=None):
        """Send raw data after a deliminater

        Send raw data with newline after `delim` is received.

        Args:
            delim (bytes): The deliminater
            data (bytes) : Data to send
            timeout (int): Timeout (in second)

        Returns:
            bytes: Received bytes before `delim` comes.
        """
        if isinstance(data, str):
            data = str2bytes(data)
        recv_data = self.recvuntil(delim, timeout=timeout)
        self.sendline(data, timeout=timeout)
        return recv_data

    def sh(self, timeout=None):
        """Alias for interactive
        """
        self.interactive(timeout)

    def interactive(self, timeout=None):
        """Interactive mode
        """
        def thread_recv():
            while not flag.isSet():
                try:
                    data = self.recv(size=4096, timeout=0.1)
                    if data is not None:
                        sys.stdout.write(bytes2str(data))
                        sys.stdout.flush()
                except TimeoutError:
                    pass
                except EOFError:
                    logger.error("Receiver EOF")
                    break
                except ConnectionAbortedError:
                    logger.error("Receiver EOF")
                    break
                time.sleep(0.1)

        flag = threading.Event()
        th = threading.Thread(target=thread_recv)
        th.setDaemon(True)
        th.start()

        try:
            while not flag.isSet():
                data = input("{bold}{blue}[ptrlib]${end} ".format(
                    bold=Color.BOLD, blue=Color.BLUE, end=Color.END
                ))
                if self._socket() is None:
                    logger.error("Connection already closed")
                    break
                if data is None:
                    flag.set()
                else:
                    try:
                        self.sendline(data)
                    except ConnectionAbortedError:
                        logger.error("Sender EOF")
                        break
                time.sleep(0.1)
        except KeyboardInterrupt:
            flag.set()

        while th.is_alive():
            th.join(timeout = 0.1)
            time.sleep(0.1)

    def __enter__(self):
        return self

    def __exit__(self, e_type, e_value, traceback):
        self.close()

    @abstractmethod
    def close(self):
        pass

    @abstractmethod
    def shutdown(self, target):
        pass


#!/usr/bin/env python3
import click
import logging

from conf import DEFAULT_LOGGER
from lib import set_logger

_LOGGER = logging.getLogger(DEFAULT_LOGGER)

_help = """
Execute DB migration based on the {version}.py file located in the migration folder.\
 Users can manage version history for DB migration.\n
Example usages:\n
    migrate.py version [-f <config_yml_path>] [-d]\n
The contents included in config yml:\n
    - BATCH_SIZE (type: int)\n
        A number of rows to be sent as a batch to the database\n
    - DB_NAME_MAP (type: dict)\n
        This is used because the database name is different depending on the environment.\n
    - LOG_PATH\n
        default: ${HOME}/db_migration_log/{version}.log
"""

@click.command(help=_help)
@click.argument('version')
@click.option('-f', '--file', 'file_path', type=click.Path(exists=True), help='Config file (YAML)', required=True)
def main(version, file_path=None):
    set_logger(version, file_path)

    module = _get_module(version)
    getattr(module, 'main')(file_path)

def _change_version_name(version: str):
    return 'v' + version.replace('.', '_')

def _get_module(version):
    changed_version = _change_version_name(version)
    return __import__(f'migration.{changed_version}', fromlist=['main'])

if __name__ == '__main__':
    main()


import numpy as np
import matplotlib.pyplot as plt
import magpylib as magpy

numSegments = 20

def createCoilMultiplex(idx,dxs,dys,deltaX,deltaY,topOrBot):
    # create a Magpylib collection of Loop Sources that form a coil
 #   coil = magpy.Collection()
    trajectory = []
    radius = 5
    firstPoint = 1
    for i in range(numSegments):        #loop over the segments getting created
    #    thisSegment = [radius*np.cos(2*np.pi*i/numSegments),radius*np.sin(2*np.pi*i/numSegments),z]
    #    trajectory.append(thisSegment)
        if (i == idx):
            x = 0.75*radius*np.sin(2*np.pi*i/numSegments) + dxs[i] + deltaX
            y = 0.75*radius*np.cos(2*np.pi*i/numSegments) + dys[i]  + deltaY
            if x > radius: x = radius
            if x < -radius: x = -radius
            if y > radius: y = radius
            if y < -radius: y = -radius
        else:
            x = 0.75*radius*np.sin(2*np.pi*i/numSegments) + dxs[i]
            y = 0.75*radius*np.cos(2*np.pi*i/numSegments) + dys[i]
            if x > radius: x = radius
            if x < -radius: x = -radius
            if y > radius: y = radius
            if y < -radius: y = -radius
        if topOrBot:
            z = -np.sqrt(radius*radius-x*x)
        else:
            z = np.sqrt(radius*radius-x*x)
        thisSegment = [x,y,z]
        trajectory.append(thisSegment)
        if firstPoint:
            fp = thisSegment
            firstPoint = 0
    trajectory.append(fp)
    coil = magpy.current.Line(1,trajectory)
    return coil

def showPlots(thisCoil,grid,amp,Bin):
    # create figure using Matplotlib
    fig = plt.figure(figsize=(8,4))
    ax1 = fig.add_subplot(121, projection='3d')  # 3D-axis
    ax2 = fig.add_subplot(122,)                  # 2D-axis
    # display the coil on ax1
    thisCoil.show(canvas=ax1)
    # display field in figure with matplotlib
    ax2.streamplot(grid[:,:,0], grid[:,:,2], Bin[:,:,0], Bin[:,:,2],
        density=2, color=np.log(amp), linewidth=1, cmap='autumn')

    plt.tight_layout()
    plt.show()

def findOffField(tB):
    totalOff = 0
    for i in range(tB.shape[0]):
        for j in range(tB.shape[1]):
            totalOff = totalOff + tB[i,j,0]*tB[i,j,0]
    return totalOff

if __name__ == "__main__":
    dxs = [0]*numSegments
#    dxs[0:5] = [3,3,3,3,3]
    dys = [0]*numSegments
    firstStep = 1
    finalOff = 999
    nOpts = 10
    delta = 0.2
    for opt in range(nOpts):            #loop over optimization steps
        for i in range(numSegments):    #loop over which segment gets moved
            finalOff = 999
            coil1 = createCoilMultiplex(i,dxs,dys,0,0,0) #create the two coils
            coil2 = createCoilMultiplex(i,dxs,dys,0,0,1)
            coll = magpy.Collection(coil1,coil2, style_label='coll')
            ts = np.linspace(-3,3,30)
            grid = np.array([[(x,0,z) for x in ts] for z in ts])
            tB = magpy.getB(coll,grid)
            tAmp =  np.linalg.norm(tB, axis=2)
            totalOff1 = findOffField(tB)
            if firstStep:
                firstStep = 0
                startOff = totalOff1
            if totalOff1 < finalOff :
                finalOff = totalOff1
                thisDelta = [0,0]
                finalColl = coll
                finalTAmp = tAmp
                finalTb = tB

            coil1 = createCoilMultiplex(i,dxs,dys,delta,0,0)
            coil2 = createCoilMultiplex(i,dxs,dys,delta,0,1)
            coll = magpy.Collection(coil1,coil2, style_label='coll')
            tB = magpy.getB(coll,grid)
            tAmp =  np.linalg.norm(tB, axis=2)
            totalOff2 = findOffField(tB)
            if totalOff2 < finalOff :
                finalOff = totalOff2
                thisDelta = [delta,0]
                finalColl = coll
                finalTAmp = tAmp
                finalTb = tB
            coil1 = createCoilMultiplex(i,dxs,dys,0,delta,0)
            coil2 = createCoilMultiplex(i,dxs,dys,0,delta,1)
            coll = magpy.Collection(coil1,coil2, style_label='coll')
            tB = magpy.getB(coll,grid)
            tAmp =  np.linalg.norm(tB, axis=2)
            totalOff3 = findOffField(tB)
            if totalOff3 < finalOff :
                finalOff = totalOff3
                thisDelta =  [0,delta]
                finalColl = coll
                finalTAmp = tAmp
                finalTb = tB

            coil1 = createCoilMultiplex(i,dxs,dys,-delta,0,0)
            coil2 = createCoilMultiplex(i,dxs,dys,-delta,0,1)
            coll = magpy.Collection(coil1,coil2, style_label='coll')
            tB = magpy.getB(coll,grid)
            tAmp =  np.linalg.norm(tB, axis=2)
            totalOff4 = findOffField(tB)
            if totalOff4 < finalOff :
                finalOff = totalOff4
                thisDelta =  [-delta,0]
                finalColl = coll
                finalTAmp = tAmp
                finalTb = tB

            coil1 = createCoilMultiplex(i,dxs,dys,0,-delta,0)
            coil2 = createCoilMultiplex(i,dxs,dys,0,-delta,1)
            coll = magpy.Collection(coil1,coil2, style_label='coll')
            tB = magpy.getB(coll,grid)
            tAmp =  np.linalg.norm(tB, axis=2)
            totalOff5 = findOffField(tB)
            if totalOff5 < finalOff :
                finalOff = totalOff5
                thisDelta =  [0,-delta]
                finalColl = coll
                finalTAmp = tAmp
                finalTb = tB

            dxs[i] = dxs[i] + thisDelta[0]
            dys[i] = dys[i] + thisDelta[1]
#            print(dxs,dys)

 #       print(finalOff)
    print(startOff)
    print(finalOff)

    showPlots(finalColl,grid,finalTAmp,finalTb)


"""Performance metrics loggers

This module contains all data collectors that record events while simulations
are being executed and compute performance metrics.

Currently implemented data collectors allow users to measure cache hit ratio,
latency, path stretch and link load.

To create a new data collector, it is sufficient to create a new class
inheriting from the `DataCollector` class and override all required methods.
"""
import collections

from icarus.registry import register_data_collector
from icarus.tools import cdf
from icarus.util import Tree, inheritdoc

__all__ = [
    "DataCollector",
    "CollectorProxy",
    "CacheHitRatioCollector",
    "LinkLoadCollector",
    "LatencyCollector",
    "PathStretchCollector",
    "DummyCollector",
]

class DataCollector:
    """Object collecting notifications about simulation events and measuring
    relevant metrics.
    """

    def __init__(self, view, **params):
        """Constructor

        Parameters
        ----------
        view : NetworkView
            An instance of the network view
        params : keyworded parameters
            Collector parameters
        """
        self.view = view

    def start_session(self, timestamp, receiver, content):
        """Notifies the collector that a new network session started.

        A session refers to the retrieval of a content from a receiver, from
        the issuing of a content request to the delivery of the content.

        Parameters
        ----------
        timestamp : int
            The timestamp of the event
        receiver : any hashable type
            The receiver node requesting a content
        content : any hashable type
            The content identifier requested by the receiver
        """
        pass

    def cache_hit(self, node):
        """Reports that the requested content has been served by the cache at
        node *node*.

        Parameters
        ----------
        node : any hashable type
            The node whose cache served the content
        """
        pass

    def cache_miss(self, node):
        """Reports that the cache at node *node* has been looked up for
        requested content but there was a cache miss.

        Parameters
        ----------
        node : any hashable type
            The node whose cache served the content
        """
        pass

    def server_hit(self, node):
        """Reports that the requested content has been served by the server at
        node *node*.

        Parameters
        ----------
        node : any hashable type
            The server node which served the content
        """
        pass

    def request_hop(self, u, v, main_path=True):
        """Reports that a request has traversed the link *(u, v)*

        Parameters
        ----------
        u : any hashable type
            Origin node
        v : any hashable type
            Destination node
        main_path : bool, optional
            If *True*, indicates that link link is on the main path that will
            lead to hit a content. It is normally used to calculate latency
            correctly in multicast cases. Default value is *True*
        """
        pass

    def content_hop(self, u, v, main_path=True):
        """Reports that a content has traversed the link *(u, v)*

        Parameters
        ----------
        u : any hashable type
            Origin node
        v : any hashable type
            Destination node
        main_path : bool, optional
            If *True*, indicates that this link is being traversed by content
            that will be delivered to the receiver. This is needed to
            calculate latency correctly in multicast cases. Default value is
            *True*
        """
        pass

    def end_session(self, success=True):
        """Reports that the session is closed, i.e. the content has been
        successfully delivered to the receiver or a failure blocked the
        execution of the request

        Parameters
        ----------
        success : bool, optional
            *True* if the session was completed successfully, *False* otherwise
        """
        pass

    def results(self):
        """Returns the aggregated results measured by the collector.

        Returns
        -------
        results : dict
            Dictionary mapping metric with results.
        """
        pass

# Note: The implementation of CollectorProxy could be improved to avoid having
# to rewrite almost identical methods, for example by playing with __dict__
# attribute. However, it was implemented this way to make it more readable and
# easier to understand.
class CollectorProxy(DataCollector):
    """This class acts as a proxy for all concrete collectors towards the
    network controller.

    An instance of this class registers itself with the network controller and
    it receives notifications for all events. This class is responsible for
    dispatching events of interests to concrete collectors.
    """

    EVENTS = (
        "start_session",
        "end_session",
        "cache_hit",
        "cache_miss",
        "server_hit",
        "request_hop",
        "content_hop",
        "results",
    )

    def __init__(self, view, collectors):
        """Constructor

        Parameters
        ----------
        view : NetworkView
            An instance of the network view
        collector : list of DataCollector
            List of instances of DataCollector that will be notified of events
        """
        self.view = view
        self.collectors = {
            e: [c for c in collectors if e in type(c).__dict__] for e in self.EVENTS
        }

    @inheritdoc(DataCollector)
    def start_session(self, timestamp, receiver, content):
        for c in self.collectors["start_session"]:
            c.start_session(timestamp, receiver, content)

    @inheritdoc(DataCollector)
    def cache_hit(self, node):
        for c in self.collectors["cache_hit"]:
            c.cache_hit(node)

    @inheritdoc(DataCollector)
    def cache_miss(self, node):
        for c in self.collectors["cache_miss"]:
            c.cache_miss(node)

    @inheritdoc(DataCollector)
    def server_hit(self, node):
        for c in self.collectors["server_hit"]:
            c.server_hit(node)

    @inheritdoc(DataCollector)
    def request_hop(self, u, v, main_path=True):
        for c in self.collectors["request_hop"]:
            c.request_hop(u, v, main_path)

    @inheritdoc(DataCollector)
    def content_hop(self, u, v, main_path=True):
        for c in self.collectors["content_hop"]:
            c.content_hop(u, v, main_path)

    @inheritdoc(DataCollector)
    def end_session(self, success=True):
        for c in self.collectors["end_session"]:
            c.end_session(success)

    @inheritdoc(DataCollector)
    def results(self):
        return Tree(**{c.name: c.results() for c in self.collectors["results"]})

@register_data_collector("LINK_LOAD")
class LinkLoadCollector(DataCollector):
    """Data collector measuring the link load"""

    def __init__(self, view, req_size=150, content_size=1500):
        """Constructor

        Parameters
        ----------
        view : NetworkView
            The network view instance
        req_size : int
            Average size (in bytes) of a request
        content_size : int
            Average size (in byte) of a content
        """
        self.view = view
        self.req_count = collections.defaultdict(int)
        self.cont_count = collections.defaultdict(int)
        if req_size <= 0 or content_size <= 0:
            raise ValueError("req_size and content_size must be positive")
        self.req_size = req_size
        self.content_size = content_size
        self.t_start = -1
        self.t_end = 1

    @inheritdoc(DataCollector)
    def start_session(self, timestamp, receiver, content):
        if self.t_start < 0:
            self.t_start = timestamp
        self.t_end = timestamp

    @inheritdoc(DataCollector)
    def request_hop(self, u, v, main_path=True):
        self.req_count[(u, v)] += 1

    @inheritdoc(DataCollector)
    def content_hop(self, u, v, main_path=True):
        self.cont_count[(u, v)] += 1

    @inheritdoc(DataCollector)
    def results(self):
        duration = self.t_end - self.t_start
        used_links = set(self.req_count.keys()).union(set(self.cont_count.keys()))
        link_loads = {
            link: (
                self.req_size * self.req_count[link]
                + self.content_size * self.cont_count[link]
            )
            / duration
            for link in used_links
        }
        link_loads_int = {
            link: load
            for link, load in link_loads.items()
            if self.view.link_type(*link) == "internal"
        }
        link_loads_ext = {
            link: load
            for link, load in link_loads.items()
            if self.view.link_type(*link) == "external"
        }
        mean_load_int = (
            sum(link_loads_int.values()) / len(link_loads_int)
            if len(link_loads_int) > 0
            else 0
        )
        mean_load_ext = (
            sum(link_loads_ext.values()) / len(link_loads_ext)
            if len(link_loads_ext) > 0
            else 0
        )
        return Tree(
            {
                "MEAN_INTERNAL": mean_load_int,
                "MEAN_EXTERNAL": mean_load_ext,
                "PER_LINK_INTERNAL": link_loads_int,
                "PER_LINK_EXTERNAL": link_loads_ext,
            }
        )

@register_data_collector("LATENCY")
class LatencyCollector(DataCollector):
    """Data collector measuring latency, i.e. the delay taken to delivery a
    content.
    """

    def __init__(self, view, cdf=False):
        """Constructor

        Parameters
        ----------
        view : NetworkView
            The network view instance
        cdf : bool, optional
            If *True*, also collects a cdf of the latency
        """
        self.cdf = cdf
        self.view = view
        self.req_latency = 0.0
        self.sess_count = 0
        self.latency = 0.0
        if cdf:
            self.latency_data = collections.deque()

    @inheritdoc(DataCollector)
    def start_session(self, timestamp, receiver, content):
        self.sess_count += 1
        self.sess_latency = 0.0

    @inheritdoc(DataCollector)
    def request_hop(self, u, v, main_path=True):
        if main_path:
            self.sess_latency += self.view.link_delay(u, v)

    @inheritdoc(DataCollector)
    def content_hop(self, u, v, main_path=True):
        if main_path:
            self.sess_latency += self.view.link_delay(u, v)

    @inheritdoc(DataCollector)
    def end_session(self, success=True):
        if not success:
            return
        if self.cdf:
            self.latency_data.append(self.sess_latency)
        self.latency += self.sess_latency

    @inheritdoc(DataCollector)
    def results(self):
        results = Tree({"MEAN": self.latency / self.sess_count})
        if self.cdf:
            results["CDF"] = cdf(self.latency_data)
        return results

@register_data_collector("CACHE_HIT_RATIO")
class CacheHitRatioCollector(DataCollector):
    """Collector measuring the cache hit ratio, i.e. the portion of content
    requests served by a cache.
    """

    def __init__(self, view, off_path_hits=False, per_node=True, content_hits=False):
        """Constructor

        Parameters
        ----------
        view : NetworkView
            The NetworkView instance
        off_path_hits : bool, optional
            If *True* also records cache hits from caches not on located on the
            shortest path. This metric may be relevant only for some strategies
        content_hits : bool, optional
            If *True* also records cache hits per content instead of just
            globally
        """
        self.view = view
        self.off_path_hits = off_path_hits
        self.per_node = per_node
        self.cont_hits = content_hits
        self.sess_count = 0
        self.cache_hits = 0
        self.serv_hits = 0
        if off_path_hits:
            self.off_path_hit_count = 0
        if per_node:
            self.per_node_cache_hits = collections.defaultdict(int)
            self.per_node_server_hits = collections.defaultdict(int)
        if content_hits:
            self.curr_cont = None
            self.cont_cache_hits = collections.defaultdict(int)
            self.cont_serv_hits = collections.defaultdict(int)

    @inheritdoc(DataCollector)
    def start_session(self, timestamp, receiver, content):
        self.sess_count += 1
        if self.off_path_hits:
            source = self.view.content_source(content)
            self.curr_path = self.view.shortest_path(receiver, source)
        if self.cont_hits:
            self.curr_cont = content

    @inheritdoc(DataCollector)
    def cache_hit(self, node):
        self.cache_hits += 1
        if self.off_path_hits and node not in self.curr_path:
            self.off_path_hit_count += 1
        if self.cont_hits:
            self.cont_cache_hits[self.curr_cont] += 1
        if self.per_node:
            self.per_node_cache_hits[node] += 1

    @inheritdoc(DataCollector)
    def server_hit(self, node):
        self.serv_hits += 1
        if self.cont_hits:
            self.cont_serv_hits[self.curr_cont] += 1
        if self.per_node:
            self.per_node_server_hits[node] += 1

    @inheritdoc(DataCollector)
    def results(self):
        n_sess = self.cache_hits + self.serv_hits
        hit_ratio = self.cache_hits / n_sess
        results = Tree(**{"MEAN": hit_ratio})
        if self.off_path_hits:
            results["MEAN_OFF_PATH"] = self.off_path_hit_count / n_sess
            results["MEAN_ON_PATH"] = results["MEAN"] - results["MEAN_OFF_PATH"]
        if self.cont_hits:
            cont_set = set(
                list(self.cont_cache_hits.keys()) + list(self.cont_serv_hits.keys())
            )
            cont_hits = {
                i: (
                    self.cont_cache_hits[i]
                    / (self.cont_cache_hits[i] + self.cont_serv_hits[i])
                )
                for i in cont_set
            }
            results["PER_CONTENT"] = cont_hits
        if self.per_node:
            for v in self.per_node_cache_hits:
                self.per_node_cache_hits[v] /= n_sess
            for v in self.per_node_server_hits:
                self.per_node_server_hits[v] /= n_sess
            results["PER_NODE_CACHE_HIT_RATIO"] = self.per_node_cache_hits
            results["PER_NODE_SERVER_HIT_RATIO"] = self.per_node_server_hits
        return results

@register_data_collector("PATH_STRETCH")
class PathStretchCollector(DataCollector):
    """Collector measuring the path stretch, i.e. the ratio between the actual
    path length and the shortest path length.
    """

    def __init__(self, view, cdf=False):
        """Constructor

        Parameters
        ----------
        view : NetworkView
            The network view instance
        cdf : bool, optional
            If *True*, also collects a cdf of the path stretch
        """
        self.view = view
        self.cdf = cdf
        self.req_path_len = collections.defaultdict(int)
        self.cont_path_len = collections.defaultdict(int)
        self.sess_count = 0
        self.mean_req_stretch = 0.0
        self.mean_cont_stretch = 0.0
        self.mean_stretch = 0.0
        if self.cdf:
            self.req_stretch_data = collections.deque()
            self.cont_stretch_data = collections.deque()
            self.stretch_data = collections.deque()

    @inheritdoc(DataCollector)
    def start_session(self, timestamp, receiver, content):
        self.receiver = receiver
        self.source = self.view.content_source(content)
        self.req_path_len = 0
        self.cont_path_len = 0
        self.sess_count += 1

    @inheritdoc(DataCollector)
    def request_hop(self, u, v, main_path=True):
        self.req_path_len += 1

    @inheritdoc(DataCollector)
    def content_hop(self, u, v, main_path=True):
        self.cont_path_len += 1

    @inheritdoc(DataCollector)
    def end_session(self, success=True):
        if not success:
            return
        req_sp_len = len(self.view.shortest_path(self.receiver, self.source))
        cont_sp_len = len(self.view.shortest_path(self.source, self.receiver))
        req_stretch = self.req_path_len / req_sp_len
        cont_stretch = self.cont_path_len / cont_sp_len
        stretch = (self.req_path_len + self.cont_path_len) / (req_sp_len + cont_sp_len)
        self.mean_req_stretch += req_stretch
        self.mean_cont_stretch += cont_stretch
        self.mean_stretch += stretch
        if self.cdf:
            self.req_stretch_data.append(req_stretch)
            self.cont_stretch_data.append(cont_stretch)
            self.stretch_data.append(stretch)

    @inheritdoc(DataCollector)
    def results(self):
        results = Tree(
            {
                "MEAN": self.mean_stretch / self.sess_count,
                "MEAN_REQUEST": self.mean_req_stretch / self.sess_count,
                "MEAN_CONTENT": self.mean_cont_stretch / self.sess_count,
            }
        )
        if self.cdf:
            results["CDF"] = cdf(self.stretch_data)
            results["CDF_REQUEST"] = cdf(self.req_stretch_data)
            results["CDF_CONTENT"] = cdf(self.cont_stretch_data)
        return results

@register_data_collector("DUMMY")
class DummyCollector(DataCollector):
    """Dummy collector to be used for test cases only."""

    def __init__(self, view):
        """Constructor

        Parameters
        ----------
        view : NetworkView
            The network view instance
        output : stream
            Stream on which debug collector writes
        """
        self.view = view

    @inheritdoc(DataCollector)
    def start_session(self, timestamp, receiver, content):
        self.session = dict(
            timestamp=timestamp,
            receiver=receiver,
            content=content,
            cache_misses=[],
            request_hops=[],
            content_hops=[],
        )

    @inheritdoc(DataCollector)
    def cache_hit(self, node):
        self.session["serving_node"] = node

    @inheritdoc(DataCollector)
    def cache_miss(self, node):
        self.session["cache_misses"].append(node)

    @inheritdoc(DataCollector)
    def server_hit(self, node):
        self.session["serving_node"] = node

    @inheritdoc(DataCollector)
    def request_hop(self, u, v, main_path=True):
        self.session["request_hops"].append((u, v))

    @inheritdoc(DataCollector)
    def content_hop(self, u, v, main_path=True):
        self.session["content_hops"].append((u, v))

    @inheritdoc(DataCollector)
    def end_session(self, success=True):
        self.session["success"] = success

    def session_summary(self):
        """Return a summary of latest session

        Returns
        -------
        session : dict
            Summary of session
        """
        return self.session


from django.shortcuts import render, redirect
from django.http import HttpResponse, HttpResponseBadRequest
from django.views.decorators.csrf import csrf_exempt
from .models import *
from common.models import *
import random
from django.contrib.auth import login, authenticate, logout
from django.core.exceptions import PermissionDenied
from common.decorators import logged_in_or_basicauth, require_token_permissions
import json
from django.utils import timezone
from dateutil.relativedelta import relativedelta

def update_rating(user, subject_id, value):
    Rating.objects.filter(user=user, subject_id=subject_id).delete()

    r = Rating(user=user, subject_id=subject_id, value=value)
    r.save()

@logged_in_or_basicauth
@require_token_permissions("can_view_recommendations")
def get(request):
    rec_type = request.GET.get('t', '')
    if len(rec_type) == 0:
        recs = Recommendation.objects.filter(user=request.user)
    else:
        recs = Recommendation.objects.filter(user=request.user, rec_type=rec_type)
    if recs.count() == 0:
        return HttpResponse('No recommendations yet. Try again tomorrow!')
    resp = {rec.rec_type: json.loads(rec.subjects) for rec in recs}
    return HttpResponse(json.dumps(resp), content_type="application/json")

@csrf_exempt
@logged_in_or_basicauth
@require_token_permissions("can_view_recommendations", "can_edit_student_info")
def rate(request):
    batch = request.body
    if len(batch) > 0:
        batch_items = json.loads(batch)
        for item in batch_items:
            try:
                value = int(item['v'])
                if item['s'] == None or len(item['s']) > 10:
                    return HttpResponseBadRequest('
Missing subject ID
')
                if value is None:
                    return HttpResponseBadRequest('
Missing rating value
')
                update_rating(request.user, item['s'], value)
            except:
                return HttpResponseBadRequest('
Bad input
')
        resp = { 'received': True }
    else:
        subject_id = request.POST.get('s', '')
        value = request.POST.get('v', '')
        try:
            value = int(value)
            if subject_id == None or len(subject_id) == 0 or len(subject_id) > 10:
                return HttpResponseBadRequest('
Missing subject ID
')
            if value is None:
                return HttpResponseBadRequest('
Missing rating value
')
            if request.user.username != str(user_id):
                raise PermissionDenied
            update_rating(request.user, subject_id, value)
            resp = { 'u': request.user.username, 's': subject_id, 'v': value, 'received': True }
        except:
            return HttpResponseBadRequest('
Bad input
')
    return HttpResponse(json.dumps(resp), content_type="application/json")


'''Classify - train or use a classifier'''

import bisect
import hashlib
import h5py
import logging
import numpy as np
import os
import pickle
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.decomposition import PCA as RandomizedPCA
import tempfile

import cellprofiler.module as cpm
import cellprofiler.image as cpi
import cellprofiler.setting as cps

logger = logging.getLogger(__name__)

USE_DOT = True
DEFAULT_N_ESTIMATORS = 25
DEFAULT_MIN_SAMPLES_PER_LEAF = 10
DEFAULT_RADIUS = 9
DEFAULT_N_FEATURES = 100

MODE_CLASSIFY = "Classify"
MODE_TRAIN = "Train"

AA_ADVANCED = "Advanced"
AA_AUTOMATIC = "Automatic"

G_TRAINING_SET = "TrainingSet"
DS_KERNEL = "Kernel"
DS_IMAGE_NUMBER = "ImageNumber"
DS_COORDS = "Coords"
A_VERSION = "Version"
A_CLASS = 'Class'
A_DIGEST = 'MD5Digest'
G_SAMPLING = "Sampling"
G_FILTERS = "Filters"
G_CLASSIFIERS = "Classifiers"
G_IMAGES = "Images"

CLS_GROUND_TRUTH = "GroundTruth"
CLS_KERNEL = "Kernel"
CLS_FILTER = "Filter"
CLS_CLASSIFIER = "Classifier"
CLS_SAMPLING = "Sampling"

SRC_OBJECTS = "Objects"
SRC_ILASTIK = "Ilastik"

ROUNDS = [("initial", 100000, 0),
          ("middle", 75000, 25000),
          ("final", 50000, 50000)]

class Classify(cpm.Module):
    variable_revision_number = 1
    category = "Image Processing"
    module_name = "Classify"
    def create_settings(self):
        self.mode = cps.Choice("Classify or train?",
                               [MODE_CLASSIFY, MODE_TRAIN])
        self.advanced_or_automatic = cps.Choice(
            "Configuration mode", [AA_AUTOMATIC, AA_ADVANCED],
            doc = """Do you want to automatically choose the training parameters
            or use the defaults?""")
        self.radius = cps.Integer("Radius", DEFAULT_RADIUS, 1)
        self.n_features = cps.Integer(
            "Number of features", DEFAULT_N_FEATURES, 1,
            doc = """The classifier runs a feature reduction set. This creates
            Eigentextures which are representative texture patches
            found throughout the image. The module scores each patch around
            a pixel according to how much it has each of these textures and
            those scores are fed into the final classifier. Raise the number of
            features if some of the textures or edges of your classes are
            misclassified. Lower the number of features to improve processing
            time or to reduce overfitting if you have a smaller amount of
            ground truth.
            """)
        self.n_estimators = cps.Integer(
            "Number of estimators", DEFAULT_N_ESTIMATORS, 1,
            doc = """The classifier uses a voting scheme where it trains this
            many estimators. It purposefully does a bad job training and makes
            up for this deficit by having many poor classification judges.
            This protects against overfitting by not relying on having a single
            classifier that is very good at classifying the ground truth, but
            mistakenly uses irrelevant information to do so. Raise the number
            of estimators if the classifier is making obvious mistakes with
            unwarranted certainty. Lower the number of estimators to improve
            processing speed.""")
        self.min_samples_per_leaf = cps.Integer(
            "Minimum samples per leaf", DEFAULT_MIN_SAMPLES_PER_LEAF, 1,
            doc = """This setting determines the minimum number of ground truth
            pixels that the classifier will use to split a decision tree.
            There must be at least this number of example pixels in each branch
            for the classifier to have confidence that the split is real and
            not just an artifact of an irrelevant measurement.

            Lower this setting if the classifier does a good job on most of the
            pixels but does not draw sharp distinctions between one class and
            another at the border between the classes (e.g. at the edges of
            cells). Raise this setting if the classifier misclassifies pixels
            that are clearly not the right class - this is overtraining.
            """)
        self.path = cps.DirectoryPath("Classifier folder")
        def get_directory_fn():
            '''Get the directory for the file name'''
            return self.path.get_absolute_path()
        def set_directory_fn(path):
            dir_choice, custom_path = self.path.get_parts_from_path(path)
            self.path.join_parts(dir_choice, custom_path)
        self.filename = cps.FilenameText(
            "Classifier file", "Classifier.cpclassifier",
            get_directory_fn = get_directory_fn,
            set_directory_fn = set_directory_fn,
            exts = [("Pixel classifier (*.cpclassifier)", "*.cpclassifier"),
                    ("All files (*.*)", "*.*")])
        self.gt_source = cps.Choice(
            "Ground truth source", [SRC_OBJECTS, SRC_ILASTIK],
        doc="""
        The ground truth data can either be taken from objects or can be
        the exported TIF "labels" output of Ilastik.
        """)
        self.labels_image = cps.ImageNameSubscriber(
            "Ilastik labels image", "labels.tif",
            doc="""
            Used only if the ground truth source is "Ilastik"

            This image should be the exported labels image from Ilastik.
            """)
        self.wants_background_class = cps.Binary(
            "Do you want a background class?", True)
        self.background_class_name = cps.Text("Background class name",
                                              "Background")
        self.object_classes = []
        self.object_class_count = cps.HiddenCount(self.object_classes,
                                                  "Object class count")
        self.add_objects(False)
        self.add_objects_button = cps.DoSomething(
            "Add another class", "Add", self.add_objects)
        self.label_classes = []
        self.label_class_count = cps.HiddenCount(
            self.label_classes, "Label class count")
        self.add_labels(False)
        self.add_labels_button = cps.DoSomething(
        "Add another class", "Add", self.add_labels)

        self.images = []
        self.image_count = cps.HiddenCount(self.images, "Image count")
        self.add_image(False)
        self.add_image_button = cps.DoSomething(
            "Add another image", "Add", self.add_image)
        self.outputs = []
        self.output_count = cps.HiddenCount(self.outputs, "Output count")
        self.add_output(False)
        self.add_output_button = cps.DoSomething(
            "Add another output", "Add", self.add_output)

    def get_class_names(self, ignore=None):
        result = []
        if self.mode == MODE_TRAIN:
            if self.gt_source == SRC_OBJECTS:
                if self.wants_background_class:
                    result.append(self.background_class_name.value)
                result += [group.object_name.value
                           for group in self.object_classes]
            else:
                result += [group.class_name.value
                           for group in self.label_classes]
        else:
            try:
                with self.get_classifier("r") as c:
                    return c.get_class_names()
            except IOError:
                result.append("None")
        return result

    def add_objects(self, can_remove=True):
        group = cps.SettingsGroup()
        group.append("object_name",  cps.ObjectNameSubscriber(
            "Object name", "Nuclei"))
        if can_remove:
            group.append("remover", cps.RemoveSettingButton(
                "Remove object", "Remove", self.object_classes, group))
        self.object_classes.append(group)

    def add_labels(self, can_remove=True):
        group = cps.SettingsGroup()
        group.append("class_name", cps.AlphanumericText(
            "Class name", "Class %d" % (len(self.label_classes)+1),
            doc="""
            The name to give to pixels of this class (e.g. "Foreground")

            You should add one class for each class you defined in Ilastik"""))
        if can_remove:
            group.append("remover", cps.RemoveSettingButton(
                "Remove object", "Remove", self.label_classes, group))
        self.label_classes.append(group)

    def add_image(self, can_remove=True):
        group = cps.SettingsGroup()
        group.append("image_name", cps.ImageNameSubscriber(
            "Image name", "DNA"))
        if can_remove:
            group.append("remover", cps.RemoveSettingButton(
                "Remove object", "Remove", self.images, group))
        self.images.append(group)

    def add_output(self, can_remove=True):
        group = cps.SettingsGroup()
        group.append(
            "output_image",
            cps.ImageNameProvider("Output image", "Probability"))

        group.append("class_name", cps.Choice(
            "Class name", choices=self.get_class_names(),
            choices_fn = self.get_class_names))
        if can_remove:
            group.append("remover", cps.RemoveSettingButton(
                "Remove object", "Remove", self.outputs, group))
        self.outputs.append(group)

    def settings(self):
        result = [
            self.object_class_count, self.label_class_count,
            self.image_count, self.output_count,
            self.mode, self.path, self.filename,
            self.advanced_or_automatic,
            self.radius, self.n_features, self.n_estimators,
            self.min_samples_per_leaf, self.gt_source,
            self.labels_image,
            self.wants_background_class, self.background_class_name]
        for group in self.object_classes:
            result += group.pipeline_settings()
        for group in self.label_classes:
            result += group.pipeline_settings()
        for group in self.images:
            result += group.pipeline_settings()
        for group in self.outputs:
            result += group.pipeline_settings()
        return result

    def visible_settings(self):
        result = [self.mode, self.path, self.filename]
        if self.mode == MODE_TRAIN:
            result.append(self.advanced_or_automatic)
            if self.advanced_or_automatic == AA_ADVANCED:
                result += [self.radius, self.n_features, self.n_estimators,
                           self.min_samples_per_leaf]
        for group in self.images:
            result += group.visible_settings()
        result.append(self.add_image_button)
        if self.mode == MODE_TRAIN:
            self.filename.mode = cps.FilenameText.MODE_OVERWRITE
            result.append(self.gt_source)
            if self.gt_source == SRC_OBJECTS:
                result.append(self.wants_background_class)
                if self.wants_background_class:
                    result.append(self.background_class_name)
                for group in self.object_classes:
                    result += group.visible_settings()
                result.append(self.add_objects_button)
            else:
                result.append(self.labels_image)
                for group in self.label_classes:
                    result += group.visible_settings()
                result.append(self.add_labels_button)
        else:
            self.filename.mode = cps.FilenameText.MODE_OPEN
            for group in self.outputs:
                result += group.visible_settings()
            result.append(self.add_output_button)
        return result

    def prepare_settings(self, setting_values):
        for count, sequence, add_fn in zip(
            [int(_) for _ in setting_values[:4]],
            (self.object_classes, self.label_classes, self.images, self.outputs),
            (self.add_objects, self.add_labels, self.add_image, self.add_output)):
            del sequence[:]
            for idx in range(count):
                add_fn()

    def is_aggregation_module(self):
        return self.mode == MODE_TRAIN

    def get_classifier(self, mode):
        path = os.path.join(self.path.get_absolute_path(), self.filename.value)
        return PixelClassifier(path, mode)

    def get_radius(self):
        if self.advanced_or_automatic == AA_AUTOMATIC:
            return DEFAULT_RADIUS
        return self.radius.value

    def get_n_features(self):
        if self.advanced_or_automatic == AA_AUTOMATIC:
            return DEFAULT_N_FEATURES
        return self.n_features.value

    def get_n_estimators(self):
        if self.advanced_or_automatic == AA_AUTOMATIC:
            return DEFAULT_N_ESTIMATORS
        return self.n_estimators.value

    def get_min_samples_per_leaf(self):
        if self.advanced_or_automatic == AA_AUTOMATIC:
            return DEFAULT_MIN_SAMPLES_PER_LEAF
        return self.min_samples_per_leaf.value

    def prepare_group(self, workspace, grouping, image_numbers):
        if self.mode == MODE_TRAIN:
            with self.get_classifier("w") as c:
                assert isinstance(c, PixelClassifier)
                r = self.get_radius()
                i, j = np.mgrid[-r:r+1, -r:r+1]
                kernel_mask = i*i + j*j <= r*r
                n_features = np.sum(kernel_mask)
                kernel = np.vstack([
                    np.column_stack([
                        np.ones(n_features, int) * channel, # C offset
                        np.zeros(n_features, int), # T offset
                        np.zeros(n_features, int), # Z offset
                        i[kernel_mask],
                        j[kernel_mask]])
                    for channel in range(len(self.images))])
                c.set_kernel(kernel)
                for class_name in self.get_class_names():
                    c.add_class(class_name)

    def run(self, workspace):
        if self.mode == MODE_TRAIN:
            self.run_train(workspace)
        else:
            self.run_classify(workspace)

    def get_5d_image(self, workspace):
        '''Compile a 5d image from the channel planes'''
        pixels = []
        for group in self.images:
            image_name = group.image_name.value
            img = workspace.image_set.get_image(
                image_name, must_be_grayscale=True).pixel_data
            #
            # [[img]] adds Z and T of dimension 1 to the uber array
            #
            pixels.append([[img]])
        return np.array(pixels)

    def run_train(self, workspace):
        pixels = self.get_5d_image(workspace)
        image_number = workspace.measurements.image_number
        with self.get_classifier("a") as c:
            assert isinstance(c, PixelClassifier)
            c.add_image(pixels, image_number)
            gt = []
            if self.gt_source == SRC_OBJECTS:
                bg = np.ones(pixels.shape[-2:], bool)
                for group in self.object_classes:
                    object_name = group.object_name.value
                    objects = workspace.object_set.get_objects(object_name)
                    fg = np.zeros(bg.shape, bool)
                    for plane, _ in objects.get_labels():
                        fg[plane > 0] = True
                    bg[fg] = False
                    gt.append((object_name, fg))
                if self.wants_background_class:
                    gt.append((self.background_class_name.value, bg))
            else:
                label_image_name = self.labels_image.value
                img = workspace.image_set.get_image(label_image_name)
                pixel_data = (img.pixel_data * img.scale).astype(int)
                for idx, group in enumerate(self.label_classes):
                    class_name = group.class_name.value
                    gt.append((class_name, pixel_data == idx+1))
            for object_name, fg in gt:
                i, j = np.where(fg)
                c.add_ground_truth(
                    object_name,
                    image_number,
                    np.column_stack([np.zeros(len(i), int)] * 3 + [i, j]))

    def post_group(self, workspace, grouping):
        if self.mode == MODE_TRAIN:
            last_round_name = None
            for round_name, n_random, n_error in ROUNDS:
                self.do_training_round(
                    last_round_name, round_name,
                    n_random / len(self.get_class_names()),
                    n_error / len(self.get_class_names()))
            with self.get_classifier("a") as c:
                assert isinstance(c, PixelClassifier)
                c.config_final_pipeline("final", "final")

    def do_training_round(
        self, name_in, name_out, n_random_samples, n_error_samples):
        '''Perform a round of training

        name_in - name of the filter bank and classifier to use to find the
                  error samples. None if no error samples.
        name_out - name for output classifier
        n_random_samples - # of samples randomly drawn from each class
        n_error_samples - # of samples drawn from errors
        '''
        with self.get_classifier("a") as c:
            assert isinstance(c, PixelClassifier)
            #
            # Sample
            #
            fb_sample_name = name_out + "_filter_bank"
            classifier_sample_name = name_out + "_classifier"
            for sample_name in fb_sample_name, classifier_sample_name:
                d = {}
                for idx, class_name in enumerate(c.get_class_names()):
                    gt = c.get_ground_truth(class_name)
                    if name_in is not None and n_error_samples > 0:
                        probs = c.run_pipeline(name_in, name_in, gt)[:, idx]
                        order = np.argsort(probs)
                        error_idx = order[:n_error_samples]
                        other_idx = order[n_error_samples:]
                    else:
                        error_idx = np.zeros(0, int)
                        other_idx = np.arange(gt.shape[0])
                    if len(other_idx) > n_random_samples:
                        r = c.random_state(str(name_in)+name_out+class_name)
                        other_idx = r.choice(
                            other_idx, size=n_random_samples, replace=False)
                    sample_idx = np.hstack((error_idx, other_idx))
                    d[class_name] = sample_idx
                c.add_sampling(sample_name, d)
            samples, classes = c.sample(fb_sample_name)
            c.make_filter_bank(
                samples, classes, name_out, self.get_n_features())
            samples, classes = c.sample(classifier_sample_name)
            filtered = c.use_filter_bank(name_out, samples)
            algorithm = ExtraTreesClassifier(
                n_estimators = self.get_n_estimators(),
                min_samples_leaf=self.get_min_samples_per_leaf())
            c.fit(name_out, filtered, classes, algorithm)

    def run_classify(self, workspace):
        pixels = self.get_5d_image(workspace)
        #
        # Process the image in chunks
        #
        with self.get_classifier("r") as c:
            assert isinstance(c, PixelClassifier)
            class_names = c.get_class_names()
            prob_idxs = np.array([
                class_names.index(group.class_name.value)
                for group in self.outputs])
            chunk_size = 128
            prob_maps = np.zeros(
                (len(prob_idxs), pixels.shape[3], pixels.shape[4]))
            for i in range(0, pixels.shape[3], chunk_size):
                iend = min(i+chunk_size, pixels.shape[3])
                for j in range(0, pixels.shape[4], chunk_size):
                    jend = min(j+chunk_size, pixels.shape[4])
                    ii, jj = [_.flatten() for _ in np.mgrid[i:iend, j:jend]]
                    coords = np.column_stack((
                        np.zeros(len(ii), int),
                        np.zeros(len(ii), int),
                        np.zeros(len(ii), int),
                        ii, jj))
                    samples = c.get_samples(pixels, coords)
                    probs = c.run_final_pipeline(samples)
                    prob_maps[:, i:iend, j:jend] = \
                        probs[:, prob_idxs].reshape(
                            iend - i, jend - j, len(prob_maps)).transpose(2, 0, 1)
        for i, group in enumerate(self.outputs):
            image_name = group.output_image.value
            image = cpi.Image(prob_maps[i])
            workspace.image_set.add(image_name, image)
        if self.show_window:
            workspace.display_data.input_images = [
                pixels[i].reshape(*pixels.shape[-2:])
                for i in range(pixels.shape[0])]
            workspace.display_data.output_images = [
                prob_maps[i] for i in range(len(prob_maps))]

    def display(self, workspace, figure):
        if self.mode == MODE_CLASSIFY:
            figure.set_subplots((2, max(len(self.images), len(self.outputs))))
            for i, (group, image) in enumerate(
                zip(self.images, workspace.display_data.input_images)):
                figure.subplot_imshow_bw(0, i, image,
                                         title = group.image_name.value)
            for i, (group, image) in enumerate(
                zip(self.outputs, workspace.display_data.output_images)):
                figure.subplot_imshow_bw(1, i, image,
                                         title = group.output_image.value)

#
# The classifier class is here mostly to simplify the initial development
# and it would get moved out to Centrosome eventually.
#
class PixelClassifier(object):
    '''Represents a classifier stored in an HDF5 file

    The parts:

    Kernel - this is the patch that's taken from the pixel's neighborhood. It
             has the shape, NxM where N is the # of points in the patch and
             M are the indices relative to the pixel. The kernel dimensions
             are C, T, Z, Y, X.
    TrainingSet - the ground truth data. Each label has a corresponding
             dataset stored using the name of the class. The dataset has the
             shape, S x N, where N is the # of points in the patch and S is
             the # of samples for that class.
    Filters - the filters applied to the kernel to derive features. Each
              filter is a vector of length N.
    Classifier - the classifier is pickled after being trained and is stored
                 in a dataset.
    '''
    version = 1
    def __init__(self, path, mode, classifier_path = None):
        '''Either create or load the classifier from a file

        path - path to the file
        mode - "r" for read-only, "w" for overwrite (new file) or "a" for
               read-write access
        classifier_path - path to the sub-groups within the HDF file, defaults
                          to the root.
        '''
        self.f = h5py.File(path, mode)
        self.root = \
            self.f if classifier_path is None else self.f[classifier_path]
        if mode == "w":
            self.f.attrs[A_VERSION] = self.version
            self.g_training_set = self.root.create_group(G_TRAINING_SET)
            self.g_filters = self.root.create_group(G_FILTERS)
            self.g_sampling = self.root.create_group(G_SAMPLING)
            self.g_classifiers = self.root.create_group(G_CLASSIFIERS)
            self.g_images = self.root.create_group(G_IMAGES)
        else:
            self.g_training_set = self.root[G_TRAINING_SET]
            self.g_filters = self.root[G_FILTERS]
            self.g_sampling = self.root[G_SAMPLING]
            self.g_classifiers = self.root[G_CLASSIFIERS]
            self.g_images = self.root[G_IMAGES]
        self.classifier_cache = {}

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        self.f.flush()
        del self.root
        del self.g_training_set
        del self.g_filters
        del self.g_classifiers
        del self.g_sampling
        self.f.close()
        del self.f

    def random_state(self, extra = ''):
        '''Return a random state based on the ground truth sampled

        '''
        if A_DIGEST not in self.g_training_set.attrs:
            md5 = hashlib.md5()
            for class_name in self.get_class_names():
                md5.update(self.get_ground_truth(class_name).value.data)
            self.g_training_set.attrs[A_DIGEST] = md5.hexdigest()
        return np.random.RandomState(np.frombuffer(
            self.g_training_set.attrs[A_DIGEST] + extra, np.uint8))

    @staticmethod
    def get_instances(group, class_name):
        '''Get the keys for a group for objects of a given class'''
        return sorted([k for k in group.keys()
                       if group[k].attrs[A_CLASS] == class_name])

    def set_kernel(self, kernel):
        '''Set the kernel used to sample pixels from a neighborhood

        kernel - an N x 5 matrix of N offsets by 5 dimensions (C, T, Z, Y, X)
        '''
        if DS_KERNEL in self.root.keys():
            del self.root[DS_KERNEL]
        ds = self.root.create_dataset(DS_KERNEL, data=kernel)
        ds.attrs[A_CLASS] = CLS_KERNEL

    def get_kernel(self):
        return self.root[DS_KERNEL][:]

    def add_class(self, class_name):
        ds = self.g_training_set.create_dataset(
            class_name,
            shape = (0, 6),
            dtype = np.int32,
            chunks = (4096, 6),
            maxshape = (None, 6))
        ds.attrs[A_CLASS] = CLS_GROUND_TRUTH

    def get_class_names(self):
        '''Get the names of the classifier's classes'''
        return self.get_instances(self.g_training_set, CLS_GROUND_TRUTH)

    def get_ground_truth(self, class_name):
        '''Get the ground truth for a class

        class_name - the name of the class

        returns an S X 6 where the first index is the image number and
        the remaining are the coordinates of the GT pixel in C, T, Z, Y, X form
        '''
        return self.g_training_set[class_name]

    @property
    def gt_chunk_size(self):
        '''The size of a chunk of ground truth that fits in memory'''
        kernel_size = self.get_kernel().shape[0]
        chunk_size = int(50*1000*1000 / kernel_size)
        return chunk_size

    @property
    def pix_chunk_size(self):
        return 1000 * 1000

    def add_image(self, image, image_number):
        image_number = str(image_number)
        if image_number in self.g_images.keys():
            del self.g_images[image_number]
        self.g_images.create_dataset(image_number, data = image)

    def get_image(self, image_number):
        image_number = str(image_number)
        return self.g_images[image_number].value

    def add_ground_truth(self, class_name, image_number, coordinates):
        '''Add ground truth to a class

        class_name - name of the class
        image_number - the image number as reported in add_image
        pixels - an S x 5 matrix of S samples and 5 pixel coordinates
        '''
        coordinates = np.column_stack(
            (np.ones(coordinates.shape[0], coordinates.dtype) * image_number,
             coordinates))
        ds = self.get_ground_truth(class_name)
        ds_idx = ds.shape[0]
        ds.resize(ds_idx + coordinates.shape[0], axis = 0)
        ds[ds_idx:] = coordinates

        if A_DIGEST in self.g_training_set.attrs:
            del self.g_training_set.attrs[A_DIGEST]

    def get_samples(self, image, pixels):
        '''Extract samples from an image at given pixels

        image - a C, T, Z, Y, X image

        pixels - an S x 5 matrix where the columns are the C, T, Z, Y, X
                 coordinates and the rows are the samples to collect

        returns an S x N matrix where N is the size of the kernel
        '''
        kernel = self.get_kernel()[np.newaxis, :, :]
        coords = pixels[:, np.newaxis, :] + kernel
        #
        # Boundary reflection
        #
        coords[coords < 0] = np.abs(coords[coords < 0])
        for i, axis_size in enumerate(image.shape):
            mask = coords[:, :, i] >= axis_size
            coords[mask, i] = axis_size * 2 - coords[mask, i] - 1
        #
        # Samples
        #
        samples = image[coords[:, :, 0],
                        coords[:, :, 1],
                        coords[:, :, 2],
                        coords[:, :, 3],
                        coords[:, :, 4]]
        return samples

    def add_sampling(self, sampling_name, d_index):
        '''Add a sampling of the ground truth

        sampling_name - a name for the sampling
        d_index - a dictionary of indices. The key is the class name and
                  the value is a vector of indices into the class's ground truth
        '''
        if sampling_name in self.g_sampling.keys():
            del self.g_sampling[sampling_name]
        g = self.g_sampling.create_group(sampling_name)
        for k, v in d_index.iteritems():
            g.create_dataset(k, data=v)

    def sample(self, sampling_name):
        '''Return sample and vector of class indexes

        sampling_name - the name of the sampling

        Returns a sample which is S x N and vector of length S which is
        composed of indexes into the class names returned by get_class_names.
        S is the length of the sum of all samples in the sampling
        '''
        g = self.g_sampling[sampling_name]
        samples = []
        classes = []
        for idx, class_name in enumerate(self.get_class_names()):
            if class_name in g.keys():
                sampling = g[class_name][:]
                classes.append(np.ones(len(sampling), np.uint8) * idx)
                gt = self.get_ground_truth(class_name)
                #
                # h5py datasets are not addressable via an array of indices
                # in the way that numpy arrays are. A mask of the array elements
                # to be processed is handled by a loop through each selected
                # element; it takes ~ hours to process arrays of our size. The
                # ground truth may be too large to bring into memory as a
                # Numpy array.
                #
                # We sort the sampling indices and then process in chunks.
                #
                if len(gt) == len(sampling):
                    logger.debug("Extracting %d samples from %s" %
                                 (len(sampling), class_name))
                    samples.append(gt[:])
                else:
                    chunk_size = self.pix_chunk_size
                    sampling.sort()
                    sindx = 0
                    for gtidx in range(0, len(gt), chunk_size):
                        gtidx_end = min(gtidx+chunk_size, len(gt))
                        if sampling[sindx] >= gtidx_end:
                            continue
                        sindx_end = bisect.bisect_left(
                            sampling[sindx:], gtidx_end) + sindx
                        logger.debug(
                            "Extracting %d samples from %s %d:%d" %
                            (sindx_end-sindx, class_name, gtidx, gtidx_end))
                        samples.append(gt[:][sampling[sindx:sindx_end], :])
                        sindx = sindx_end
                        if sindx >= len(gt):
                            break
        samples = np.vstack(samples)
        classes = np.hstack(classes)
        #
        # Order by image number.
        #
        order = np.lexsort(
            [samples[:, _] for _ in reversed(range(samples.shape[1]))])
        samples = samples[order]
        classes = classes[order]
        counts = np.bincount(samples[:, 0])
        image_numbers = np.where(counts > 0)[0]
        counts = counts[image_numbers]
        idxs = np.hstack([[0], np.cumsum(counts)])
        result = []
        for image_number, idx, idx_end in zip(
            image_numbers, idxs[:-1], idxs[1:]):
            image = self.get_image(image_number)
            result.append(self.get_samples(image, samples[idx:idx_end, 1:]))

        return np.vstack(result), classes

    def make_filter_bank(self, sampling, classes, filter_bank_name, n_filters,
                         algorithm=None):
        '''Make a filter bank using PCA

        sampling - a sampling of the ground truth
        classes - a vector of the same length as the sampling giving the
                  indexes of the classes of each sample
        filter_bank_name - the name to assign to the filter bank
        n_filters - # of filters to create

        algorithm - an object that can be fitted using algorithm.fit(X, Y)
                    and can transform using algorithm.transform(X). Default
                    is RandomizedPCA.
        '''
        if algorithm is None:
            r = self.random_state(filter_bank_name)
            algorithm = RandomizedPCA(n_filters,
                                      random_state = r)
        algorithm.fit(sampling, classes)
        if hasattr(algorithm, "components_"):
            components = algorithm.components_
            if len(components) > n_filters:
                components = components[:n_filters]
            if filter_bank_name in self.g_filters.keys():
                del self.g_filters[filter_bank_name]
            ds = self.g_filters.create_dataset(filter_bank_name,
                                               data = components)
            ds.attrs[A_CLASS] = CLS_FILTER
        else:
            s = pickle.dumps(algorithm)
            ds = self.g_filters.create_dataset(filter_bank_name,
                                               data = s)
            ds.attrs[A_CLASS] = CLS_CLASSIFIER

    def use_filter_bank(self, filter_bank_name, sample):
        '''Transform a sample using a filter bank'''
        ds = self.g_filters[filter_bank_name]
        if ds.attrs[A_CLASS] == CLS_FILTER:
            if USE_DOT:
                result = np.dot(sample, ds[:].T)
            else:
                #
                # A dot product... but cluster's np.dot is so XXXXed
                #
                chunk_size = self.gt_chunk_size
                result = []
                for idx in range(0, len(sample), chunk_size):
                    idx_end = min(idx + chunk_size, len(sample))
                    logger.debug("Processing dot product chunk %d:%d of %d" %
                                 (idx, idx_end, len(sample)))
                    result.append(np.sum(sample[idx:idx_end, :, np.newaxis] *
                                         ds[:].T[np.newaxis, :, :], 1))
                result = np.vstack(result)
        else:
            algorithm = pickle.loads(ds.value)
            result = algorithm.transform(sample)
        return result

    def fit(self, classifier_name, sample, classes, algorithm=None):
        '''Fit an algorithm to data and save

        classifier_name - save using this name
        sample - S samples x N features
        classes - S class labels indexing into the class names
        algorithm - algorithm to use to train
        '''
        if algorithm is None:
            algorithm = ExtraTreesClassifier(
                n_estimators=N_ESTIMATORS,
                min_samples_leaf = MIN_SAMPLES_PER_LEAF)
        algorithm.fit(sample, classes)
        s = pickle.dumps(algorithm)
        if classifier_name in self.g_classifiers.keys():
            del self.g_classifiers[classifier_name]
        ds = self.g_classifiers.create_dataset(classifier_name, data = s)
        ds.attrs[A_CLASS] = CLS_CLASSIFIER

    def predict_proba(self, classifier_name, sample):
        if classifier_name not in self.classifier_cache:
            algorithm = pickle.loads(self.g_classifiers[classifier_name].value)
            self.classifier_cache[classifier_name] = algorithm
        else:
            algorithm = self.classifier_cache[classifier_name]
        return algorithm.predict_proba(sample)

    def run_pipeline(self, filter_bank_name, classifier_name, sample):
        filtered = self.use_filter_bank(filter_bank_name, sample)
        return self.predict_proba(classifier_name, filtered)

    def config_final_pipeline(self, filter_bank_name, classifier_name):
        self.root.attrs["FilterBankName"] = filter_bank_name
        self.root.attrs["ClassifierName"] = classifier_name

    def run_final_pipeline(self, sample):
        return self.run_pipeline(self.root.attrs["FilterBankName"],
                                 self.root.attrs["ClassifierName"],
                                 sample)

if __name__ == "__main__":
    logging.basicConfig()
    logging.root.setLevel(logging.DEBUG)
    import cellprofiler.pipeline as cpp
    import cellprofiler.preferences as cpprefs
    cpprefs.set_default_output_directory("c:/temp/output/classify")
    pipeline = cpp.Pipeline()
    pipeline.load("c:/temp/output/classify/classify.cpproj")
    module = pipeline.modules()[-1]
    module.post_group(None, None)
    from cellprofiler.utilities.cpjvm import cp_stop_vm
    cp_stop_vm()


from argparse import ArgumentParser, Namespace
from concurrent.futures import ThreadPoolExecutor
from contextlib import nullcontext, redirect_stderr, redirect_stdout
from io import StringIO
from multiprocessing import cpu_count
from pathlib import Path
from subprocess import DEVNULL, STDOUT, CalledProcessError, run
from sys import executable, exit, stderr, version_info
from textwrap import dedent
from typing import Union

from .consts import IS_WIN, REQUIREMENTS, RT_DIR, RT_PY, TOP_LEVEL, VARS

try:
    from shlex import join
    from typing import Literal

    if version_info < (3, 8, 2):
        raise ImportError()
except ImportError:
    print("⛔ python < 3.8.2", file=stderr)
    exit(1)

def parse_args() -> Namespace:
    parser = ArgumentParser()

    sub_parsers = parser.add_subparsers(dest="command", required=True)

    with nullcontext(sub_parsers.add_parser("run")) as p:
        p.add_argument("--socket", required=True)
        p.add_argument("--xdg")

    with nullcontext(sub_parsers.add_parser("deps")) as p:
        p.add_argument("--xdg")

    return parser.parse_args()

args = parse_args()
command: Union[Literal["deps"], Literal["run"]] = args.command

_XDG = Path(args.xdg) if args.xdg is not None else None

_RT_DIR = _XDG / "coqrt" if _XDG else RT_DIR
_RT_PY = (
    (_RT_DIR / "Scripts" / "python.exe" if IS_WIN else _RT_DIR / "bin" / "python3")
    if _XDG
    else RT_PY
)
_LOCK_FILE = _RT_DIR / "requirements.lock"
_EXEC_PATH = Path(executable)
_EXEC_PATH = _EXEC_PATH.parent.resolve() / _EXEC_PATH.name
_REQ = REQUIREMENTS.read_text()

_IN_VENV = _RT_PY == _EXEC_PATH

if command == "deps":
    assert not _IN_VENV

    try:
        from venv import EnvBuilder

        print("...", flush=True)
        with redirect_stdout(StringIO()), redirect_stderr(StringIO()):
            EnvBuilder(
                system_site_packages=False,
                with_pip=True,
                upgrade=True,
                symlinks=not IS_WIN,
                clear=True,
            ).create(_RT_DIR)
    except (ImportError, CalledProcessError):
        msg = "Please install python3-venv separately. (apt, yum, apk, etc)"
        print(msg, file=stderr)
        exit(1)
    else:
        proc = run(
            (
                _RT_PY,
                "-m",
                "pip",
                "install",
                "--upgrade",
                "pip",
            ),
            cwd=TOP_LEVEL,
            stdin=DEVNULL,
            stderr=STDOUT,
        )
        if proc.returncode:
            print("Installation failed, check :message", file=stderr)
            exit(proc.returncode)
        proc = run(
            (
                _RT_PY,
                "-m",
                "pip",
                "install",
                "--upgrade",
                "--force-reinstall",
                "--requirement",
                str(REQUIREMENTS),
            ),
            cwd=TOP_LEVEL,
            stdin=DEVNULL,
            stderr=STDOUT,
        )
        if proc.returncode:
            print("Installation failed, check :message", file=stderr)
            exit(proc.returncode)
        else:
            _LOCK_FILE.write_text(_REQ)
            msg = """
            ---
            You can now use :COQnow
            """
            print(dedent(msg), file=stderr)

elif command == "run":
    try:
        lock = _LOCK_FILE.read_text()
    except Exception:
        lock = ""
    try:
        if not _IN_VENV:
            raise ImportError()
        elif lock != _REQ:
            raise ImportError()
        else:
            import pynvim
            import pynvim_pp
            import std2
            import yaml
    except ImportError:
        msg = f"""
        Please update dependencies using :COQdeps
        -
        -
        Dependencies will be installed privately inside `{VARS}`
        `{join(("rm", "-rf", str(TOP_LEVEL.name)))}` will cleanly remove everything
        """
        msg = dedent(msg)
        print(msg, end="", file=stderr)
        exit(1)
    else:
        from pynvim import attach
        from pynvim_pp.client import run_client

        from .client import CoqClient

        nvim = attach("socket", path=args.socket)

        with ThreadPoolExecutor(max_workers=min(16, cpu_count() + 10)) as pool:
            code = run_client(nvim, pool=pool, client=CoqClient(pool=pool))
            exit(code)

else:
    assert False