from typing import Optional
from absl import logging
from intrinsic.icon.equipment import icon_equipment_pb2
from intrinsic.icon.python import icon_api
from intrinsic.skills.proto import equipment_pb2
from intrinsic.util.grpc import connection

ICON2_POSITION_PART_KEY = "Icon2PositionPart"
ICON2_TORQUE_PART_KEY = "Icon2TorquePart"
ICON2_GRIPPER_PART_KEY = "Icon2GripperPart"
ICON2_ADIO_PART_KEY = "Icon2AdioPart"
ICON2_FORCE_TORQUE_SENSOR_PART_KEY = "Icon2ForceTorqueSensorPart"
ICON2_RANGEFINDER_PART_KEY = "Icon2RangefinderPart"

def make_icon_equipment_selector(
    with_position_controlled_part: bool = False,
    with_torque_controlled_part: bool = False,
    with_gripper_part: bool = False,
    with_adio_part: bool = False,
    with_force_torque_sensor_part: bool = False,
    with_rangefinder_part: bool = False,
    with_observation_stream: bool = False,
) -> equipment_pb2.EquipmentSelector:
  """Creates an equipment selector for an ICON equipment handle with specific parts.

  Args:
    with_position_controlled_part: If true, requires a position-controlled part.
    with_torque_controlled_part: If true, requires a torque-controlled part.
    with_gripper_part: If true, requires a gripper part.
    with_adio_part: If true, requires an adio part.
    with_force_torque_sensor_part: If true, requires a force_torque_sensor part.
    with_rangefinder_part: If true, requires a rangefinder part.
    with_observation_stream: If true, requires an observation stream config.

  Returns:
    A populated equipment selector.
  """

  equipment_type_names = ["Icon2Connection"]
  if with_position_controlled_part:
    equipment_type_names.append(ICON2_POSITION_PART_KEY)
  if with_torque_controlled_part:
    equipment_type_names.append(ICON2_TORQUE_PART_KEY)
  if with_gripper_part:
    equipment_type_names.append(ICON2_GRIPPER_PART_KEY)
  if with_adio_part:
    equipment_type_names.append(ICON2_ADIO_PART_KEY)
  if with_force_torque_sensor_part:
    equipment_type_names.append(ICON2_FORCE_TORQUE_SENSOR_PART_KEY)
  if with_rangefinder_part:
    equipment_type_names.append(ICON2_RANGEFINDER_PART_KEY)
  if with_observation_stream:
    equipment_type_names.append("IconRobotObservationStreamParams")

  return equipment_pb2.EquipmentSelector(
      equipment_type_names=equipment_type_names
  )

def _get_params_from_connection_info(
    equipment_handle: equipment_pb2.EquipmentHandle,
) -> Optional[connection.ConnectionParams]:
  """Creates params to start an ICON client from a handle's connection_info.

  Args:
    equipment_handle: The equipment handle for the robot we want to control.

  Returns:
    The connected ConnectionParams if the handle provides enough information.
  """
  if not equipment_handle.HasField("connection_info"):
    return None
  if not equipment_handle.connection_info.HasField("grpc"):
    return None
  return connection.ConnectionParams(
      equipment_handle.connection_info.grpc.address,
      equipment_handle.connection_info.grpc.server_instance,
      equipment_handle.connection_info.grpc.header,
  )

def init_icon_client(
    equipment_handle: equipment_pb2.EquipmentHandle,
) -> icon_api.Client:
  """Creates a client for talking to the icon server.

  Args:
    equipment_handle: The equipment handle for the robot we want to control.

  Returns:
    The connected Client.

  Raises:
    KeyError: If equipment_handle does not include any connection config.
  """
  connection_params = _get_params_from_connection_info(equipment_handle)

  if not connection_params:
    raise KeyError("No ICON connection config was provided.")

  logging.info("ICON connection_params: %s", connection_params)
  return icon_api.Client.connect_with_params(connection_params, insecure=True)

def get_position_part_name(
    equipment_handle: equipment_pb2.EquipmentHandle,
) -> str:
  """Gets the name of the Icon2PositionPart from the equipment data.

  Args:
    equipment_handle: The equipment handle for the robot we want to control.

  Returns:
    The part name.

  Raises:
    KeyError: If equipment_handle does not include an Icon2PositionPart data.
  """
  icon_position_part = icon_equipment_pb2.Icon2PositionPart()
  pos_key = ICON2_POSITION_PART_KEY
  if pos_key not in equipment_handle.equipment_data:
    raise KeyError(
        "%s is not in equipment_handle.equipment_data. Available: %r"
        % (pos_key, equipment_handle.equipment_data.keys())
    )
  equipment_handle.equipment_data[pos_key].contents.Unpack(icon_position_part)
  logging.info("ICON position_part: %s", icon_position_part)
  return icon_position_part.part_name

def get_torque_part_name(
    equipment_handle: equipment_pb2.EquipmentHandle,
) -> str:
  """Gets the name of the Icon2TorquePart from the equipment data.

  Args:
    equipment_handle: The equipment handle for the robot we want to control.

  Returns:
    The part name.

  Raises:
    KeyError: If equipment_handle does not include an Icon2TorquePart data.
  """
  icon_torque_part = icon_equipment_pb2.Icon2TorquePart()
  torque_key = ICON2_TORQUE_PART_KEY
  if torque_key not in equipment_handle.equipment_data:
    raise KeyError(
        "%s is not in equipment_handle.equipment_data. Available: %r"
        % (torque_key, equipment_handle.equipment_data.keys())
    )
  equipment_handle.equipment_data[torque_key].contents.Unpack(icon_torque_part)
  logging.info("ICON torque_part: %s", icon_torque_part)
  return icon_torque_part.part_name

def get_gripper_part_name(
    equipment_handle: equipment_pb2.EquipmentHandle,
) -> str:
  """Gets the name of the Icon2GripperPart from the equipment data.

  Args:
    equipment_handle: The equipment handle for the robot we want to control.

  Returns:
    The part name.

  Raises:
    KeyError: If equipment_handle does not include an Icon2GripperPart data.
  """
  icon_gripper_part = icon_equipment_pb2.Icon2GripperPart()
  gripper_key = ICON2_GRIPPER_PART_KEY
  if gripper_key not in equipment_handle.equipment_data:
    raise KeyError(
        "%s is not in equipment_handle.equipment_data. Available: %r"
        % (gripper_key, equipment_handle.equipment_data.keys())
    )
  equipment_handle.equipment_data[gripper_key].contents.Unpack(
      icon_gripper_part
  )
  logging.info("ICON gripper_part: %s", icon_gripper_part)
  return icon_gripper_part.part_name

def get_force_torque_sensor_part_name(
    equipment_handle: equipment_pb2.EquipmentHandle,
) -> str:
  """Gets the name of the Icon2ForceTorqueSensorPart from the equipment data.

  Args:
    equipment_handle: The equipment handle for the robot we want to control.

  Returns:
    The part name.

  Raises:
    KeyError: If equipment_handle does not include an Icon2ForceTorqueSensorPart
    data.
  """
  icon_force_torque_sensor_part = (
      icon_equipment_pb2.Icon2ForceTorqueSensorPart()
  )
  ft_key = ICON2_FORCE_TORQUE_SENSOR_PART_KEY
  if ft_key not in equipment_handle.equipment_data:
    raise KeyError(
        "%s is not in equipment_handle.equipment_data. Available: %r"
        % (ft_key, equipment_handle.equipment_data.keys())
    )
  equipment_handle.equipment_data[ft_key].contents.Unpack(
      icon_force_torque_sensor_part
  )
  logging.info(
      "ICON force_torque_sensor_part: %s", icon_force_torque_sensor_part
  )
  return icon_force_torque_sensor_part.part_name

def get_adio_part_name(equipment_handle: equipment_pb2.EquipmentHandle) -> str:
  """Gets the name of the Icon2AdioPart from the equipment data.

  Args:
    equipment_handle: The equipment handle for the adio we want to read/control.

  Returns:
    The part name.

  Raises:
    KeyError: If equipment_handle does not include an Icon2AdioPart data.
    ValueError: If adio_part does not include an icon_target data, i.e. a adio
      configuration that uses a different backend than icon.
  """
  adio_key = ICON2_ADIO_PART_KEY
  icon_adio_part = icon_equipment_pb2.Icon2AdioPart()
  if adio_key not in equipment_handle.equipment_data:
    raise KeyError(
        "%s is not in equipment_handle.equipment_data. Available: %r"
        % (adio_key, equipment_handle.equipment_data.keys())
    )
  equipment_handle.equipment_data[adio_key].contents.Unpack(icon_adio_part)
  if not icon_adio_part.HasField("icon_target"):
    raise ValueError(
        "adio part is not an icon_target and thus does not have a part name."
    )
  return icon_adio_part.icon_target.part_name

def get_rangefinder_part_name(
    equipment_handle: equipment_pb2.EquipmentHandle,
) -> str:
  """Gets the name of the Icon2RangefinderPart from the equipment data.

  Args:
    equipment_handle: The equipment handle for the robot we want to control.

  Returns:
    The part name.

  Raises:
    KeyError: If equipment_handle does not include Icon2RangefinderPart data.
  """
  icon_rangefinder_part = icon_equipment_pb2.Icon2RangefinderPart()
  part_key = ICON2_RANGEFINDER_PART_KEY
  if part_key not in equipment_handle.equipment_data:
    raise KeyError(
        "%s is not in equipment_handle.equipment_data. Available: %r"
        % (part_key, equipment_handle.equipment_data.keys())
    )
  if not equipment_handle.equipment_data[part_key].contents.Unpack(
      icon_rangefinder_part
  ):
    raise ValueError(
        "Failed to unpack %s as 'Icon2RangefinderPart'. Actual type is '%r'"
        % (part_key, icon_rangefinder_part.DESCRIPTOR)
    )

  logging.info("ICON %s: %s", ICON2_RANGEFINDER_PART_KEY, icon_rangefinder_part)
  return icon_rangefinder_part.part_name


from Exercise import MoneyModel
import matplotlib.pyplot as plt
plt.show()

starter_model = MoneyModel(10)
for i in range(5):
    starter_model.step()

agent_wealth = [a.wealth for a in starter_model.schedule.agents]
# Create a histogram with seaborn
g = sns.histplot(agent_wealth, discrete=True)
g.set(title="Wealth distribution", xlabel="Wealth", ylabel="Number of agents");
 # The semicolon is just to avoid printing the object representation


# Imports
import numpy as np
import cv2
from skimage.filters import threshold_multiotsu

def multiThreshold(img, margin):
    # Denoise image
    img = cv2.fastNlMeansDenoising(img)

    # Calculate two threshold values
    thresholds = threshold_multiotsu(img)

    # If the distance between the threshold is below a margin, consider there to be two classes
    if (thresholds[1] - thresholds[0]) < margin:
        ret, segmented = cv2.threshold(img, 2, 2, cv2.THRESH_BINARY+cv2.THRESH_OTSU)
        segmented = np.where(segmented == 0, 1, 2)

    # If not, consider there to be three classes
    else:
        segmented = np.digitize(img, bins=thresholds).astype(np.uint8)
    return segmented


#!/usr/bin/env python
import sys
sys.path.insert(0, "../..")
import matplotlib.pyplot as plt
import time
from planning_python.environment_interface.env_2d import Env2D
from planning_python.state_lattices.common_lattice.xy_analytic_lattice import XYAnalyticLattice
from planning_python.cost_functions.cost_function import PathLengthNoAng
from planning_python.heuristic_functions.heuristic_function import EuclideanHeuristicNoAng, ManhattanHeuristicNoAng
from planning_python.data_structures.planning_problem import PlanningProblem
from planning_python.planners.astar import Astar
import os

#Step 1: Load environment from file
envfile = os.path.abspath("../../databases/bugtrap.png")
env_params = {'x_lims': [0, 100], 'y_lims': [0,100]}
e = Env2D()
e.initialize(envfile, env_params)

#Step 2: Create lattice to overlay on environment
lattice_params = dict()
lattice_params['x_lims']          = [0, 100] # Used to calculate number of cells in lattice (should ideally be consistent with env_params['x_lims'])
lattice_params['y_lims']          = [0, 100] # Used to calculate number of cells in lattice (should ideally be consistent with env_params['y_lims'])
lattice_params['resolution']      = [1, 1]   # Used to calculate number of cells in lattice + conversion from discrete to continuous space and vice-versa
lattice_params['origin']          = (0, 0)   # Used for conversion from discrete to continuous and vice-versa.
lattice_params['rotation']        = 0        # Used for conversion from discrete to continuous and vice-versa (This plus origin define lattice-->world transform)
lattice_params['connectivity']    = 'eight_connected' #Lattice connectivity (can be four or eight connected for xylattice)
lattice_params['path_resolution'] = 1      #Resolution for defining edges and doing collision checking (in meters)

l = XYAnalyticLattice(lattice_params)

#Step 3: Create cost and heuristic objects
cost_fn = PathLengthNoAng()                   #Penalize length of path
heuristic_fn = EuclideanHeuristicNoAng()

#Step 4: Create a planning problem
prob_params = {'heuristic_weight': 1}        #Planner is not greedy at all
start_n = l.state_to_node((0,0))
goal_n = l.state_to_node((100, 100))
prob = PlanningProblem(e, l, cost_fn, heuristic_fn, prob_params, start_n, goal_n, visualize=True)

#Step 4: Create Planner object and ask it to solve the planning problem
planner = Astar(prob)
plan_start_time = time.time()
path, path_cost, num_expansions, came_from, cost_so_far, c_obs = planner.plan()
planning_time = time.time() - plan_start_time
print('Path: ', path)
print('Path Cost: ', path_cost)
print('Number of Expansions: ', num_expansions)
print('Time taken: ', planning_time)

plt.show()


import mimetypes
import posixpath

from compiled_file_system import SingleFile
from directory_zipper import DirectoryZipper
from docs_server_utils import ToUnicode
from file_system import FileNotFoundError
from future import Gettable, Future
from path_canonicalizer import PathCanonicalizer
from path_util import AssertIsValid, Join, ToDirectory
from special_paths import SITE_VERIFICATION_FILE
from third_party.handlebar import Handlebar
from third_party.markdown import markdown

_MIMETYPE_OVERRIDES = {
  # SVG is not supported by mimetypes.guess_type on AppEngine.
  '.svg': 'image/svg+xml',
}

class ContentAndType(object):
  '''Return value from ContentProvider.GetContentAndType.
  '''

  def __init__(self, content, content_type):
    self.content = content
    self.content_type = content_type

class ContentProvider(object):
  '''Returns file contents correctly typed for their content-types (in the HTTP
  sense). Content-type is determined from Python's mimetype library which
  guesses based on the file extension.

  Typically the file contents will be either str (for binary content) or
  unicode (for text content). However, HTML files *may* be returned as
  Handlebar templates (if |supports_templates| is True on construction), in
  which case the caller will presumably want to Render them.

  Zip file are automatically created and returned for .zip file extensions if
  |supports_zip| is True.

  |default_extensions| is a list of file extensions which are queried when no
  file extension is given to GetCanonicalPath/GetContentAndType.  Typically
  this will include .html.
  '''

  def __init__(self,
               name,
               compiled_fs_factory,
               file_system,
               object_store_creator,
               default_extensions=(),
               supports_templates=False,
               supports_zip=False):
    # Public.
    self.name = name
    self.file_system = file_system
    # Private.
    self._content_cache = compiled_fs_factory.Create(file_system,
                                                     self._CompileContent,
                                                     ContentProvider)
    self._path_canonicalizer = PathCanonicalizer(file_system,
                                                 object_store_creator,
                                                 default_extensions)
    self._default_extensions = default_extensions
    self._supports_templates = supports_templates
    if supports_zip:
      self._directory_zipper = DirectoryZipper(compiled_fs_factory, file_system)
    else:
      self._directory_zipper = None

  @SingleFile
  def _CompileContent(self, path, text):
    assert text is not None, path
    _, ext = posixpath.splitext(path)
    mimetype = _MIMETYPE_OVERRIDES.get(ext, mimetypes.guess_type(path)[0])
    if ext == '.md':
      # See http://pythonhosted.org/Markdown/extensions
      # for details on "extensions=".
      content = markdown(ToUnicode(text),
                         extensions=('extra', 'headerid', 'sane_lists'))
      if self._supports_templates:
        content = Handlebar(content, name=path)
      mimetype = 'text/html'
    elif mimetype is None:
      content = text
      mimetype = 'text/plain'
    elif mimetype == 'text/html':
      content = ToUnicode(text)
      if self._supports_templates:
        content = Handlebar(content, name=path)
    elif (mimetype.startswith('text/') or
          mimetype in ('application/javascript', 'application/json')):
      content = ToUnicode(text)
    else:
      content = text
    return ContentAndType(content, mimetype)

  def GetCanonicalPath(self, path):
    '''Gets the canonical location of |path|. This class is tolerant of
    spelling errors and missing files that are in other directories, and this
    returns the correct/canonical path for those.

    For example, the canonical path of "browseraction" is probably
    "extensions/browserAction.html".

    Note that the canonical path is relative to this content provider i.e.
    given relative to |path|. It does not add the "serveFrom" prefix which
    would have been pulled out in ContentProviders, callers must do that
    themselves.
    '''
    AssertIsValid(path)
    base, ext = posixpath.splitext(path)
    if self._directory_zipper and ext == '.zip':
      # The canonical location of zip files is the canonical location of the
      # directory to zip + '.zip'.
      return self._path_canonicalizer.Canonicalize(base + '/').rstrip('/') + ext
    return self._path_canonicalizer.Canonicalize(path)

  def GetContentAndType(self, path):
    '''Returns the ContentAndType of the file at |path|.
    '''
    AssertIsValid(path)
    base, ext = posixpath.splitext(path)

    # Check for a zip file first, if zip is enabled.
    if self._directory_zipper and ext == '.zip':
      zip_future = self._directory_zipper.Zip(ToDirectory(base))
      return Future(delegate=Gettable(
          lambda: ContentAndType(zip_future.Get(), 'application/zip')))

    # If there is no file extension, look for a file with one of the default
    # extensions.
    #
    # Note that it would make sense to guard this on Exists(path), since a file
    # without an extension may actually exist, but it's such an uncommon case
    # it hardly seems worth the potential performance hit.
    if not ext:
      for default_ext in self._default_extensions:
        if self.file_system.Exists(path + default_ext).Get():
          path += default_ext
          break

    return self._content_cache.GetFromFile(path)

  def Cron(self):
    futures = [self._path_canonicalizer.Cron()]
    for root, _, files in self.file_system.Walk(''):
      for f in files:
        futures.append(self.GetContentAndType(Join(root, f)))
        # Also cache the extension-less version of the file if needed.
        base, ext = posixpath.splitext(f)
        if f != SITE_VERIFICATION_FILE and ext in self._default_extensions:
          futures.append(self.GetContentAndType(Join(root, base)))
      # TODO(kalman): Cache .zip files for each directory (if supported).
    return Future(delegate=Gettable(lambda: [f.Get() for f in futures]))

  def __repr__(self):
    return 'ContentProvider of <%s>' % repr(self.file_system)


from typing import Union
from magma.bit import Bit
from magma.logging import root_logger
from magma.primitives.mux import mux
from magma.t import Type, In, Out, Direction
from magma.protocol_type import MagmaProtocol
from magma.tuple import Product, ProductKind

_logger = root_logger()

class ReadyValidException(Exception):
    pass

def _check_T(cls, T):
    if not issubclass(T, (Type, MagmaProtocol)):
        raise TypeError(
            f"{cls}[T] expected T to be a subclass of m.Type or"
            f" m.MagmaProtocol not {T}"
        )
    undirected_T = T.undirected_t
    if undirected_T is not T:
        _logger.warning(
            f"Type {T} used with ReadyValid is not undirected, converting"
            " to undirected type"
        )
    return undirected_T

def _maybe_add_data(cls, fields, T, qualifier):
    if T is None:
        return
    T = _check_T(cls, T)
    fields["data"] = qualifier(T)

class HandShakeError(Exception):
    pass

class EnqError(HandShakeError):
    def __init__(self, cls):
        super().__init__(f"{cls} does not support enq")

class NoEnqError(HandShakeError):
    def __init__(self, cls):
        super().__init__(f"{cls} does not support no_enq")

class DeqError(HandShakeError):
    def __init__(self, cls):
        super().__init__(f"{cls} does not support deq")

class NoDeqError(HandShakeError):
    def __init__(self, cls):
        super().__init__(f"{cls} does not support no_deq")

def _enq_error(self, value, when=True):
    raise EnqError(type(self))

def _no_enq_error(self, value, when=True):
    raise NoEnqError(type(self))

def _deq_error(self, value, when=True):
    raise DeqError(type(self))

def _no_deq_error(self, value, when=True):
    raise NoDeqError(type(self))

class HandShakeKind(ProductKind):
    """Undirected handshake type."""
    _qualifers_ = {
        "output": lambda x: x,
        "input": lambda x: x,
        "data": lambda x: x
    }

    def _make_fields(cls, T):
        """
        Each HandShake implementation should define a dict of _field_names_ for
        the signals (e.g. "ready" and "valid").  For consistency, we define the
        output/input names in terms of a Producer (i.e. ReadyValid outputs a
        valid and inputs a ready).

        Each implementation should define a dict of _qualifiers_ used to
        qualify the ports for producer, consumer, etc...
        """
        fields = {
            cls._field_names_["handshake_output"]:
                cls._qualifers_["output"](Bit),
            cls._field_names_["handshake_input"]:
                cls._qualifers_["input"](Bit)
        }
        _maybe_add_data(cls, fields, T, cls._qualifers_["data"])
        return fields

    def __getitem__(cls, T: Union[Type, MagmaProtocol]):
        assert cls._field_names_ is not None, "Undefined field name impl"
        fields = cls._make_fields(T)

        bases = (cls, )
        if hasattr(cls, "Base_T"):
            # Qualified variants include the unqualified type as a base class.
            # e.g. Producer(ReadyValid[Bits[8]]) will be a subtype of
            # ReadyValid[Bits[8]].
            bases += (cls.Base_T[T], )
        name = f"{cls}[{T}]"

        # Populates namespace for proper cache behavior (based on
        # hwtypes.adt_meta logic).
        ns = {}
        ns['__module__'] = cls.__module__
        name = f'{cls}[{T}]'
        ns['__qualname__'] = name

        # _cache_handler is implemented in hwtypes.adt_meta.
        return cls._cache_handler(True, fields, name, bases, ns)

    def __new__(mcs, name, bases, namespace, fields=None, **kwargs):
        cls = super().__new__(mcs, name, bases, namespace, fields, **kwargs)

        if not cls._field_names_:
            # Skip for base class (variants already exist and pointer is not
            # needed).
            return cls

        variants = (HandShakeMonitor, HandShakeDriver, HandShakeConsumer,
                    HandShakeProducer)
        if issubclass(cls, variants):
            return cls  # these don't need variants

        # Metaprogram variants for base type (e.g. for ReadyValid[Bits[8]],
        # create Producer(ReadyValid[Bits[8]])).
        for variant in variants:
            # Strip "HandShake" prefix for string name.
            suffix = f"{variant.__name__}"[len("HandShake"):]

            # Make variant type with appropriate base classes.
            variant_T = type(f"{suffix}({cls.__name__})",
                             (variant, cls), {})

            # Store pointer to variant (used by qualifiers).
            setattr(cls, f"{suffix}_T", variant_T)
            variant_T.Base_T = cls  # store pointer to base type

        return cls

    def flip(cls):
        raise TypeError(f"Cannot flip {cls.__name__} (undirected)")

    def qualify(cls, direction):
        if direction is Direction.In:
            return Monitor(cls)
        if direction is Direction.Out:
            return Driver(cls)
        if direction is Direction.Undirected:
            return Undirected(cls)
        raise TypeError(f"Cannot qualify {cls.__name__} with {direction}")

    @property
    def undirected_data_t(cls):
        try:
            data = cls.field_dict["data"]
        except KeyError:
            return None
        return data.undirected_t

    def __str__(cls):
        return cls.__name__

class HandShake(Product, metaclass=HandShakeKind):
    _field_names_ = None  # must be defined by implementation

    def fired(self):
        """
        Returns a bit that is high when ready and valid are both high.
        """
        raise NotImplementedError

class HandShakeProducerKind(HandShakeKind):
    _qualifers_ = {"output": Out, "input": In, "data": Out}

    def __getitem__(cls, T: Union[Type, MagmaProtocol]):
        t = super().__getitem__(T)
        if T is None:
            # Remove deq/no_deq methods if no data.
            t.deq = _deq_error
            t.no_deq = _no_deq_error
        return t

    def flip(cls):
        return Consumer(cls)

class HandShakeConsumerKind(HandShakeKind):
    _qualifers_ = {"output": In, "input": Out, "data": In}

    def __getitem__(cls, T: Union[Type, MagmaProtocol]):
        t = super().__getitem__(T)
        if T is None:
            t.enq = _enq_error
            t.no_enq = _no_enq_error
        return t

    def flip(cls):
        return Producer(cls)

class HandShakeConsumer(HandShake, metaclass=HandShakeConsumerKind):
    def fired(self):
        if not self.valid.driven():
            raise ReadyValidException(
                "Cannot invoke fired on consumer when valid isn't driven"
            )

        return self.ready & self.valid.value()

    def enq(self, value, when=True):
        """
        Produces the data `value` when `when` is high.

        Uses "last connect semantics", which allows multiple `enq` methods to
        be invoked with different conditions with precedence given to the later
        invocations.
        """
        if when is not True:
            if not (self.valid.driven() and self.data.driven()):
                raise ReadyValidException(
                    "Cannot use enq when without default driver"
                )
            curr_valid = self.valid.value()
            when |= curr_valid
            self.valid.unwire(curr_valid)

            curr_data = self.data.value()
            value = mux([curr_data, value], when)
            self.data.unwire(curr_data)
        self.valid @= when
        self.data @= value
        return value

    def no_enq(self, when=True):
        """
        Indicates no enqueue occurs.  Valid is set to false, and bits are
        connected to 0. (TODO: Should be an unitialized wire (Don't Care))
        """
        if when is not True:
            if not (self.valid.driven() and self.data.driven()):
                raise ReadyValidException(
                    "Cannot use no_deq with condition without an existing"
                    "valid and data driver"
                )
            valid = self.valid.value()
            data = self.data.value()
            self.valid.unwire(valid)
            self.data.unwire(data)
            self.valid @= mux([valid, False], when)
            self.data @= mux([data, 0], when)
        else:
            self.valid @= False
            self.data @= 0

class HandShakeProducer(HandShake, metaclass=HandShakeProducerKind):
    def fired(self):
        if not self.ready.driven():
            raise ReadyValidException(
                "Cannot invoke fired on producer when ready isn't driven"
            )
        return self.ready.value() & self.valid

    def deq(self, when=True):
        """
        Assert ready on this port and return associated data when `when` is
        high.

        This is typically used when valid has been asserted on the producer
        side.
        """
        if when is not True:
            if not self.ready.driven():
                raise ReadyValidException(
                    "Cannot use deq with when without a default ready driver"
                )
            ready = self.ready.value()
            self.ready.unwire(ready)
            self.ready @= ready | when
        else:
            self.ready @= when
        return self.data

    def no_deq(self, when=True):
        """
        Indicates no dequeue occurs, ready is set to False.
        """
        if when is not True:
            if not self.ready.driven():
                raise ReadyValidException(
                    "Cannot use no_deq with condition without an existing"
                    " ready driver"
                )
            ready = self.ready.value()
            self.ready.unwire(ready)
            self.ready @= mux([ready, False], when)
        else:
            self.ready @= False

class HandShakeMonitorKind(HandShakeKind):
    _qualifers_ = {"output": In, "input": In, "data": In}

    def flip(cls):
        return Driver(cls)

class HandShakeMonitor(HandShake, metaclass=HandShakeMonitorKind):
    pass

class HandShakeDriverKind(HandShakeKind):
    _qualifers_ = {"output": Out, "input": Out, "data": Out}

    def flip(cls):
        return Monitor(cls)

class HandShakeDriver(HandShake, metaclass=HandShakeDriverKind):
    pass

class ReadyValid(HandShake):
    """
    An I/O Bundle containing 'valid' and 'ready' signals that handshake
    the transfer of data stored in the 'data' subfield.

    The base protocol is undirected and is meant to be used with the modifiers:
    * m.Consumer(m.ReadyValid[T])  # data/valid are inputs, ready is output
    * m.Producer(m.ReadyValid[T])  # data/valid are outputs, ready is input

    T is the type of data to be wrapped in Ready/Valid.

    To create a ReadyValid type with no data, use T=None.  In this case, there
    will be no data port.

    The actual semantics of ready/valid are enforced via the use of concrete
    subclasses.
    """
    _field_names_ = {"handshake_output": "valid", "handshake_input": "ready"}

class Decoupled(ReadyValid):
    """
    `valid` indicates the prdoucer has put valid data in `data`, and `ready`
    indicates that the consumer is ready to accept the data this cycle.  No
    requirements are made on the signaling of ready or valid.
    """
    # TODO: Add down convert logic from Irrevocable consumer to Decoupled,
    # dropping guarantees of irrevocability.  Cannot be used on producer
    pass

class EnqIO(Decoupled.Producer_T):
    """
    Alias for Producer(Decoupled[T]).

    drives (outputs) `valid` and `data`, inputs `ready`.
    """
    pass

class DeqIO(Decoupled.Consumer_T):
    """
    Alias for Consumer(Decoupled[T]).

    drives (outputs) `ready`, inputs `valid` and `data`.
    """
    pass

class Irrevocable(ReadyValid):
    """
    Promises not to change the value of `data` after a cycle where `valid` is
    high and `ready` is low.  Additionally, once `valid` is raised, it will
    never be lowered until after `ready` has also been raised.
    """
    # TODO: Add upconvert logic from Decoupled producer to Irrevocable (can use
    # Irrevocable where a Decoupled is expected). Cannot be used on consumer.
    pass

class CreditValid(HandShake):
    _field_names_ = {"handshake_output": "valid", "handshake_input": "credit"}

def Consumer(T: HandShakeKind):
    if not issubclass(T, HandShake):
        raise TypeError(f"Consumer({T}) is unsupported")
    undirected_T = T.undirected_data_t
    return T.Consumer_T[undirected_T]

def Producer(T: HandShakeKind):
    if not issubclass(T, HandShake):
        raise TypeError(f"Producer({T}) is unsupported")
    undirected_T = T.undirected_data_t
    return T.Producer_T[undirected_T]

def Monitor(T: HandShakeKind):
    if not issubclass(T, HandShake):
        raise TypeError(f"Monitor({T}) is unsupported")
    undirected_T = T.undirected_data_t
    return T.Monitor_T[undirected_T]

def Driver(T: HandShakeKind):
    if not issubclass(T, HandShake):
        raise TypeError(f"Driver({T}) is unsupported")
    undirected_T = T.undirected_data_t
    return T.Driver_T[undirected_T]

def Undirected(T: HandShakeKind):
    if not issubclass(T, HandShake):
        raise TypeError(f"Undirected({T}) is unsupported")
    if not hasattr(T, 'Base_T'):
        return T  # already undirected
    undirected_T = T.undirected_data_t
    return T.Base_T[undirected_T]

def is_producer(T):
    return isinstance(T, (HandShakeProducerKind, HandShakeProducer))

def is_consumer(T):
    return isinstance(T, (HandShakeConsumerKind, HandShakeConsumer))

# TODO: QueueIO and Queue


from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from keras.utils import to_categorical
import numpy as np

PerfilI = np.zeros((1, 1000))

idade = np.random.randint(25,89, size=(1, 1000))
Iendiv = np.random.randint(70, 100, size=(1, 1000))
Poup = np.random.randint(9, size=(1, 1000))
Liqui = np.random.randint(81,100, size=(1, 1000))
Cobert = np.random.randint(30, size=(1, 1000))/100
Riqueza = np.random.randint(30, size=(1, 1000))
Score = np.random.randint(499, size=(1, 1000))
Faixa_Salarial  = np.random.randint(1045,2090, size=(1, 1000))
TempoEmpresa= np.random.randint(12, size=(1, 1000))
list = (idade, Iendiv, Poup, Liqui, Cobert, Riqueza, Score, Faixa_Salarial, TempoEmpresa)
for i in list:
    PerfilI= np.vstack((PerfilI,i))
PerfilI = PerfilI.T

PerfilR = np.ones((1, 1000))
idade = np.random.randint(25,89, size=(1, 1000))
Iendiv = np.random.randint(20, 70, size=(1, 1000))
Poup = np.random.randint(10,20, size=(1, 1000))
Liqui = np.random.randint(60, 80, size=(1, 1000))
Cobert = np.random.randint(30, 180, size=(1, 1000))
Riqueza = np.random.randint(30, 50, size=(1, 1000))/100
Score = np.random.randint(500, 700, size=(1, 1000))
Faixa_Salarial  = np.random.randint(2090,5225, size=(1, 1000))
TempoEmpresa= np.random.randint(12, 36, size=(1, 1000))
list = (idade, Iendiv, Poup, Liqui, Cobert, Riqueza, Score, Faixa_Salarial, TempoEmpresa)
for i in list:
    PerfilR= np.vstack((PerfilR,i))
PerfilR = PerfilR.T

PerfilBP = np.ones((1, 1000))+1
idade = np.random.randint(25,89, size=(1, 1000))
Iendiv = np.random.randint(20, size=(1, 1000))
Poup = np.random.randint(20, 100, size=(1, 1000))
Liqui = np.random.randint(60, size=(1, 1000))
Cobert = np.random.randint(72, 288, size=(1, 1000))
Riqueza = np.random.randint(50, 100, size=(1, 1000))/100
Score = np.random.randint(700,1000, size=(1, 1000))
Faixa_Salarial  = np.random.randint(6270, 31350, size=(1, 1000))
TempoEmpresa= np.random.randint(36, 240, size=(1, 1000))
list = (idade, Iendiv, Poup, Liqui, Cobert, Riqueza, Score, Faixa_Salarial, TempoEmpresa)
for i in list:
    PerfilBP= np.vstack((PerfilBP,i))
PerfilBP = PerfilBP.T

PerfilGeral = np.zeros((1,10))
PerfilGeral= np.vstack((PerfilGeral,PerfilI))
PerfilGeral= np.vstack((PerfilGeral,PerfilR))
PerfilGeral= np.vstack((PerfilGeral,PerfilBP))
PerfilGeral = PerfilGeral[1:]
print(PerfilGeral)

X = PerfilGeral [:,1:]
y = PerfilGeral[:,:1]
y = y.astype(int)
y = to_categorical(y)

# Dividindo o dataset em dados de treino e de teste.
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 100)

# Normalização dos dados.
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)

#  EXECUÇÃO DA REDE PARA OBTENÇÃO DE UM MODELO(APREDIZAGEM) PARA PREDIÇÕES
classifier = Sequential()
classifier.add(Dense(units = 22, kernel_initializer = 'uniform', activation = 'relu', input_dim = 9))
classifier.add(Dropout(p = 0.1))
classifier.add(Dense(units = 22, kernel_initializer = 'uniform', activation = 'relu'))
# classifier.add(Dropout(p = 0.1))
classifier.add(Dense(units = 3, kernel_initializer = 'uniform', activation = 'softmax'))
classifier.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])
classifier.fit(X_train, y_train, batch_size = 32, epochs = 100)

#MELHORIAS PARA AUMENTAR A ACURÁCIA DA REDE

#implementação da rede com Keras e novas "configurações"
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import cross_val_score
def build_classifieres():
    classifier = Sequential()
    classifier.add(Dense(units = 12, kernel_initializer = 'uniform', activation = 'relu', input_dim = 9))
    classifier.add(Dense(units = 12, kernel_initializer = 'uniform', activation = 'relu'))
    classifier.add(Dense(units = 3, kernel_initializer = 'uniform', activation = 'softmax'))
    classifier.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])
    return classifier
classifieres = KerasClassifier(build_fn = build_classifieres, batch_size = 32, epochs = 100)
accuracies = cross_val_score(estimator = classifieres, X = X_train, y = y_train, cv = 10, n_jobs = -1)
a= accuracies.mean()
print(a)

# FAZENDO UMA PREDIÇÃO SIMPLES, DE UM CLIENTE FICTÍCIO, APÓS O TREINAMENTO DO REDE.
print(' ')
print(' ')
print('PREDIÇÃO DO PERFIL DO CLIENTE')
print(' ')
Client_01 = classifier.predict(sc.transform(np.array([[66,51,10,66,104,0.33,645,3997,25]])))
for i in Client_01:
    if i[0] > 0.5:
            print('Cliente JOÃO possui: PERFIL O1')
            print('............')
            print('LIBERAR CRÉDTIO DE ATÉ 1.000 REAIS')
    if i[1] > 0.5:
            print('Cliente JOÃO possui: PERFIL O2')
            print('............')
            print('LIBERAR CRÉDTIO DE ATÉ 5.000 REAIS')
    if i[2] > 0.5:
            print('Cliente JOÃO possui: PERFIL O1')
            print('............')
            print('LIBERAR CRÉDTIO DE ATÉ 20.000 REAIS')


from mmcv.runner import IterBasedRunner, RUNNERS
import torch
import numpy as np
from torch import nn
import torch.nn.functional as F
from torch.nn import Parameter
import math
from typing import (Any, Callable, Dict, List, Optional, Tuple, Union,
                    no_type_check)
import logging

def gkern2d(l=21, sig=3):
    """Returns a 2D Gaussian kernel array."""
    ax = np.arange(-l // 2 + 1., l // 2 + 1.)
    xx, yy = np.meshgrid(ax, ax)
    kernel = np.exp(-(xx ** 2 + yy ** 2) / (2. * sig ** 2))
    return kernel / (1.0 / (2 * math.pi * (sig ** 2)))

# This class is partially adapted from https://github.com/anlcnydn/bilateral/blob/master/bilateral_filter.py
# And modified to work with multiple channels
class Shift(nn.Module):
    def __init__(self, kernel_size=3):
        super(Shift, self).__init__()
        self.kernel_size = kernel_size
        self.pad = self.kernel_size//2

    def forward(self, x):
        n, c, h, w = x.size()
        x_pad = F.pad(x, (self.pad, self.pad, self.pad, self.pad))
        x_pad = x_pad.reshape(n,1,c,h+self.pad*2,w+self.pad*2)

        cat_layers = []
        for y in range(0,self.kernel_size):
            y2 = y+h
            for x in range(0, self.kernel_size):
                x2 = x+w
                xx = x_pad[:,:,:,y:y2,x:x2]
                cat_layers += [xx]
        return torch.cat(cat_layers, 1)

# This class is partially adapted from https://github.com/anlcnydn/bilateral/blob/master/bilateral_filter.py
# And modified to work with multiple channels
class BilateralFilterModule(nn.Module):
    r"""BilateralFilter computes:
        If = 1/W * Sum_{xi C Omega}(I * f(||I(xi)-I(x)||) * g(||xi-x||))
    """

    def __init__(self, channels=3, height=480, width=640, k=7, sigma_space=5, sigma_color=0.1):
        super(BilateralFilterModule, self).__init__()

        self.g = Parameter(torch.Tensor(channels,k*k))
        self.gw = gkern2d(k,sigma_space)
        self.k = k
        gw = np.tile(self.gw.reshape(1,k*k,1,1),(1,1,height,width))
        self.g.data = torch.from_numpy(gw).float()
        #shift
        self.shift = Shift(k)
        self.norm_term_color = (1.0 / (2 * math.pi * (sigma_color ** 2)))
        self.sigma_color = 2*sigma_color**2

    def forward(self, I):
        n, c, h, w = I.size()
        #Is contains the original image mutiple shifted
        Is = self.shift(I).data
        Iex = I.expand(*Is.size())
        D = ((Is-Iex)**2).sum(2)
        g_i = torch.exp(-D / self.sigma_color) * self.norm_term_color
        g_i_s = g_i * self.g.data
        W_denom = torch.sum(g_i_s,dim=1)
        g_i_s = g_i_s.reshape(n, self.k*self.k, 1, h, w).expand((n, self.k*self.k, c, h, w))
        If = torch.sum(g_i_s*Is,dim=1) / W_denom
        return If

class BilateralFilterTorch(object):
    def __init__(self, prob=0.5, max_blur=15, img_size=None, channels=3):
        self.prob = prob
        self.max_blur = max_blur
        self.filter = []
        h,w = img_size
        for k in range(max_blur//2):
            f = BilateralFilterModule(channels,h,w, k=k*2+1, sigma_space=75.0, sigma_color=75.0/255.0)
            f.cuda()
            self.filter.append(f)

    def bilateral_filter(self, img):
        with torch.no_grad():
            c,h,w = img.shape
            img = img.reshape(1, c, h, w)
            img = img.to('cuda')
            blur = np.random.choice(self.max_blur//2)
            color = self.filter[blur](img)
            img_out = color[0]
            return img_out

    def __call__(self, results):
        results["bilateral_filter"] = False
        if np.random.random() < self.prob:
            results["img"] = self.bilateral_filter(results["img"])
            results["bilateral_filter"] = True
            return results
        return results

@RUNNERS.register_module()
class BilateralFilterRunner(IterBasedRunner):
    def __init__(self,
                 model: torch.nn.Module,
                 batch_processor: Optional[Callable] = None,
                 optimizer: Union[Dict, torch.optim.Optimizer, None] = None,
                 work_dir: Optional[str] = None,
                 logger: Optional[logging.Logger] = None,
                 meta: Optional[Dict] = None,
                 max_iters: Optional[int] = None,
                 max_epochs: Optional[int] = None,
                 prob=0.5, max_blur=15,
                 img_size=None, channels=3) -> None:
        super(BilateralFilterRunner, self).__init__(model, batch_processor,
                                                    optimizer, work_dir, logger,
                                                    meta, max_iters, max_epochs)
        self.filter = BilateralFilterTorch(prob, max_blur, img_size, channels)

    def filter_batch(self, batch):
        for i, img_metas in enumerate(batch["img_metas"].data[0]):
            res_dict = dict()
            res_dict["img"] = batch["img"].data[0][i]
            res_dict = self.filter(res_dict)
            batch["img_metas"].data[0][i]["bilateral_filter"] = res_dict["bilateral_filter"]
            batch["img"].data[0][i] = res_dict["img"]
        return batch

    def train(self, data_loader, **kwargs):
        self.model.train()
        self.mode = 'train'
        self.data_loader = data_loader
        self._epoch = data_loader.epoch
        data_batch = next(data_loader)
        data_batch = self.filter_batch(data_batch)
        self.data_batch = data_batch
        self.call_hook('before_train_iter')
        outputs = self.model.train_step(data_batch, self.optimizer, **kwargs)
        if not isinstance(outputs, dict):
            raise TypeError('model.train_step() must return a dict')
        if 'log_vars' in outputs:
            self.log_buffer.update(outputs['log_vars'], outputs['num_samples'])
        self.outputs = outputs
        self.call_hook('after_train_iter')
        del self.data_batch
        self._inner_iter += 1
        self._iter += 1


"""
=======================
01. Plot firing pattern
=======================

This example demonstrates how to inspect the firing
pattern of cells in the HNN model.
"""

# Authors: Mainak Jas <mjas@harvard.mgh.edu>
#          Nick Tolley <nick nicholas_tolley@brown.edu>

import os.path as op
import tempfile

###############################################################################
# Let us import ``hnn_core``.

import hnn_core
from hnn_core import read_spikes, jones_2009_model, simulate_dipole

###############################################################################
# Now let's build the network. We have used the same weights as in the
# :ref:`evoked example <sphx_glr_auto_examples_plot_simulate_evoked.py>`.
import matplotlib.pyplot as plt

net = jones_2009_model()

###############################################################################
# ``net`` does not have any driving inputs and only defines the local network
# connectivity. Let us go ahead and first add a distal evoked drive.
# We need to define the AMPA and NMDA weights for the connections. An
# "evoked drive" defines inputs that are normally distributed with a certain
# mean and standard deviation.

weights_ampa_d1 = {'L2_basket': 0.006562, 'L2_pyramidal': 7e-6,
                   'L5_pyramidal': 0.142300}
weights_nmda_d1 = {'L2_basket': 0.019482, 'L2_pyramidal': 0.004317,
                   'L5_pyramidal': 0.080074}
synaptic_delays_d1 = {'L2_basket': 0.1, 'L2_pyramidal': 0.1,
                      'L5_pyramidal': 0.1}
net.add_evoked_drive(
    'evdist1', mu=63.53, sigma=3.85, numspikes=1, weights_ampa=weights_ampa_d1,
    weights_nmda=weights_nmda_d1, location='distal',
    synaptic_delays=synaptic_delays_d1, event_seed=274)

###############################################################################
# The reason it is called an "evoked drive" is it can be used to simulate
# waveforms resembling evoked responses. Here, we show how to do it with two
# proximal drives which drive current up the dendrite and one distal drive
# which drives current down the dendrite producing the negative deflection.
weights_ampa_p1 = {'L2_basket': 0.08831, 'L2_pyramidal': 0.01525,
                   'L5_basket': 0.19934, 'L5_pyramidal': 0.00865}
synaptic_delays_prox = {'L2_basket': 0.1, 'L2_pyramidal': 0.1,
                        'L5_basket': 1., 'L5_pyramidal': 1.}

# all NMDA weights are zero; pass None explicitly
net.add_evoked_drive(
    'evprox1', mu=26.61, sigma=2.47, numspikes=1, weights_ampa=weights_ampa_p1,
    weights_nmda=None, location='proximal',
    synaptic_delays=synaptic_delays_prox, event_seed=544)

###############################################################################
# Now we add the second proximal evoked drive and simulate the network
# dynamics with somatic voltage recordings enabled. Note: only AMPA weights
# differ from first.
weights_ampa_p2 = {'L2_basket': 0.000003, 'L2_pyramidal': 1.438840,
                   'L5_basket': 0.008958, 'L5_pyramidal': 0.684013}
# all NMDA weights are zero; omit weights_nmda (defaults to None)
net.add_evoked_drive(
    'evprox2', mu=137.12, sigma=8.33, numspikes=1,
    weights_ampa=weights_ampa_p2, location='proximal',
    synaptic_delays=synaptic_delays_prox, event_seed=814)

dpls = simulate_dipole(net, tstop=170., record_vsec='soma')

###############################################################################
# Here, we explain more details about the data structures and how they can
# be used to better interpret the data. The cell IDs (gids) uniquely define
# neurons in the network and are stored in the :class:`~hnn_core.Network`
# object as a dictionary
gid_ranges = net.gid_ranges
print(net.gid_ranges)

###############################################################################
# Simulated voltage in the soma and other cell sections are stored in
# :class:`~hnn_core.CellResponse` as a dictionary. The CellResponse object
# stores data produced by individual cells including spikes,
# voltages and currents.
trial_idx = 0
vsec = net.cell_response.vsec[trial_idx]
print(vsec.keys())

###############################################################################
# We can plot the firing pattern of individual cells by indexing with the gid
gid = 170
plt.figure(figsize=(4, 4), constrained_layout=True)
plt.plot(net.cell_response.times, vsec[gid]['soma'])
plt.title('%s (gid=%d)' % (net.gid_to_type(gid), gid))
plt.xlabel('Time (ms)')
plt.ylabel('Voltage (mV)')
plt.show()

###############################################################################
# Also, we can plot the spikes in the network and write them to text files.
# Note that we can use formatting syntax to specify the filename pattern
# with which each trial will be written ('spk_1.txt', 'spk_2.txt, ...). To
# read spikes back in, we can use wildcard expressions.
net.cell_response.plot_spikes_raster()
with tempfile.TemporaryDirectory() as tmp_dir_name:
    net.cell_response.write(op.join(tmp_dir_name, 'spk_%d.txt'))
    cell_response = read_spikes(op.join(tmp_dir_name, 'spk_*.txt'))
cell_response.plot_spikes_raster()

###############################################################################
# We can additionally calculate the mean spike rates for each cell class by
# specifying a time window with ``tstart`` and ``tstop``.
all_rates = cell_response.mean_rates(tstart=0, tstop=170,
                                     gid_ranges=net.gid_ranges,
                                     mean_type='all')
trial_rates = cell_response.mean_rates(tstart=0, tstop=170,
                                       gid_ranges=net.gid_ranges,
                                       mean_type='trial')
print('Mean spike rates across trials:')
print(all_rates)
print('Mean spike rates for individual trials:')
print(trial_rates)

###############################################################################
# Finally, we can plot the soma voltage along with the spiking activity with
# raster plots and histograms for the pyramidal cells.

fig, axes = plt.subplots(3, 1, figsize=(5, 7), sharex=True)

for idx in range(10):  # only 10 cells per cell-type
    gid = gid_ranges['L2_pyramidal'][idx]
    axes[0].plot(net.cell_response.times, vsec[gid]['soma'], color='g')
    gid = gid_ranges['L5_pyramidal'][idx]
    axes[0].plot(net.cell_response.times, vsec[gid]['soma'], color='r')
net.cell_response.plot_spikes_raster(ax=axes[1])
net.cell_response.plot_spikes_hist(ax=axes[2],
                                   spike_types=['L5_pyramidal',
                                                'L2_pyramidal'])


#!/usr/bin/env python
"""
A script to split the sample sheet index information and reverse complement the R2 index.

Useful for automating cutadapt workflow. Writes output as json.
"""
import pandas as pd
from Bio.Seq import Seq
from Bio.Alphabet import IUPAC
import argparse
import sys

def main(args):
    samplesheet = pd.read_table(args.samplesheet, sep=';', names=["Sample_id", "Sample_name", "index_combo", "Seqs_delivered", "Seqs_ordered"], index_col=0)
    samplesheet["R1_index"] = samplesheet.index_combo.apply(lambda x: x.split('-')[0])
    samplesheet["R2_index"] = samplesheet.index_combo.apply(lambda x: x.split('-')[1])
    samplesheet["R2_rev_index"] = samplesheet.R2_index.apply(lambda x: str(Seq(x, IUPAC.unambiguous_dna).reverse_complement()))
    samplesheet[['R1_index', 'R2_rev_index']].to_json(sys.stdout)

if __name__ == "__main__":
   parser = argparse.ArgumentParser(description=__doc__)
   parser.add_argument('samplesheet',
           help=('samplesheet with semicolon separated columns sample_id, sample_name, index_combo, seqs_delivered, seqs_ordered. '
               'The index_combo is used to get the indices'))
   args = parser.parse_args()

   main(args)


from typing import Optional

from fastapi import FastAPI

from pydantic import BaseModel

app = FastAPI()
app.counter = 0

class HelloResp(BaseModel):
    msg: str

class Product(BaseModel):
    name: str
    desc: Optional[str] = None
    price: float
    code: str
    tax: Optional[float] = None

@app.get("/")
def root():
    return{"message": "Hello World"}

# @app.get("/hello/{name}")
# def hello_name(name: str):
#     return f"Hello {name.capitalize()}"

@app.get("/counter")
def count():
    app.counter += 1
    return {f"You visited this page: {app.counter} times."}

@app.get("/hello/{name}", response_model=HelloResp)
def hello_name_view(name: str):
    return HelloResp(msg=f"Hello {name.capitalize()}")


from builtins import breakpoint
import cv2
import matplotlib.pyplot as plt
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from matplotlib.figure import Figure
import numpy as np
import torch
import torchvision
import io
import matplotlib.pyplot as plt
import PIL.Image
from torchvision.transforms import ToTensor
from .transformers import make_affine_parameters

def convert_image_np(inp, dataset='mnist'):
    """Convert a Tensor to numpy image."""
    if dataset == 'mnist':
        mean, std = 0.1307, 0.3081
    else:
        mean = np.array([0.485, 0.456, 0.406])
        std = np.array([0.229, 0.224, 0.225])

    # convert to numpy (C,H,W) -> (H,W,C)
    inp = inp.numpy().transpose((1, 2, 0))

    # normalize
    inp = std * inp + mean
    inp = np.clip(inp, 0, 1)

    return inp

# def gen_plot_input(input_array, nr_plots):
#     """Create a pyplot plot and save to buffer."""
#     plt.figure()
#     for ix in range(nr_plots):
#         plt.subplot(nr_plots, 1, ix+1)
#         plt.plot(input_array[ix, 0, :], c='darkblue')
#     buf = io.BytesIO()
#     plt.savefig(buf, format='jpeg')
#     buf.seek(0)
#     return buf

def gen_plot_input(transformed_array, nr_plots):
    plt.figure()
    colors = ['darkgoldenrod', 'sienna', 'darkred']
    for ix in range(nr_plots):
        plt.subplot(nr_plots, 1, ix+1)
        plt.plot(transformed_array[ix, 0, :], c=colors[ix])
    buf = io.BytesIO()
    plt.savefig(buf, format='jpeg')
    buf.seek(0)
    return buf

def gen_plot_output(transformed_array, nr_plots, batch_size):
    plt.figure()
    # colors = ['darkgreen', 'darkgoldenrod', 'darkred']
    nr_samples = 3
    alphas = [0.5, 0.75, 1]
    # Plot the results side-by-side
    for data_ix in range(nr_plots):
        plt.subplot(nr_plots, 1, data_ix+1)
        for sample_ix in range(nr_samples):
            plt.plot(transformed_array[data_ix + (sample_ix * batch_size), 0, :],
                     alpha=alphas[sample_ix], c='sienna')
    buf = io.BytesIO()
    plt.savefig(buf, format='jpeg')
    buf.seek(0)
    return buf

def normalize(images):
    return images / torch.max(images)

def visualize_stn(model, data, x_high_res, opt):
    with torch.no_grad():

        # IMAGE VISUALIZATION
        if opt.xdim == 2:
            data = data[:16]  # just visualize the first 16
            in_grid = convert_image_np(torchvision.utils.make_grid(data.cpu()), opt.dataset.lower())
            in_grid = (in_grid * 255).astype(np.uint8)
            in_grid = np.transpose(in_grid, (2, 0, 1))
            if opt.model.lower() == 'cnn':
                return in_grid, None, None, None

            if x_high_res is not None:
                x_high_res = x_high_res[:16]  # just visualize the first 16

            if opt.model.lower() == 'stn':
                transformed_input_tensor, thetas = model.forward_localizer(data, x_high_res)

            elif opt.model.lower() == 'pstn':
                transformed_input_tensor, thetas, _ = model.forward_localizer(data, x_high_res)

            out_grid = convert_image_np(torchvision.utils.make_grid(transformed_input_tensor.cpu()),
                                        opt.dataset.lower())
            out_grid = (out_grid * 255).astype(np.uint8)
            out_grid = np.transpose(out_grid, (2, 0, 1))

            bbox_images = None
            if opt.num_param < 4 and opt.num_param > 1:
                affine_params = make_affine_parameters(thetas)
                bbox_images = visualize_bbox(data.cpu(), affine_params, opt)
            # TODO: implement bounding boxed for 4 param trafos

            return in_grid, out_grid, thetas, bbox_images

        # TIMESERIES VISUALIZATIONS
        if opt.xdim == 1:
            batch_size = data.shape[0]
            nr_plots = min(3, batch_size)  # just visualize the first 3
            input_array = data.cpu().numpy()
            plt.close(); plt.close(); plt.close()
            # Prepare the input plot
            plot_buf = gen_plot_input(input_array, nr_plots)
            image = PIL.Image.open(plot_buf)
            image = ToTensor()(image)
            plt.close(); plt.close(); plt.close()

            if opt.model.lower() == 'cnn':
                return image, None, None, None

            if opt.model.lower() == 'stn':
                transformed_input, theta = model.forward_localizer(data, x_high_res=None)
                transformed_input = transformed_input.cpu().numpy()
                # prepare the STN output plot
                out_plot_buf = gen_plot_input(transformed_input, nr_plots)
                out_image = PIL.Image.open(out_plot_buf)
                out_image = ToTensor()(out_image)
                return image, out_image, theta, None

            elif opt.model.lower() == 'pstn':
                transformed_input, sampled_theta, _ = model.forward_localizer(data, x_high_res=None)
                transformed_input = transformed_input.cpu().numpy()
                # prepare the P-STN output plot
                out_plot_buf = gen_plot_output(transformed_input, nr_plots, batch_size)
                out_image = PIL.Image.open(out_plot_buf)
                out_image = ToTensor()(out_image)
                return image, out_image, sampled_theta, None

def visualize_bbox(data, affine_params, opt):
    if opt.transformer_type == 'diffeomorphic':
        return None

    batch_size = data.shape[0]
    affine_params = torch.stack(affine_params.split([opt.N] * opt.test_samples * batch_size))
    sorted_params = torch.zeros(batch_size, opt.test_samples, opt.N, 2, 3, dtype=affine_params.dtype)
    for i in range(len(affine_params)):
        # im, sample, N, 2, 3
        sorted_params[i % batch_size, i // batch_size, :, :, :] = affine_params[i, :, :]

    images = []
    for j, (im, params) in enumerate(zip(data, sorted_params)):

        im = np.transpose(im.numpy(), (1, 2, 0))
        im = denormalize(im)

        if im.shape[2] == 1:
            im = np.stack((im[:, :, 0],) * 3, axis=-1)

        if torch.isnan(params).any(): continue

        im = add_bounding_boxes(im, params, opt.N, opt.test_samples, mode_='crop', heatmap=opt.heatmap)

        images.append(np.transpose(im, (2, 0, 1)))

    if len(images) > 0:

        images = torch.FloatTensor(images)
        if 'mnist' in opt.dataset.lower():
            images = convert_image_np(torchvision.utils.make_grid(images - 0.3), opt.dataset.lower())
            images = (images * 255).astype(np.uint8)
            images = np.transpose(images, (2, 0, 1))
        else:
            images = torchvision.utils.make_grid(images, normalize=True)

        return images

    return None

def denormalize(image):
    im = (image - np.min(image)) / (np.max(image) - np.min(image))
    return im

def add_bounding_boxes(image, affine_params, num_branches, num_samples, mode_='crop', heatmap=True):
    color = [(255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 0), (255, 0, 255), (0, 255, 255)]

    image *= 255
    im = image.astype(np.uint8).copy()

    if mode_ == 'crop':
        w = int(im.shape[0])
        h = int(im.shape[1])

    for j in range(num_samples):
        for i in range(num_branches):
            if mode_ == 'crop':
                x = affine_params[j, i, 0, 2]
                y = affine_params[j, i, 1, 2]
                s_x = affine_params[j, i, 0, 0]
                s_y = affine_params[j, i, 1, 1]

                # define bbox by top left corner and define coordinates system with origo in top left corner
                x = int((x + 1 - s_x) * w // 2)
                y = int((y + 1 - s_y) * h // 2)

                # 1. (x + 1) * w / 2
                # 2. - s_x * w / 2

                # => (x + 1 - s_x) * w / 2

                if heatmap:

                    overlay = im.copy()
                    cv2.rectangle(overlay, (x, y), (x + w // 2, y + h // 2), color[i % len(color)],
                                  -1)  # A filled rectangle

                    alpha = 0.05  # Transparency factor.

                    # Following line overlays transparent rectangle over the image
                    im = cv2.addWeighted(overlay, alpha, im, 1 - alpha, 0)
                else:
                    cv2.rectangle(im, (x, y), (x + int(w * s_x), y + int(h * s_y)), color[i % len(color)], 1)

    return im


from datetime import datetime
import crython
from loguru import logger

from check_holiday import is_workday
import sys
import os

root_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))

chat_code_path = os.path.join(root_path,'威凡/信息交互自动化')
sys.path.insert(
    0, chat_code_path
)
from chat_robot_2 import send_msg_to_feishu
from tushare_stock2 import auto_get_tushare_data

logger.add("logs/{time}.log")

@crython.job(expr="*/3 * * * * * *")
def run_daily_weekday():
    if is_workday():
        day = datetime.today()
        logger.info(f"今天是{day}")
        auto_get_tushare_data()

@crython.job(expr="*/10 * * * * * *")
def run_daily_weekend():
    # if not is_workday():
    day = datetime.today()
    logger.info(f"今天是{day}")
    logger.info("温馨提醒:今天不是工作日")
    result = auto_get_tushare_data()
    send_msg_to_feishu(result)

if __name__ == "__main__":
    crython.start()
    crython.join()

分布式任务/任务调度/金融接口调度.py


import requests
from pyquery import PyQuery as pyq
import os

if not os.path.exists('papers'):
    os.mkdir('papers')

url_base = "http://openaccess.thecvf.com/%s"

src = pyq(url=url_base % '/CVPR2018.py')('#content')('dl').eq(0).children()

for i in range(0, src.length, 3):
    paper_name = src.eq(i)('a').html()
    download_url = url_base % src.eq(i+2)('a').eq(0).attr('href')
    print('Downloading paper #%d: %s' % (i/3+1, src.eq(i)('a').html()))

    r = requests.get(download_url)
    with open("papers/%s.pdf" % paper_name.replace('/', '\\'), 'wb') as f:
        f.write(r.content)


import re
from abc import abstractmethod

from bs4 import BeautifulSoup

from api.imgur.imgur_api import ImgurAPI
from models.image import RedditPostImage, ImgurImage
from parsers.base_parser import BaseParser
from utils.utils import *

LOGGER = logging.getLogger(__name__)

class ImgurBaseParser(BaseParser):

    @abstractmethod
    def get_images(self, post):
        raise NotImplementedError("A concrete parser must be used")

    @staticmethod
    def is_imgur_direct_url(url: str) -> bool:
        return tidy_up_url(url)[url.rfind(".") + 1:] in ["jpeg", "png", "jpg", "gif", "gifv"]

    @staticmethod
    def get_image_from_direct_url(post) -> RedditPostImage:
        image_url = tidy_up_url(post.url)

        if image_url.endswith("gifv"):
            image_url = image_url.replace("gifv", "mp4")

        image_file = ImgurBaseParser.get_file_name_from_url(image_url)

        image = RedditPostImage(image_url, post, image_file)

        return image

    @staticmethod
    def get_file_name_from_url(url: str) -> str:
        return url[url.rfind('/') + 1:]

class ImgurHTMLParser(ImgurBaseParser):
    def __init__(self) -> None:
        self._imgur_url_pattern = re.compile(r"(imgur.com/(.*))(\?.*)?")

    def get_images(self, post) -> list:
        images = []

        if "/a/" in post.url or "/gallery/" in post.url:
            # This is an album submission

            html_source = requests.get(post.url).text

            soup = BeautifulSoup(html_source, "lxml")
            matches = soup.select('img.post-image-placeholder')

            if not matches:
                matches = soup.select("img[itemprop]")

            if not matches:
                raise NotAbleToDownloadException("Couldn't process %s" % post.url)

            for m in matches:
                image_url = tidy_up_url(m['src'])
                image_file = ImgurBaseParser.get_file_name_from_url(image_url)
                images.append(RedditPostImage(image_url, post, image_file))

        elif ImgurHTMLParser.is_imgur_direct_url(post.url):
            # This is a direct url
            images.append(ImgurHTMLParser.get_image_from_direct_url(post))

        elif "imgur.com/" in post.url:
            # This is an Imgur page with a single image

            html_source = requests.get(post.url).text
            soup = BeautifulSoup(html_source, "lxml")

            # Let's try first to find those tags that will hold the image inside of an href property
            match = soup.select("a.zoom")

            if not match:
                match = soup.select("link[rel=image_src]")

            if match:
                image_url = match[0]["href"]
            else:
                # If no match was found inside of an href property, let's try now with the tags that contain it inside
                #  of a src property
                match = soup.select("img.post-image-placeholder")

                if not match:
                    match = soup.select("img[itemprop]")
                if not match:
                    match = soup.select("img")
                if not match:
                    raise NotAbleToDownloadException("Couldn't process %s" % post.url)

                image_url = match[0]['src']

            image_url = tidy_up_url(image_url)
            image_file = ImgurHTMLParser.get_file_name_from_url(image_url)

            images.append(RedditPostImage(image_url, post, image_file))
        else:
            raise NotAbleToDownloadException("Couldn't process %s" % post.url)

        return images

class ImgurAPIParser(ImgurBaseParser):

    def get_images(self, post) -> list:

        is_reddit_post = True

        # If we are using the direct Imgur download, and a URL is passed, let's try to use this as the URL
        if isinstance(post, str):
            url = post
            is_reddit_post = False
        else:
            url = post.url

        # First check if it is a direct URL so that we avoid querying the API
        if ImgurAPIParser.is_imgur_direct_url(url):
            return [ImgurAPIParser.get_image_from_direct_url(post)]

        urls = ImgurAPI.get_image_urls(url)
        image_entities = []

        for u in urls:
            if is_reddit_post:
                image_entities.append(RedditPostImage(u, post, ImgurAPIParser.get_file_name_from_url(u)))
            else:
                image_entities.append(ImgurImage(u, ImgurAPIParser.get_file_name_from_url(u)))

        return image_entities


from dotenv import load_dotenv
import datetime
import os

class Car():
    load_dotenv()

    def __init__(self):    #constractor
        self.feul = int(os.getenv('feul'))
        self.feulco =  float(os.getenv('feulco'))
        self.money =   int(os.getenv('money'))
        self.gear =    int(os.getenv('gear'))
        self.speed =   int(os.getenv('speed'))
        self.status = int(os.getenv('status'))

    def start(self):
        '''
        :name : artium
        : date : 23.1
        :desc : the func start the car and do ->  gear up from 0 to 1
        :return : car status 1/0
        '''
        if  self.status ==0 :
            if self.feul > 0:
                self.status = 1
                self.gear  = 0
                return self.status
        else:
            return 'car alredy driven  '

    def close(self):
        '''
        :name : artium brovarnik
        date : 23.1.23
        desc : the func close the car -> gear,speed,status = 0
        :return: car status
        '''
        if self.status !=0:
            self.gear = 0
            self.speed = 0
            self.status = 0
            return self.status
        else :
            return 'the car alredy in turned off'

    def get_speed(self,gear):
        '''
        :name : artium brovarnik
        :date : 24.1.23
        :param gear:
        :return: speed of the car
        '''
        x = os.getenv(f'max_gear')
        if (gear>int(os.getenv('max_gear')) or gear < 0):
            raise ValueError(x.format(os.getenv('error_value_range')))

        else:
            self.speed = gear * int(os.getenv('mph'))
            return self.speed

    def how_much_liter_to_destination(self,destination):
        '''
        : desc : the func culc how much liters feul we need by distans
        :param destination
        :return: liters by km
        '''
        return destination / int(os.getenv('feulco'))

    def drive(self,km):
        '''
        :desc : the func get km and culc how much fuel you have after driving
        :param km that you want drive:
        :return: feul amount after drive
        '''
        if  self.status ==0:
            self.start()
        if self.feul > self.how_much_liter_to_destination(km):
            self.feul= self.feul- self.how_much_liter_to_destination(km)
        elif self.feul < self.how_much_liter_to_destination(km):
            self.feul_for_drive(km)
        else:
            raise OverflowError( os.getenv('er_feu'))

    def feul_for_drive(self,km):
        '''
        :name artium brovarnik
        :data 23.1.23
        desc: calc if you have feul to this distance , if not buy and update your money
        :param km
        :return: money after buying
        '''
        if (self.feul< self.how_much_liter_to_destination(km)):

            liter_ineed_buy = self.how_much_liter_to_destination(km) - self.feul
            if (self.money>= liter_ineed_buy*int(os.getenv('liter_price'))):
                self.buy_benzin(liter_ineed_buy)
                return self.money
            else:
                raise(os.getenv('error_value_money'))
        else:
            self.feul = self.feul- self.how_much_liter_to_destination(km)
            return self.money

    def buy_benzin(self,liter):
        '''
        :name: artium brovarnik
        :date : 23.1.23
        :desc: calc how much money i have after buying fuel , check that you have this sum of money
        :param liter
        :return: money after buying
        '''
        money_i_need = liter*int(os.getenv('liter_price'))
        if self.money >= money_i_need:
            self.money = self.money - money_i_need
            return self.money
        else:
            raise ValueError(os.getenv('error_value_money'))

    def get_gear_by_speed(self,sp):
        '''
        desc: func return your gear by speed
        :param sp:
        :return: your gear level
        '''
        if sp<=0:
            raise ValueError(os.getenv('error_value_speed'))
        elif sp > 0 and sp<30:
            return 1
        elif sp>30 and sp<=60:
            return 2
        elif sp>60 and sp<=90:
            return 3
        elif sp>90 and sp<=120:
            return 4
        elif sp>120 and sp<=150:
            return 5
        else: os.getenv('max_gear')

    def gear_up(self):
        '''
        : name : artium brovarnik
        : date : 23.1.23
        decs : the func add 1 to gear , check that cannot up from level 6
        :return: gear level
        '''
        if self.gear ==int(os.getenv('max_gear')):
            x = os.getenv(f'error_value_maxGear')
            raise OverflowError(x.format(int(os.getenv('max_gear'))))
        else:
            if (self.status==1):
                self.gear = self.gear+1
                self.speed = self.speed +int(os.getenv('mph'))
                return self.gear
            else: raise ValueError(os.getenv('error_value_startcar'))

    def write_to_log_file(self,str):
        """
        : name : artium brovarnik
        :date : 23.1.23
        :desc : write all test result Pass / Fail  in log file
        :param str :string that describe the error
        :return: new line in log file
        """
        f= open('logfsfile.txt', 'a')
        f.write(f"{str}, {datetime.datetime.now()}, \n")
        f.close()


from sklearn.svm import SVC
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve,roc_auc_score
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from matplotlib import cm

def tuning_C_given(SV_ls):
    """
    given model list of last training , delve into the peak range and search better performance
    """
    best_C=np.argmax([model.auc for model in SV_ls])
    C_ls = [model.C for model in SV_ls]
    if best_C != 0:
        lower=C_ls[best_C-1]
    else:
        lower=C_ls[best_C]
    if best_C != len(C_ls) -1:
        upper=C_ls[best_C+1]
    else:
        upper=C_ls[best_C]

    return np.linspace(lower,upper,10)

def printting_training_result(SV_ls):
    print('-----------------------')
    print('   C \t \t  AUC')
    print('-----------------------')
    for model in SV_ls:
        print(' %.3f \t \t %.4f'%(model.C,model.auc))
    print('-----------------------')

def SVC_model(X_train,X_val,y_train,y_val,**kwarg):
    """
    given Training set and validation set, return a model with y_proba,roc curve,ROC_AUC
    arguments:
        X_train, np array
        X_val,   np array
        y_train, shape (n,)
        y_val,   shape (n,)
        **kwarg  : the key word arguments to class SVC()
    """
    model = SVC(probability=True,**kwarg)
    model.fit(X_train,y_train)
    model.y_proba=model.predict_proba(X_val)
    model.roc = roc_curve(y_true=y_val,y_score=model.y_proba[:,1])
    model.auc = roc_auc_score(y_true=y_val,y_score=model.y_proba[:,1])
    return model

def AUC_trace(model_ls,formatt='%.2f'):
    """
    input a list of SVM model, plot the roc_auc trace
    """
    C_ls = [model.C for model in model_ls]
    auc_ls = [model.auc for model in model_ls]
    plt.figure(figsize=(0.6*len(C_ls),6))
    plt.plot(auc_ls);
    plt.title('auc to C',fontsize=18);
    plt.ylabel('Area Under Curve',fontsize=12);
    plt.xlabel('hyper-parameter : C',fontsize=12);
    plt.xticks(np.arange(0,len(auc_ls)),[formatt%f for f in C_ls],fontsize=11);
    plt.yticks(fontsize=12);

vec_aa_dict = {0: 'A', 1: 'V', 2: 'I', 3: 'L', 4: 'M', 5: 'F', 6: 'Y', 7: 'W', 8: 'S', 9: 'T', 10: 'N', 11: 'Q', 12: 'R', 13: 'H', 14: 'K', 15: 'D', 16: 'E', 17: 'C', 18: 'U', 19: 'G', 20: 'P', 21: '-',22:'&'}

aa_vec_dict = {'A': 0, 'V': 1, 'I': 2, 'L': 3, 'M': 4, 'F': 5, 'Y': 6, 'W': 7, 'S': 8, 'T': 9, 'N': 10, 'Q': 11, 'R': 12, 'H': 13, 'K': 14, 'D': 15, 'E': 16, 'C': 17, 'U': 18, 'G': 19, 'P': 20, '-': 21, 'B': 22, 'J': 22, 'X': 22, 'Z': 22}

def flip_dict(dic):
    key=list(dic.keys())
    values=list(dic.values())
    if len(values) != len(np.unique(values)):
        return dict(zip(values,key))
    else:
        raise ValueError('flipped dict key has multiple pointer')

ratio_of = lambda y : np.sum(y==1)/len(y)

class SVM_omega(object):
    global aa_vec_dict
    def __init__(self,coef,encoder_dict=aa_vec_dict,threshold='scale',keep2raw=None):
        global aa_vec_dict
        global vec_aa_dict
        self.coef = coef[0]
        self.n_alphabet = len(np.unique(list(encoder_dict.values())))
        self.seq_len = len(self.coef)/self.n_alphabet
        self.threshold = np.quantile(np.abs(self.coef[np.where(self.coef != 0)[0]]),0.85) if threshold=='scale' else threshold
        self.aa2int = encoder_dict
        self.int2aa = vec_aa_dict if encoder_dict is aa_vec_dict else flip_dict(self.aa2int)
        self.important_sites = np.where(np.abs(self.coef)>self.threshold)[0]
        self.sites_weight = self.coef[self.important_sites]
        self.sites_in_seq = self.important_sites%self.seq_len
        self.sites_position = self.important_sites//self.seq_len
        self.sites_aa = [self.int2aa[i] for i in self.sites_position]
        self.DF = self.dataframe()

        if len(self.important_sites) == 0:
            self.non_zero_hist()
            raise ValueError('Invalid threshold, please rechoose a threshold or try "scale"')

        if keep2raw is not None:
            self.raw_sites=np.array(keep2raw)[self.sites_in_seq.astype(int)]
            self.DF = self.dataframe(keep2raw=keep2raw)

    def non_zero_hist(self,bins=20,**kwarg):
        plt.figure(figsize=(8,6))
        plt.hist(self.coef[np.where(self.coef != 0)[0]],bins=bins,**kwarg)

    def dataframe(self,columns=['sites_in_seq','sites_aa', 'sites_weight','sites_position'],keep2raw=None):
        if keep2raw is not None:
            columns=['raw_sites','sites_aa', 'sites_weight','sites_position']
        dataframe=pd.DataFrame(dict(zip(columns,[self.__getattribute__(x) for x in columns]))).sort_values(columns[0])
        dataframe.index = range(dataframe.shape[0])
        return dataframe

    def letter_plot(self,cmap=cm.cubehelix,**kwarg):
        figsize=(self.DF.shape[0]/4,3)
        plt.figure(figsize=figsize)
        plt.plot(range(self.DF.shape[0]),[1]*self.DF.shape[0],color='white')
        for i in range(self.DF.shape[0]):
            plt.text(i,1,self.DF.iloc[i,1],size=13+2*self.DF.iloc[i,2],color=cmap(self.DF.iloc[i,3]/22),**kwarg)


import os
import cv2
import json
from tqdm import tqdm
from DataExtractor import DataExtractor
from OpticalFlow import OpticalFlow
from utils import read_file, write_data

def handle_windows_path(path):
    drive = path.split(':')[0]
    path = path.replace('\\', '/')
    return path.replace(f'{drive}:/', f'/mnt/{drive.lower()}/')

config = read_file('config.json')
config['data_dir'] = handle_windows_path(config['data_dir'])
config['facs_dir']['SAMM'] = handle_windows_path(config['facs_dir']['SAMM'])
config['facs_dir']['MMEW'] = handle_windows_path(config['facs_dir']['MMEW'])
config['facs_dir']['CASME_II'] = handle_windows_path(config['facs_dir']['CASME_II'])
config['out_dir'] = handle_windows_path(config['out_dir'])
config['raw_file'] = f"{config['out_dir']}/{config['raw_file']}"
config['dataset'] = f"{config['out_dir']}/{config['dataset']}"
config['face_bounding_box'] = f"{config['out_dir']}/{config['face_bounding_box']}"

extractor = DataExtractor(config['emotions'],
                        config['data_dir'],
                        config['facs_dir'])
OpF = OpticalFlow(config['emotions'], config['db_OF'])

def get_onset_apex(emotions, dbs):
    print('\nExtracting onset and apex frames...')
    data = {}

    for emotion in tqdm(emotions):
        if not emotion in data.keys():
            data[emotion] = {
                'onset': [],
                'apex': []
            }

        for db in dbs:
            data[emotion] = extractor.get_data(db, emotion, data[emotion].copy())

    with open(f'{config["raw_file"]}', 'w') as f:
        json.dump(data, f, indent=4)

def generate_features(emotions, out_dir):
    print('\nGenerating features...')
    data = read_file(config['raw_file'])

    for expr in emotions:
        for i, (onset, apex) in enumerate(zip(data[expr]['onset'], data[expr]['apex'])):
            subject = onset[0] or apex[0]

            if 'SAMM' in onset[1] or 'SAMM' in apex[1]:
                db = 'SAMM'
            elif 'MMEW' in onset[1] or 'MMEW' in apex[1]:
                db = 'MMEW'
            elif 'CASME_II' in onset[1] or 'CASME_II' in apex[1]:
                db = 'CASME_II'

            print(f"{expr} - ({i+1}/{len(data[expr]['onset'])})", end=' | ')
            os.makedirs(f"{out_dir}/{db}", exist_ok=True)
            OpF.generate_optical_flow(i+1, f"{out_dir}/{db}", db, expr, subject, onset[1], apex[1])

    dataset = OpF.get_dataset()
    face_bounding_box = OpF.get_face_bounding_box()

    write_data(dataset, config['dataset'])
    write_data(face_bounding_box, config['face_bounding_box'])

def crop_resize(dim):
    print('\nCropping and resizing images...')
    dataset = read_file(config['dataset'])
    face_bounding_box = read_file(config['face_bounding_box'])

    for (OF, OS, H, V) in tqdm(dataset['X']):
        if 'SAMM' in OF or 'CASME_II' in OF:
            crop = True

            img = cv2.imread(OF)
            x, y, w, h = face_bounding_box[OF]
            OpF.crop_resize(OF, img, int(x), int(y), int(w), int(h), dim, crop)
            img = cv2.imread(OS)
            x, y, w, h = face_bounding_box[OS]
            OpF.crop_resize(OS, img, int(x), int(y), int(w), int(h), dim, crop)
            img = cv2.imread(H)
            x, y, w, h = face_bounding_box[H]
            OpF.crop_resize(H, img, int(x), int(y), int(w), int(h), dim, crop)
            img = cv2.imread(V)
            x, y, w, h = face_bounding_box[V]
            OpF.crop_resize(V, img, int(x), int(y), int(w), int(h), dim, crop)
        elif 'MMEW' in OF:
            crop = False

            img = cv2.imread(OF)
            OpF.crop_resize(OF, img, None, None, None, None, dim, crop)
            img = cv2.imread(OS)
            OpF.crop_resize(OS, img, None, None, None, None, dim, crop)
            img = cv2.imread(H)
            OpF.crop_resize(H, img, None, None, None, None, dim, crop)
            img = cv2.imread(V)
            OpF.crop_resize(V, img, None, None, None, None, dim, crop)

def main():
    get_onset_apex(config['emotions'], config['dbs'])
    generate_features(config['emotions'], config['out_dir'])
    crop_resize(config['img_dim'])

if __name__ == '__main__':
    main()


from django.core.management.base import BaseCommand, CommandError

from students.models import Group

class Command(BaseCommand):
    help = 'Show list of groups and students in them.'

    def handle(self, *args, **options):
        try:
            for group in Group.objects.all():
                self.stdout.write('=== {0} ==='.format(group.name))
                if group.students.all():
                    for student in group.students.all():
                        self.stdout.write(student.full_name)
                else:
                    self.stdout.write('There no students in this group.')
        except Group.DoesNotExist:
            raise CommandError('Does not exist any groups.')


#!/usr/lib/env python
#
# Module Documentation Override Check
#
# Verifies that any modules present in the _templates/overrides directory have all of their
# their exported functions included in the doc file.
import importlib
from pathlib import Path
import sys

modules_to_skip = ['metpy.xarray']

failed = False
for full_path in (Path('_templates') / 'overrides').glob('metpy.*.rst'):

    module = full_path.with_suffix('').name
    if module in modules_to_skip:
        continue

    # Get all functions in the module
    i = importlib.import_module(module)
    functions = set(i.__all__)

    # Get all lines in the file
    with open(full_path) as f:
        lines = f.read().splitlines()
    lines = {line.strip() for line in lines}

    # Check for any missing functions
    missing_functions = functions - lines

    if missing_functions:
        failed = True
        print('ERROR - The following functions are missing from the override file ' +
              str(full_path.name) + ': ' + ', '.join(missing_functions), file=sys.stderr)

# Report status
if failed:
    sys.exit(1)
else:
    print('Override check successful.')


# -*- coding: utf-8 -*-
# This code reconstructs an in-line hologram captured with the Raspberry Pi Setup

# manage the imports
import numpy as np
import matplotlib.pyplot as plt
import os
import time
import FresnelPropagator as FP
import MultiSliceViewer as MSV

# Taken from this link: https://matplotlib.org/2.1.2/gallery/animation/image_slices_viewer.html

class IndexTracker(object):
    def __init__(self, ax, X):
        self.ax = ax
        ax.set_title('use scroll wheel to navigate images')

        self.X = X
        rows, cols, self.slices = X.shape
        self.ind = self.slices//2

        self.im = ax.imshow(self.X[:, :, self.ind])
        self.update()

    def onscroll(self, event):
        print("%s %s" % (event.button, event.step))
        if event.button == 'up':
            self.ind = (self.ind + 1) % self.slices
        else:
            self.ind = (self.ind - 1) % self.slices
        self.update()

    def update(self):
        self.im.set_data(self.X[:, :, self.ind])
        ax.set_ylabel('slice %s' % self.ind)
        self.im.axes.figure.canvas.draw()

# define some experimental parameters
my_holo_file = 'testholo.jpg' #'test_microplastic.jpg'
my_background_file = 'test2bg.jpg'
mychannel = 2 # select the color channel 0,1,2
mysize = None #1024 # region of interest around the center coordinate

# define acquisition parameters
pixelsize = 1.4e-6; #0.6e-6#.2e-6#3.000e-6;
lambda0 = 450e-9; # in nanometer #530e-9;

# start and stop z-focus measure
stepsize = 0.001;

# read the raw-hologram
myholo = 1.*plt.imread(my_holo_file);

# select only the blue channel (because we used the blue LED)
myholo = myholo[:,:,mychannel ]

# if we acquired the background, subtract it!
if(0):
    mybak = 1.*imread(my_background_file)
    mybak = mybak[:,:,mychannel ]
    myholo = myholo/(20+mybak) # add some value to avoid dividing by zero

# crop out the ROI around the center to speed up computation (RADIX 2 please!)
holosize = myholo.shape
if mysize is not None:
    myholo =  myholo[int(holosize[0]/2-mysize/2):int(holosize[0]/2+mysize/2), int(holosize[0]/2-mysize/2):int(holosize[0]/2+mysize/2)]
else:
    # make square
    mysize = np.min(holosize)
    myholo =  myholo[int(holosize[0]/2-mysize/2):int(holosize[0]/2+mysize/2), int(holosize[0]/2-mysize/2):int(holosize[0]/2+mysize/2)]

# display intermediate result
plt.imshow(myholo)
plt.colorbar()
plt.show()

# normalize hologram
myholo = myholo/np.max(myholo);

# estimate hologram's amplitude
myamplitude = np.sqrt(myholo);

j = 0;
## Backpropagate the Hologram
# Fresnel Propagation - test out a range of z-values to find the one that
# brings the bug into focus
myholo_rec = [];
for i in range(0,20):
    #     [Ef, H] = fresnelprop(hologram,lambda,i,ps);
    zpos = i*stepsize;
    Ef = FP.FresnelPropagator(myholo, pixelsize, lambda0, zpos)

    myholo_rec.append(FP.abssqr(Ef))

    print("Now processed z-slice number"+str(i)+" at z-position (mum) "+str(zpos))

# convert list to numpy array
myholo_rec = np.array(myholo_rec)
#myholo_rec = np.transpose(myholo_rec, [1,2,0]) # make sure Z-axis is the last one

import NanoImagingPack as nip
nip.image(myholo_rec)
# now create a 3D image-stack viewer
fig, ax = plt.subplots(1, 1)
tracker = IndexTracker(ax, myholo_rec)

fig.canvas.mpl_connect('scroll_event', tracker.onscroll)
plt.show()