text:
# Copyright (c) 2015 Shotgun Software Inc.
#
# CONFIDENTIAL AND PROPRIETARY
#
# This work is provided "AS IS" and subject to the Shotgun Pipeline Toolkit
# Source Code License included in this distribution package. See LICENSE.
# By accessing, using, copying or modifying this work you indicate your
# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
# not expressly granted therein are reserved by Shotgun Software Inc.

from sgtk.platform.qt import QtCore, QtGui
import sgtk

class WorkAreaButton(QtGui.QToolButton):
    """
    UX for switching work area.

    This displays a "change work area" button which a user can interact with
    The button is designed to expand so that it is subtle until a user
    hovers over it.

    :signal clicked(str, int): Fires when someone clicks the change
        work area button. Arguments passed are the entity type and entity id
    """

    WIDGET_WIDTH_COLLAPSED = 30
    WIDGET_HEIGHT = 30

    NON_WORK_AREA_TYPES = [
        "PublishedFile",
        "Project",
        "TankPublishedFile",
        "Version",
        "Note",
        "Group",
        "HumanUser",
        "ScriptUser",
        "ApiUser",
        "ClientUser",
        "Department",
        "Cut",
        "CutItem",
        "Delivery",
        "Playlist",
        "Ticket"
    ]

    change_work_area = QtCore.Signal(str, int)

    def __init__(self, parent):
        """
        :param parent: The model parent.
        :type parent: :class:`~PySide.QtGui.QObject`
        """
        super(WorkAreaButton, self).__init__(parent)

        # an icon to represent all items which
        # aren't the current work area
        self._normal_icon = QtGui.QIcon()
        self._normal_icon.addPixmap(
            QtGui.QPixmap(":/tk_multi_infopanel/pin.png"),
            QtGui.QIcon.Normal,
            QtGui.QIcon.Off
        )

        # an icon to represent the current work area
        self._current_work_area_icon = QtGui.QIcon()
        self._current_work_area_icon.addPixmap(
            QtGui.QPixmap(":/tk_multi_infopanel/pin_blue.png"),
            QtGui.QIcon.Disabled,
            QtGui.QIcon.Off
        )
        self.setIcon(self._normal_icon)
        self.setIconSize(QtCore.QSize(self.WIDGET_WIDTH_COLLAPSED, self.WIDGET_HEIGHT))

        self._bundle = sgtk.platform.current_bundle()

        self._entity_type = None
        self._entity_id = None
        self._is_static = False

        self._caption = "Set Work Area"
        self._width = 120

        self.clicked.connect(self._on_click)

        self.setVisible(False)

    def set_up(self, entity_type, entity_id):
        """
        Sets up the button for a given entity.

        :param entity_type: Entity type to set up button for
        :param entity_id: Entity id to set up button for
        """
        self._entity_id = entity_id
        self._entity_type = entity_type

        if not self._bundle.get_setting("enable_context_switch"):
            # context switch button not enabled
            return

        # figure out if this is the current project
        context = self._bundle.context
        context_entity = context.task or context.entity or context.project or None

        self.setVisible(True)
        self.setEnabled(True)
        self.setIcon(self._normal_icon)
        self._is_static = False

        if context_entity and context_entity["type"] == entity_type and context_entity["id"] == entity_id:
            # the current work area
            self.setPopupMode(QtGui.QToolButton.DelayedPopup)
            self.setToolTip(
                "This is your current work area.\n"
                "The work you do will be associated with this item in Shotgun."
            )
            # set blue icon
            self.setIcon(self._current_work_area_icon)
            # disable the button
            self.setEnabled(False)
            # make sure it doesn't pop on mouseover
            self._is_static = True

        elif entity_type in self.NON_WORK_AREA_TYPES:
            # don't show the ctx selector for some types
            self.setToolTip("This cannot be a work area.")
            # disable the button
            self.setEnabled(False)
            # make sure it doesn't pop on mouse over
            self._is_static = True

        else:
            if entity_type == "Task":
                self._caption = "Set Work Area"
                self.setToolTip("Click to set your work area to the current task.")

            else:
                self._caption = "Pick Work Area"
                self.setToolTip("Click to select a task.")

        self._init_default_state()

    def _init_default_state(self):
        """
        Sets up the default collapsed state of the button
        """
        self.setText("")
        self.setToolButtonStyle(QtCore.Qt.ToolButtonIconOnly)
        self.setMinimumSize(QtCore.QSize(self.WIDGET_WIDTH_COLLAPSED, self.WIDGET_HEIGHT))
        self.setMaximumSize(QtCore.QSize(self.WIDGET_WIDTH_COLLAPSED, self.WIDGET_HEIGHT))
        # tell the style sheet to adjust
        self.setProperty("is_expanded", False)
        self.style().unpolish(self)
        self.style().polish(self)

    def _on_click(self):
        """
        Executed when the button is clicked
        """
        self.change_work_area.emit(self._entity_type, self._entity_id)

    def enterEvent(self, evt):
        """
        QT Mouse enter event
        """
        if not self._is_static:
            # not the current work area. so expand the button
            self.setText(self._caption)
            self.setToolButtonStyle(QtCore.Qt.ToolButtonTextBesideIcon)
            self.setMinimumSize(QtCore.QSize(self._width, self.WIDGET_HEIGHT))
            self.setMaximumSize(QtCore.QSize(self._width, self.WIDGET_HEIGHT))
            # tell the style sheet to adjust
            self.setProperty("is_expanded", True)
            self.style().unpolish(self)
            self.style().polish(self)

        return super(WorkAreaButton, self).enterEvent(evt)

    def leaveEvent(self, evt):
        """
        QT Mouse leave event
        """
        if not self._is_static:
            # collapse button after a delay
            QtCore.QTimer.singleShot(300, self._init_default_state)

        return super(WorkAreaButton, self).leaveEvent(evt)

class FloatingWorkAreaButton(WorkAreaButton):
    """
    UX for switching work area.

    This displays a "change work area" button which a user can interact with
    The button is designed to expand so that it is subtle until a user
    hovers over it.

    Derives from :class:`WorkAreaButton` and positions the widget
    relative to the bottom-right corner of the parent widget.

    :signal clicked(str, int): Fires when someone clicks the change
        work area button. Arguments passed are the entity type and entity id
    """

    RIGHT_OFFSET = 6
    BOTTOM_OFFSET = 6

    def __init__(self, parent):
        """
        :param right_side_offset: Right hand side offset in pixels
        :param bottom_offset: Bottom offset in pixels
        :param parent: The model parent.
        :type parent: :class:`~PySide.QtGui.QObject`
        """
        super(FloatingWorkAreaButton, self).__init__(parent)

        # hook up a listener to the parent window so this widget
        # follows along when the parent window changes size
        filter = ResizeEventFilter(parent)
        filter.resized.connect(self._on_parent_resized)
        parent.installEventFilter(filter)

    def set_up(self, entity_type, entity_id):
        """
        Sets up the button for a given entity.

        :param entity_type: Entity type to set up button for
        :param entity_id: Entity id to set up button for
        """
        if entity_type in self.NON_WORK_AREA_TYPES:
            # hide the widget
            self.setVisible(False)
        else:
            # base class implementation
            super(FloatingWorkAreaButton, self).set_up(entity_type, entity_id)

    def __position_widget(self):
        """
        Moves the widget to the bottom-right corner of the parent widget.
        """
        self.move(
            self.parentWidget().width() - self.width() - self.RIGHT_OFFSET,
            self.parentWidget().height() - self.height() - self.BOTTOM_OFFSET
        )

    def _init_default_state(self):
        """
        Sets up the default collapsed state of the button
        """
        super(FloatingWorkAreaButton, self)._init_default_state()
        self.__position_widget()

    def enterEvent(self, evt):
        """
        QT Mouse enter event
        """
        status = super(FloatingWorkAreaButton, self).enterEvent(evt)

        if not self._is_static:
            self.__position_widget()

        return status

    def _on_parent_resized(self):
        """
        Special slot hooked up to the event filter.
        When associated widget is resized this slot is being called.
        """
        self.__position_widget()

class ResizeEventFilter(QtCore.QObject):
    """
    Utility and helper.

    Event filter which emits a resized signal whenever
    the monitored widget resizes.

    You use it like this:

    # create the filter object. Typically, it's
    # it's easiest to parent it to the object that is
    # being monitored (in this case self.ui.thumbnail)
    filter = ResizeEventFilter(self.ui.thumbnail)

    # now set up a signal/slot connection so that the
    # __on_thumb_resized slot gets called every time
    # the widget is resized
    filter.resized.connect(self.__on_thumb_resized)

    # finally, install the event filter into the QT
    # event system
    self.ui.thumbnail.installEventFilter(filter)
    """
    resized = QtCore.Signal()

    def eventFilter(self, obj, event):
        """
        Event filter implementation.
        For information, see the QT docs:
        http://doc.qt.io/qt-4.8/qobject.html#eventFilter

        This will emit the resized signal (in this class)
        whenever the linked up object is being resized.

        :param obj: The object that is being watched for events
        :param event: Event object that the object has emitted
        :returns: Always returns False to indicate that no events
                  should ever be discarded by the filter.
        """
        # peek at the message
        if event.type() == QtCore.QEvent.Resize:
            # re-broadcast any resize events
            self.resized.emit()
        # pass it on!
        return False


text:
import functools

from django import http
from django.shortcuts import get_object_or_404, redirect

from conference import models, settings

def speaker_access(f): # pragma: no cover
    """
    Decorator that protects the view relative to a speaker.
    """
    @functools.wraps(f)
    def wrapper(request, slug, **kwargs):
        spk = get_object_or_404(models.Speaker, slug=slug)
        if request.user.is_staff or request.user == spk.user:
            full_access = True
            talks = spk.talks()
        else:
            full_access = False
            conf = models.Conference.objects.current()
            if settings.VOTING_OPENED(conf, request.user):
                if settings.VOTING_ALLOWED(request.user):
                    talks = spk.talks()
                else:
                    if settings.VOTING_DISALLOWED:
                        return redirect(settings.VOTING_DISALLOWED)
                    else:
                        raise http.Http404()
            else:
                talks = spk.talks(status='accepted')
                if talks.count() == 0:
                    raise http.Http404()

        return f(request, slug, speaker=spk, talks=talks, full_access=full_access, **kwargs)
    return wrapper

def talk_access(f): # pragma: no cover
    """
    Decorator that protects the view relative to a talk.
    """
    @functools.wraps(f)
    def wrapper(request, slug, **kwargs):
        tlk = get_object_or_404(models.Talk, slug=slug)
        if request.user.is_anonymous():
            full_access = False
        elif request.user.is_staff:
            full_access = True
        else:
            try:
                tlk.get_all_speakers().get(user__id=request.user.id)
            except (models.Speaker.DoesNotExist, models.Speaker.MultipleObjectsReturned):
                # The MultipleObjectsReturned can happen if the user is not logged on and .id is None
                full_access = False
            else:
                full_access = True

        # if the talk is unconfirmed can access:
        #   * superusers or speakers (full access = True)
        #   * if the community voting is in progress who has the right to vote
        if tlk.status == 'proposed' and not full_access:
            conf = models.Conference.objects.current()
            if not settings.VOTING_OPENED(conf, request.user):
                return http.HttpResponseForbidden()
            if not settings.VOTING_ALLOWED(request.user):
                if settings.VOTING_DISALLOWED:
                    return redirect(settings.VOTING_DISALLOWED)
                else:
                    return http.HttpResponseForbidden()

        return f(request, slug, talk=tlk, full_access=full_access, **kwargs)
    return wrapper

def profile_access(f): # pragma: no cover
    """
    Decorator which protect the relative view to a profile.
    """
    @functools.wraps(f)
    def wrapper(request, slug, **kwargs):
        try:
            profile = models.AttendeeProfile.objects\
                .select_related('user')\
                .get(slug=slug)
        except models.AttendeeProfile.DoesNotExist:
            raise http.Http404()

        if request.user.is_staff or request.user == profile.user:
            full_access = True
        else:
            full_access = False
            # if the profile belongs to a speaker with talk of "accepted" is visible
            # whatever you say the same profile.
            accepted = models.TalkSpeaker.objects\
                .filter(speaker__user=profile.user)\
                .filter(talk__status='accepted')\
                .count()
            if not accepted:
                # if the community voting is open and the profile belongs to a speaker
                # with the talk in the race page is visible
                conf = models.Conference.objects.current()
                if not (settings.VOTING_OPENED(conf, request.user) and settings.VOTING_ALLOWED(request.user)):
                    if profile.visibility == 'x':
                        return http.HttpResponseForbidden()
                    elif profile.visibility == 'm' and request.user.is_anonymous():
                        return http.HttpResponseForbidden()
        return f(request, slug, profile=profile, full_access=full_access, **kwargs)
    return wrapper

domain: python


text:
# -*- coding: utf-8 -*-

"""
The MIT License (MIT)

Copyright (c) 2015-2020 Rapptz

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""

import io
import os.path

class File:
    """A parameter object used for :meth:`abc.Messageable.send`
    for sending file objects.

    .. note::

        File objects are single use and are not meant to be reused in
        multiple :meth:`abc.Messageable.send`s.

    Attributes
    -----------
    fp: Union[:class:`str`, :class:`io.BufferedIOBase`]
        A file-like object opened in binary mode and read mode
        or a filename representing a file in the hard drive to
        open.

        .. note::

            If the file-like object passed is opened via ``open`` then the
            modes 'rb' should be used.

            To pass binary data, consider usage of ``io.BytesIO``.

    filename: Optional[:class:`str`]
        The filename to display when uploading to Discord.
        If this is not given then it defaults to ``fp.name`` or if ``fp`` is
        a string then the ``filename`` will default to the string given.
    spoiler: :class:`bool`
        Whether the attachment is a spoiler.
    """

    __slots__ = ("fp", "filename", "_original_pos", "_owner", "_closer")

    def __init__(self, fp, filename=None, *, spoiler=False):
        self.fp = fp

        if isinstance(fp, io.IOBase):
            if not (fp.seekable() and fp.readable()):
                raise ValueError(
                    "File buffer {!r} must be seekable and readable".format(fp)
                )
            self.fp = fp
            self._original_pos = fp.tell()
            self._owner = False
        else:
            self.fp = open(fp, "rb")
            self._original_pos = 0
            self._owner = True

        # aiohttp only uses two methods from IOBase
        # read and close, since I want to control when the files
        # close, I need to stub it so it doesn't close unless
        # I tell it to
        self._closer = self.fp.close
        self.fp.close = lambda: None

        if filename is None:
            if isinstance(fp, str):
                _, self.filename = os.path.split(fp)
            else:
                self.filename = getattr(fp, "name", None)
        else:
            self.filename = filename

        if (
            spoiler
            and self.filename is not None
            and not self.filename.startswith("SPOILER_")
        ):
            self.filename = "SPOILER_" + self.filename

    def reset(self, *, seek=True):
        # The `seek` parameter is needed because
        # the retry-loop is iterated over multiple times
        # starting from 0, as an implementation quirk
        # the resetting must be done at the beginning
        # before a request is done, since the first index
        # is 0, and thus false, then this prevents an
        # unnecessary seek since it's the first request
        # done.
        if seek:
            self.fp.seek(self._original_pos)

    def close(self):
        self.fp.close = self._closer
        if self._owner:
            self._closer()


text:
from __future__ import annotations
import copy
import logging
from collections import defaultdict
from pathlib import Path
from rasa.nlu.featurizers.featurizer import Featurizer

import numpy as np
import scipy.sparse
import tensorflow as tf

from typing import Any, Dict, List, Optional, Text, Tuple, Union, Type

from rasa.engine.graph import ExecutionContext, GraphComponent
from rasa.engine.recipes.default_recipe import DefaultV1Recipe
from rasa.engine.storage.resource import Resource
from rasa.engine.storage.storage import ModelStorage
from rasa.nlu.extractors.extractor import EntityExtractorMixin
from rasa.nlu.classifiers.classifier import IntentClassifier
import rasa.shared.utils.io
import rasa.utils.io as io_utils
import rasa.nlu.utils.bilou_utils as bilou_utils
from rasa.shared.constants import DIAGNOSTIC_DATA
from rasa.nlu.extractors.extractor import EntityTagSpec
from rasa.nlu.classifiers import LABEL_RANKING_LENGTH
from rasa.utils import train_utils
from rasa.utils.tensorflow import rasa_layers
from rasa.utils.tensorflow.models import RasaModel, TransformerRasaModel
from rasa.utils.tensorflow.model_data import (
    RasaModelData,
    FeatureSignature,
    FeatureArray,
)
from rasa.nlu.constants import TOKENS_NAMES, DEFAULT_TRANSFORMER_SIZE
from rasa.shared.nlu.constants import (
    SPLIT_ENTITIES_BY_COMMA_DEFAULT_VALUE,
    TEXT,
    INTENT,
    INTENT_RESPONSE_KEY,
    ENTITIES,
    ENTITY_ATTRIBUTE_TYPE,
    ENTITY_ATTRIBUTE_GROUP,
    ENTITY_ATTRIBUTE_ROLE,
    NO_ENTITY_TAG,
    SPLIT_ENTITIES_BY_COMMA,
)
from rasa.shared.exceptions import InvalidConfigException
from rasa.shared.nlu.training_data.training_data import TrainingData
from rasa.shared.nlu.training_data.message import Message
from rasa.utils.tensorflow.constants import (
    LABEL,
    IDS,
    HIDDEN_LAYERS_SIZES,
    RENORMALIZE_CONFIDENCES,
    SHARE_HIDDEN_LAYERS,
    TRANSFORMER_SIZE,
    NUM_TRANSFORMER_LAYERS,
    NUM_HEADS,
    BATCH_SIZES,
    BATCH_STRATEGY,
    EPOCHS,
    RANDOM_SEED,
    LEARNING_RATE,
    RANKING_LENGTH,
    LOSS_TYPE,
    SIMILARITY_TYPE,
    NUM_NEG,
    SPARSE_INPUT_DROPOUT,
    DENSE_INPUT_DROPOUT,
    MASKED_LM,
    ENTITY_RECOGNITION,
    TENSORBOARD_LOG_DIR,
    INTENT_CLASSIFICATION,
    EVAL_NUM_EXAMPLES,
    EVAL_NUM_EPOCHS,
    UNIDIRECTIONAL_ENCODER,
    DROP_RATE,
    DROP_RATE_ATTENTION,
    CONNECTION_DENSITY,
    NEGATIVE_MARGIN_SCALE,
    REGULARIZATION_CONSTANT,
    SCALE_LOSS,
    USE_MAX_NEG_SIM,
    MAX_NEG_SIM,
    MAX_POS_SIM,
    EMBEDDING_DIMENSION,
    BILOU_FLAG,
    KEY_RELATIVE_ATTENTION,
    VALUE_RELATIVE_ATTENTION,
    MAX_RELATIVE_POSITION,
    AUTO,
    BALANCED,
    CROSS_ENTROPY,
    TENSORBOARD_LOG_LEVEL,
    CONCAT_DIMENSION,
    FEATURIZERS,
    CHECKPOINT_MODEL,
    SEQUENCE,
    SENTENCE,
    SEQUENCE_LENGTH,
    DENSE_DIMENSION,
    MASK,
    CONSTRAIN_SIMILARITIES,
    MODEL_CONFIDENCE,
    SOFTMAX,
)

logger = logging.getLogger(__name__)

SPARSE = "sparse"
DENSE = "dense"
LABEL_KEY = LABEL
LABEL_SUB_KEY = IDS

POSSIBLE_TAGS = [ENTITY_ATTRIBUTE_TYPE, ENTITY_ATTRIBUTE_ROLE, ENTITY_ATTRIBUTE_GROUP]

@DefaultV1Recipe.register(
    [
        DefaultV1Recipe.ComponentType.INTENT_CLASSIFIER,
        DefaultV1Recipe.ComponentType.ENTITY_EXTRACTOR,
    ],
    is_trainable=True,
)
class DIETClassifier(GraphComponent, IntentClassifier, EntityExtractorMixin):
    """A multi-task model for intent classification and entity extraction.

    DIET is Dual Intent and Entity Transformer.
    The architecture is based on a transformer which is shared for both tasks.
    A sequence of entity labels is predicted through a Conditional Random Field (CRF)
    tagging layer on top of the transformer output sequence corresponding to the
    input sequence of tokens. The transformer output for the ``__CLS__`` token and
    intent labels are embedded into a single semantic vector space. We use the
    dot-product loss to maximize the similarity with the target label and minimize
    similarities with negative samples.
    """

    @classmethod
    def required_components(cls) -> List[Type]:
        """Components that should be included in the pipeline before this component."""
        return [Featurizer]

    @staticmethod
    def get_default_config() -> Dict[Text, Any]:
        """The component's default config (see parent class for full docstring)."""
        # please make sure to update the docs when changing a default parameter
        return {
            # ## Architecture of the used neural network
            # Hidden layer sizes for layers before the embedding layers for user message
            # and labels.
            # The number of hidden layers is equal to the length of the corresponding
            # list.
            HIDDEN_LAYERS_SIZES: {TEXT: [], LABEL: []},
            # Whether to share the hidden layer weights between user message and labels.
            SHARE_HIDDEN_LAYERS: False,
            # Number of units in transformer
            TRANSFORMER_SIZE: DEFAULT_TRANSFORMER_SIZE,
            # Number of transformer layers
            NUM_TRANSFORMER_LAYERS: 2,
            # Number of attention heads in transformer
            NUM_HEADS: 4,
            # If 'True' use key relative embeddings in attention
            KEY_RELATIVE_ATTENTION: False,
            # If 'True' use value relative embeddings in attention
            VALUE_RELATIVE_ATTENTION: False,
            # Max position for relative embeddings. Only in effect if key- or value
            # relative attention are turned on
            MAX_RELATIVE_POSITION: 5,
            # Use a unidirectional or bidirectional encoder.
            UNIDIRECTIONAL_ENCODER: False,
            # ## Training parameters
            # Initial and final batch sizes:
            # Batch size will be linearly increased for each epoch.
            BATCH_SIZES: [64, 256],
            # Strategy used when creating batches.
            # Can be either 'sequence' or 'balanced'.
            BATCH_STRATEGY: BALANCED,
            # Number of epochs to train
            EPOCHS: 300,
            # Set random seed to any 'int' to get reproducible results
            RANDOM_SEED: None,
            # Initial learning rate for the optimizer
            LEARNING_RATE: 0.001,
            # ## Parameters for embeddings
            # Dimension size of embedding vectors
            EMBEDDING_DIMENSION: 20,
            # Dense dimension to use for sparse features.
            DENSE_DIMENSION: {TEXT: 128, LABEL: 20},
            # Default dimension to use for concatenating sequence and sentence features.
            CONCAT_DIMENSION: {TEXT: 128, LABEL: 20},
            # The number of incorrect labels. The algorithm will minimize
            # their similarity to the user input during training.
            NUM_NEG: 20,
            # Type of similarity measure to use, either 'auto' or 'cosine' or 'inner'.
            SIMILARITY_TYPE: AUTO,
            # The type of the loss function, either 'cross_entropy' or 'margin'.
            LOSS_TYPE: CROSS_ENTROPY,
            # Number of top intents for which confidences should be reported.
            # Set to 0 if confidences for all intents should be reported.
            RANKING_LENGTH: LABEL_RANKING_LENGTH,
            # Indicates how similar the algorithm should try to make embedding vectors
            # for correct labels.
            # Should be 0.0 < ... < 1.0 for 'cosine' similarity type.
            MAX_POS_SIM: 0.8,
            # Maximum negative similarity for incorrect labels.
            # Should be -1.0 < ... < 1.0 for 'cosine' similarity type.
            MAX_NEG_SIM: -0.4,
            # If 'True' the algorithm only minimizes maximum similarity over
            # incorrect intent labels, used only if 'loss_type' is set to 'margin'.
            USE_MAX_NEG_SIM: True,
            # If 'True' scale loss inverse proportionally to the confidence
            # of the correct prediction
            SCALE_LOSS: False,
            # ## Regularization parameters
            # The scale of regularization
            REGULARIZATION_CONSTANT: 0.002,
            # The scale of how important is to minimize the maximum similarity
            # between embeddings of different labels,
            # used only if 'loss_type' is set to 'margin'.
            NEGATIVE_MARGIN_SCALE: 0.8,
            # Dropout rate for encoder
            DROP_RATE: 0.2,
            # Dropout rate for attention
            DROP_RATE_ATTENTION: 0,
            # Fraction of trainable weights in internal layers.
            CONNECTION_DENSITY: 0.2,
            # If 'True' apply dropout to sparse input tensors
            SPARSE_INPUT_DROPOUT: True,
            # If 'True' apply dropout to dense input tensors
            DENSE_INPUT_DROPOUT: True,
            # ## Evaluation parameters
            # How often calculate validation accuracy.
            # Small values may hurt performance.
            EVAL_NUM_EPOCHS: 20,
            # How many examples to use for hold out validation set
            # Large values may hurt performance, e.g. model accuracy.
            # Set to 0 for no validation.
            EVAL_NUM_EXAMPLES: 0,
            # ## Model config
            # If 'True' intent classification is trained and intent predicted.
            INTENT_CLASSIFICATION: True,
            # If 'True' named entity recognition is trained and entities predicted.
            ENTITY_RECOGNITION: True,
            # If 'True' random tokens of the input message will be masked and the model
            # should predict those tokens.
            MASKED_LM: False,
            # 'BILOU_flag' determines whether to use BILOU tagging or not.
            # If set to 'True' labelling is more rigorous, however more
            # examples per entity are required.
            # Rule of thumb: you should have more than 100 examples per entity.
            BILOU_FLAG: True,
            # If you want to use tensorboard to visualize training and validation
            # metrics, set this option to a valid output directory.
            TENSORBOARD_LOG_DIR: None,
            # Define when training metrics for tensorboard should be logged.
            # Either after every epoch or for every training step.
            # Valid values: 'epoch' and 'batch'
            TENSORBOARD_LOG_LEVEL: "epoch",
            # Perform model checkpointing
            CHECKPOINT_MODEL: False,
            # Specify what features to use as sequence and sentence features
            # By default all features in the pipeline are used.
            FEATURIZERS: [],
            # Split entities by comma, this makes sense e.g. for a list of ingredients
            # in a recipie, but it doesn't make sense for the parts of an address
            SPLIT_ENTITIES_BY_COMMA: True,
            # If 'True' applies sigmoid on all similarity terms and adds
            # it to the loss function to ensure that similarity values are
            # approximately bounded. Used inside cross-entropy loss only.
            CONSTRAIN_SIMILARITIES: False,
            # Model confidence to be returned during inference. Currently, the only
            # possible value is `softmax`.
            MODEL_CONFIDENCE: SOFTMAX,
            # Determines whether the confidences of the chosen top intents should be
            # renormalized so that they sum up to 1. By default, we do not renormalize
            # and return the confidences for the top intents as is.
            # Note that renormalization only makes sense if confidences are generated
            # via `softmax`.
            RENORMALIZE_CONFIDENCES: False,
        }

    def __init__(
        self,
        config: Dict[Text, Any],
        model_storage: ModelStorage,
        resource: Resource,
        execution_context: ExecutionContext,
        index_label_id_mapping: Optional[Dict[int, Text]] = None,
        entity_tag_specs: Optional[List[EntityTagSpec]] = None,
        model: Optional[RasaModel] = None,
        sparse_feature_sizes: Optional[Dict[Text, Dict[Text, List[int]]]] = None,
    ) -> None:
        """Declare instance variables with default values."""
        if EPOCHS not in config:
            rasa.shared.utils.io.raise_warning(
                f"Please configure the number of '{EPOCHS}' in your configuration file."
                f" We will change the default value of '{EPOCHS}' in the future to 1. "
            )

        self.component_config = config
        self._model_storage = model_storage
        self._resource = resource
        self._execution_context = execution_context

        self._check_config_parameters()

        # transform numbers to labels
        self.index_label_id_mapping = index_label_id_mapping or {}

        self._entity_tag_specs = entity_tag_specs

        self.model = model

        self.tmp_checkpoint_dir = None
        if self.component_config[CHECKPOINT_MODEL]:
            self.tmp_checkpoint_dir = Path(rasa.utils.io.create_temporary_directory())

        self._label_data: Optional[RasaModelData] = None
        self._data_example: Optional[Dict[Text, Dict[Text, List[FeatureArray]]]] = None

        self.split_entities_config = rasa.utils.train_utils.init_split_entities(
            self.component_config[SPLIT_ENTITIES_BY_COMMA],
            SPLIT_ENTITIES_BY_COMMA_DEFAULT_VALUE,
        )

        self.finetune_mode = self._execution_context.is_finetuning
        self._sparse_feature_sizes = sparse_feature_sizes

    # init helpers
    def _check_masked_lm(self) -> None:
        if (
            self.component_config[MASKED_LM]
            and self.component_config[NUM_TRANSFORMER_LAYERS] == 0
        ):
            raise ValueError(
                f"If number of transformer layers is 0, "
                f"'{MASKED_LM}' option should be 'False'."
            )

    def _check_share_hidden_layers_sizes(self) -> None:
        if self.component_config.get(SHARE_HIDDEN_LAYERS):
            first_hidden_layer_sizes = next(
                iter(self.component_config[HIDDEN_LAYERS_SIZES].values())
            )
            # check that all hidden layer sizes are the same
            identical_hidden_layer_sizes = all(
                current_hidden_layer_sizes == first_hidden_layer_sizes
                for current_hidden_layer_sizes in self.component_config[
                    HIDDEN_LAYERS_SIZES
                ].values()
            )
            if not identical_hidden_layer_sizes:
                raise ValueError(
                    f"If hidden layer weights are shared, "
                    f"{HIDDEN_LAYERS_SIZES} must coincide."
                )

    def _check_config_parameters(self) -> None:
        self.component_config = train_utils.check_deprecated_options(
            self.component_config
        )

        self._check_masked_lm()
        self._check_share_hidden_layers_sizes()

        self.component_config = train_utils.update_confidence_type(
            self.component_config
        )

        train_utils.validate_configuration_settings(self.component_config)

        self.component_config = train_utils.update_similarity_type(
            self.component_config
        )
        self.component_config = train_utils.update_evaluation_parameters(
            self.component_config
        )

    @classmethod
    def create(
        cls,
        config: Dict[Text, Any],
        model_storage: ModelStorage,
        resource: Resource,
        execution_context: ExecutionContext,
    ) -> DIETClassifier:
        """Creates a new untrained component (see parent class for full docstring)."""
        return cls(config, model_storage, resource, execution_context)

    @property
    def label_key(self) -> Optional[Text]:
        """Return key if intent classification is activated."""
        return LABEL_KEY if self.component_config[INTENT_CLASSIFICATION] else None

    @property
    def label_sub_key(self) -> Optional[Text]:
        """Return sub key if intent classification is activated."""
        return LABEL_SUB_KEY if self.component_config[INTENT_CLASSIFICATION] else None

    @staticmethod
    def model_class() -> Type[RasaModel]:
        return DIET

    # training data helpers:
    @staticmethod
    def _label_id_index_mapping(
        training_data: TrainingData, attribute: Text
    ) -> Dict[Text, int]:
        """Create label_id dictionary."""

        distinct_label_ids = {
            example.get(attribute) for example in training_data.intent_examples
        } - {None}
        return {
            label_id: idx for idx, label_id in enumerate(sorted(distinct_label_ids))
        }

    @staticmethod
    def _invert_mapping(mapping: Dict) -> Dict:
        return {value: key for key, value in mapping.items()}

    def _create_entity_tag_specs(
        self, training_data: TrainingData
    ) -> List[EntityTagSpec]:
        """Create entity tag specifications with their respective tag id mappings."""

        _tag_specs = []

        for tag_name in POSSIBLE_TAGS:
            if self.component_config[BILOU_FLAG]:
                tag_id_index_mapping = bilou_utils.build_tag_id_dict(
                    training_data, tag_name
                )
            else:
                tag_id_index_mapping = self._tag_id_index_mapping_for(
                    tag_name, training_data
                )

            if tag_id_index_mapping:
                _tag_specs.append(
                    EntityTagSpec(
                        tag_name=tag_name,
                        tags_to_ids=tag_id_index_mapping,
                        ids_to_tags=self._invert_mapping(tag_id_index_mapping),
                        num_tags=len(tag_id_index_mapping),
                    )
                )

        return _tag_specs

    @staticmethod
    def _tag_id_index_mapping_for(
        tag_name: Text, training_data: TrainingData
    ) -> Optional[Dict[Text, int]]:
        """Create mapping from tag name to id."""
        if tag_name == ENTITY_ATTRIBUTE_ROLE:
            distinct_tags = training_data.entity_roles
        elif tag_name == ENTITY_ATTRIBUTE_GROUP:
            distinct_tags = training_data.entity_groups
        else:
            distinct_tags = training_data.entities

        distinct_tags = distinct_tags - {NO_ENTITY_TAG} - {None}

        if not distinct_tags:
            return None

        tag_id_dict = {
            tag_id: idx for idx, tag_id in enumerate(sorted(distinct_tags), 1)
        }
        # NO_ENTITY_TAG corresponds to non-entity which should correspond to 0 index
        # needed for correct prediction for padding
        tag_id_dict[NO_ENTITY_TAG] = 0

        return tag_id_dict

    @staticmethod
    def _find_example_for_label(
        label: Text, examples: List[Message], attribute: Text
    ) -> Optional[Message]:
        for ex in examples:
            if ex.get(attribute) == label:
                return ex
        return None

    def _check_labels_features_exist(
        self, labels_example: List[Message], attribute: Text
    ) -> bool:
        """Checks if all labels have features set."""

        return all(
            label_example.features_present(
                attribute, self.component_config[FEATURIZERS]
            )
            for label_example in labels_example
        )

    def _extract_features(
        self, message: Message, attribute: Text
    ) -> Dict[Text, Union[scipy.sparse.spmatrix, np.ndarray]]:

        (
            sparse_sequence_features,
            sparse_sentence_features,
        ) = message.get_sparse_features(attribute, self.component_config[FEATURIZERS])
        dense_sequence_features, dense_sentence_features = message.get_dense_features(
            attribute, self.component_config[FEATURIZERS]
        )

        if dense_sequence_features is not None and sparse_sequence_features is not None:
            if (
                dense_sequence_features.features.shape[0]
                != sparse_sequence_features.features.shape[0]
            ):
                raise ValueError(


text:
import os
import subprocess
from tempfile import NamedTemporaryFile

from jinja2 import Template

# This file designed in a way that is independent of Django
# in order to be easy (but changes are required) to be used
# outside Django in the future
# That's why is using jinja2 as a template language instead of
# Django's template language.
#
# Example of use:
# Make sure to have jinja2 template language:
# python3 -m venv venv
# pip3 install jinja2
#
# In a Python file:
# import json
# import main # or the name that this file is saved as...
#
# datapackage = json.load(open("datapackage.json"))
# main.datapackage_to_markdown(datapackage)

def datapackage_to_markdown(datapackage):
    """
    datapackage: datapackage schema as a dictionary
    returns: str with the Markdown documentation
    """
    template = Template(template_to_md)
    rendered = template.render(datapackage)

    return rendered.encode('utf-8')

def datapackage_to_pdf(datapackage):
    """
    datapackage: datapackage schema as a dictionary
    returns: binary content with the PDF or None if the conversion failed.
    """
    markdown = datapackage_to_markdown(datapackage)

    f = NamedTemporaryFile(suffix='.pdf', delete=False)
    f.close()

    command_line = ['pandoc', '--to=latex', f'--output={f.name}']

    try:
        pandoc_process = subprocess.run(command_line,
                                        input=markdown)
    except FileNotFoundError:
        os.unlink(f.name)
        raise OSError(f'FileNotFoundError trying to execute: {command_line}')
    except subprocess.CalledProcessError:
        os.unlink(f.name)
        raise RuntimeError(f'CalledProcessError trying to execute: {command_line}')

    if pandoc_process.returncode != 0:
        os.unlink(f.name)
        raise RuntimeError(f'Command {command_line} returned a PDF file of size 0')

    pdf_file = open(f.name, 'rb')

    pdf_content = pdf_file.read()
    os.unlink(f.name)

    return pdf_content

template_to_md = '''# {{ title }}

## Dataset description
{{ description }}
{% if contributors|length == 1 %}
## Contributor
{% else %}
## Contributors
{% endif %}{% for contributor in contributors %} * {{ contributor.title }} ({{ contributor.role }})
{% endfor %}
{% if keywords|length == 1 %}
## Keyword
{% else %}## Keywords
{% endif %}{% for keyword in keywords %} * {{ keyword }}
{% endfor %}
## Version
{{ version }}

## Homepage
[{{ homepage }}]({{ homepage }})

{% if licenses|length == 1 %}
## Dataset license
{% else %}
## Dataset license
{% endif %}{% for license in licenses %} * {{ license.title }} ([{{ license.name }}]({{ license.path }}))
{% endfor %}

## Resources
{% for resource in resources %}
### {{ resource.title }}
 * Name: {{ resource.name }}
 * Profile: {{ resource.profile }}
 * Path: {{ resource.path }}
{% if resource.format %} * Format: {{ resource.format }}{% endif %}
{% if resource.encoding %} * Encoding: {{ resource.encoding }}{% endif %}
{% if resource.description %} * Desription: {{ resource.description }}{% endif %}

{% if resource.schema.fields %}
#### Fields
{% for field in resource.schema.fields %} * **{{ field.name }}** ({{ field.type }}): {{ field.description }}

{% endfor %}
{% endif %}
{% endfor %}
'''


text:
#!/usr/bin/env python2.7
# coding=utf-8
from __future__ import print_function
import sys, os
curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
sys.path.append("../../amalgamation/python/")
sys.path.append("../../python/")

from mxnet_predict import Predictor
import mxnet as mx

import numpy as np
import cv2
import os

class lstm_ocr_model(object):
    # Keep Zero index for blank. (CTC request it)
    CONST_CHAR='0123456789'
    def __init__(self, path_of_json, path_of_params):
        super(lstm_ocr_model, self).__init__()
        self.path_of_json = path_of_json
        self.path_of_params = path_of_params
        self.predictor = None
        self.__init_ocr()

    def __init_ocr(self):
        num_label = 4 # Set your max length of label, add one more for blank
        batch_size = 1

        num_hidden = 100
        num_lstm_layer = 2
        init_c = [('l%d_init_c'%l, (batch_size, num_hidden)) for l in range(num_lstm_layer)]
        init_h = [('l%d_init_h'%l, (batch_size, num_hidden)) for l in range(num_lstm_layer)]
        init_states = init_c + init_h

        all_shapes = [('data', (batch_size, 80 * 30))] + init_states + [('label', (batch_size, num_label))]
        all_shapes_dict = {}
        for _shape in all_shapes:
            all_shapes_dict[_shape[0]] = _shape[1]
        self.predictor = Predictor(open(self.path_of_json).read(),
                                    open(self.path_of_params).read(),
                                    all_shapes_dict)

    def forward_ocr(self, img):
        img = cv2.resize(img, (80, 30))
        img = img.transpose(1, 0)
        img = img.reshape((80 * 30))
        img = np.multiply(img, 1/255.0)
        self.predictor.forward(data=img)
        prob = self.predictor.get_output(0)
        label_list = []
        for p in prob:
            max_index = np.argsort(p)[::-1][0]
            label_list.append(max_index)
        return self.__get_string(label_list)

    def __get_string(self, label_list):
        # Do CTC label rule
        # CTC cannot emit a repeated symbol on consecutive timesteps
        ret = []
        label_list2 = [0] + list(label_list)
        for i in range(len(label_list)):
            c1 = label_list2[i]
            c2 = label_list2[i+1]
            if c2 == 0 or c2 == c1:
                continue
            ret.append(c2)
        # change to ascii
        s = ''
        for l in ret:
            if l > 0 and l < (len(lstm_ocr_model.CONST_CHAR)+1):
                c = lstm_ocr_model.CONST_CHAR[l-1]
            else:
                c = ''
            s += c
        return s

if __name__ == '__main__':
    _lstm_ocr_model = lstm_ocr_model('ocr-symbol.json', 'ocr-0010.params')
    img = cv2.imread('sample.jpg', 0)
    _str = _lstm_ocr_model.forward_ocr(img)
    print('Result: ', _str)


text:
import os
import requests
from os.path import join, isfile

from nerblackbox.modules.datasets.formatter.base_formatter import BaseFormatter

class CoNLL2003Formatter(BaseFormatter):
    def __init__(self):
        ner_dataset = "conll2003"
        ner_tag_list = ["PER", "ORG", "LOC", "MISC"]
        super().__init__(ner_dataset, ner_tag_list)

    ####################################################################################################################
    # ABSTRACT BASE METHODS
    ####################################################################################################################
    def get_data(self, verbose: bool):
        """
        I: get data
        -----------
        :param verbose: [bool]
        :return: -
        """
        url_base = "https://raw.githubusercontent.com/patverga/torch-ner-nlp-from-scratch/master/data/conll2003/"
        targets = ["eng.train", "eng.testa", "eng.testb"]

        for target in targets:
            target_file = join(self.dataset_path, target)

            # fetch tgz from url
            if isfile(target_file):
                if verbose:
                    print(f".. file at {target_file} already exists")
            else:
                url = url_base + target
                myfile = requests.get(url, allow_redirects=True)
                open(target_file, "wb").write(myfile.content)
                if verbose:
                    print(f".. file fetched from {url} and saved at {target_file}")

    def create_ner_tag_mapping(self):
        """
        II: customize ner_training tag mapping if wanted
        -------------------------------------
        :return: ner_tag_mapping: [dict] w/ keys = tags in original data, values = tags in formatted data
        """
        return dict()

    def format_data(self):
        """
        III: format data
        ----------------
        :return: -
        """
        for phase in ["train", "val", "test"]:
            rows = self._read_original_file(phase)
            self._write_formatted_csv(phase, rows)

    def resplit_data(self, val_fraction: float):
        """
        IV: resplit data
        ----------------
        :param val_fraction: [float]
        :return: -
        """
        # train -> train
        df_train = self._read_formatted_csvs(["train"])
        self._write_final_csv("train", df_train)

        # val  -> val
        df_val = self._read_formatted_csvs(["val"])
        self._write_final_csv("val", df_val)

        # test  -> test
        df_test = self._read_formatted_csvs(["test"])
        self._write_final_csv("test", df_test)

    ####################################################################################################################
    # HELPER: READ ORIGINAL
    ####################################################################################################################
    def _read_original_file(self, phase):
        """
        III: format data
        ---------------------------------------------
        :param phase:   [str] 'train' or 'test'
        :return: _rows: [list] of [list] of [str], e.g. [[], ['Inger', 'PER'], ['säger', '0'], ..]
        """
        file_name = {
            "train": "eng.train",
            "val": "eng.testa",
            "test": "eng.testb",
        }
        file_path_original = join(self.dataset_path, file_name[phase])

        _rows = list()
        if os.path.isfile(file_path_original):
            with open(file_path_original) as f:
                for i, row in enumerate(f.readlines()):
                    _rows.append(row.strip().split())
            print(f"\n> read {file_path_original}")

        _rows = [
            [row[0], row[-1]] if (len(row) == 4 and row[0] != "-DOCSTART-") else list()
            for row in _rows
        ]

        return _rows


text:
#  Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
#
#Licensed under the Apache License, Version 2.0 (the "License");
#you may not use this file except in compliance with the License.
#You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#Unless required by applicable law or agreed to in writing, software
#distributed under the License is distributed on an "AS IS" BASIS,
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#See the License for the specific language governing permissions and
#limitations under the License.

import numpy as np
import os
from functools import partial
import logging
import time
import paddle
import paddle.fluid as fluid
import argparse
import network
import reader

logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger("fluid")
logger.setLevel(logging.INFO)

def parse_args():
    parser = argparse.ArgumentParser("gnn")
    parser.add_argument(
        '--train_path', type=str, default='./data/diginetica/train.txt', help='dir of training data')
    parser.add_argument(
        '--config_path', type=str, default='./data/diginetica/config.txt', help='dir of config')
    parser.add_argument(
        '--model_path', type=str, default='./saved_model', help="path of model parameters")
    parser.add_argument(
        '--epoch_num', type=int, default=30, help='number of epochs to train for')
    parser.add_argument(
        '--batch_size', type=int, default=100, help='input batch size')
    parser.add_argument(
        '--hidden_size', type=int, default=100, help='hidden state size')
    parser.add_argument(
        '--l2', type=float, default=1e-5, help='l2 penalty')
    parser.add_argument(
        '--lr', type=float, default=0.001, help='learning rate')
    parser.add_argument(
        '--step', type=int, default=1, help='gnn propogation steps')
    parser.add_argument(
        '--lr_dc', type=float, default=0.1, help='learning rate decay rate')
    parser.add_argument(
        '--lr_dc_step', type=int, default=3, help='the number of steps after which the learning rate decay')
    parser.add_argument(
        '--use_cuda', type=int, default=0, help='whether to use gpu')
    parser.add_argument(
        '--use_parallel', type=int, default=1, help='whether to use parallel executor')
    parser.add_argument(
        '--enable_ce', action='store_true', help='If set, run the task with continuous evaluation logs.')
    return parser.parse_args()

def train():
    args = parse_args()

    if args.enable_ce:
        SEED = 102
        fluid.default_main_program().random_seed = SEED
        fluid.default_startup_program().random_seed = SEED

    batch_size = args.batch_size
    items_num = reader.read_config(args.config_path)
    loss, acc, py_reader, feed_datas = network.network(items_num, args.hidden_size,
                                args.step)

    data_reader = reader.Data(args.train_path, True)
    logger.info("load data complete")

    use_cuda = True if args.use_cuda else False
    use_parallel = True if args.use_parallel else False
    place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()

    exe = fluid.Executor(place)
    step_per_epoch = data_reader.length // batch_size
    optimizer = fluid.optimizer.Adam(
        learning_rate=fluid.layers.exponential_decay(
            learning_rate=args.lr,
            decay_steps=step_per_epoch * args.lr_dc_step,
            decay_rate=args.lr_dc),
        regularization=fluid.regularizer.L2DecayRegularizer(
            regularization_coeff=args.l2))
    optimizer.minimize(loss)

    exe.run(fluid.default_startup_program())

    all_vocab = fluid.global_scope().var("all_vocab").get_tensor()
    all_vocab.set(
        np.arange(1, items_num).astype("int64").reshape((-1, 1)), place)

    feed_list = [e.name for e in feed_datas]

    if use_parallel:
        train_exe = fluid.ParallelExecutor(
            use_cuda=use_cuda, loss_name=loss.name)
    else:
        train_exe = exe

    logger.info("begin train")

    total_time = []
    ce_info = []
    start_time = time.time()
    loss_sum = 0.0
    acc_sum = 0.0
    global_step = 0
    PRINT_STEP = 500
    py_reader.decorate_paddle_reader(data_reader.reader(batch_size, batch_size * 20, True))
    for i in range(args.epoch_num):
        epoch_sum = []
        py_reader.start()
        try:
            while True:
                res = train_exe.run(fetch_list=[loss.name, acc.name])
                loss_sum += res[0].mean()
                acc_sum += res[1].mean()
                epoch_sum.append(res[0].mean())
                global_step += 1
                if global_step % PRINT_STEP == 0:
                    ce_info.append([loss_sum / PRINT_STEP, acc_sum / PRINT_STEP])
                    total_time.append(time.time() - start_time)
                    logger.info("global_step: %d, loss: %.4lf, train_acc: %.4lf" % (
                        global_step, loss_sum / PRINT_STEP, acc_sum / PRINT_STEP))
                    loss_sum = 0.0
                    acc_sum = 0.0
                    start_time = time.time()
        except fluid.core.EOFException:
            py_reader.reset()
        logger.info("epoch loss: %.4lf" % (np.mean(epoch_sum)))
        save_dir = os.path.join(args.model_path, "epoch_" + str(i))
        fetch_vars = [loss, acc]
        fluid.io.save_inference_model(save_dir, feed_list, fetch_vars, exe)
        logger.info("model saved in " + save_dir)

    # only for ce
    if args.enable_ce:
        gpu_num = get_cards(args)
        ce_loss = 0
        ce_acc = 0
        ce_time = 0
        try:
            ce_loss = ce_info[-1][0]
            ce_acc = ce_info[-1][1]
            ce_time = total_time[-1]
        except:
            print("ce info error")
        print("kpis\teach_pass_duration_card%s\t%s" %
                    (gpu_num, ce_time))
        print("kpis\ttrain_loss_card%s\t%f" %
                    (gpu_num, ce_loss))
        print("kpis\ttrain_acc_card%s\t%f" %
                    (gpu_num, ce_acc))

def get_cards(args):
    num = 0
    cards = os.environ.get('CUDA_VISIBLE_DEVICES')
    num = len(cards.split(","))
    return num

if __name__ == "__main__":
    train()


text:
"""Module of sample legends for some commonly used geospatial datasets.
"""
import os
import pkg_resources

# Land Cover datasets in Earth Engine https://developers.google.com/earth-engine/datasets/tags/landcover
builtin_legends = {
    # National Land Cover Database 2016 (NLCD2016) Legend https://www.mrlc.gov/data/legends/national-land-cover-database-2016-nlcd2016-legend
    'NLCD': {
        '11 Open Water': '466b9f',
        '12 Perennial Ice/Snow': 'd1def8',
        '21 Developed, Open Space': 'dec5c5',
        '22 Developed, Low Intensity': 'd99282',
        '23 Developed, Medium Intensity': 'eb0000',
        '24 Developed High Intensity': 'ab0000',
        '31 Barren Land (Rock/Sand/Clay)': 'b3ac9f',
        '41 Deciduous Forest': '68ab5f',
        '42 Evergreen Forest': '1c5f2c',
        '43 Mixed Forest': 'b5c58f',
        '51 Dwarf Scrub': 'af963c',
        '52 Shrub/Scrub': 'ccb879',
        '71 Grassland/Herbaceous': 'dfdfc2',
        '72 Sedge/Herbaceous': 'd1d182',
        '73 Lichens': 'a3cc51',
        '74 Moss': '82ba9e',
        '81 Pasture/Hay': 'dcd939',
        '82 Cultivated Crops': 'ab6c28',
        '90 Woody Wetlands': 'b8d9eb',
        '95 Emergent Herbaceous Wetlands': '6c9fb8'
    },
    # National Wetlands Inventory Legend: https://www.fws.gov/wetlands/data/Mapper-Wetlands-Legend.html
    'NWI': {
        'Freshwater- Forested and Shrub wetland': (0, 136, 55),
        'Freshwater Emergent wetland': (127, 195, 28),
        'Freshwater pond': (104, 140, 192),
        'Estuarine and Marine wetland': (102, 194, 165),
        'Riverine': (1, 144, 191),
        'Lakes': (19, 0, 124),
        'Estuarine and Marine Deepwater': (0, 124, 136),
        'Other Freshwater wetland': (178, 134, 86)
    },
    # MCD12Q1.051 Land Cover Type Yearly Global 500m https://developers.google.com/earth-engine/datasets/catalog/MODIS_051_MCD12Q1
    'MODIS/051/MCD12Q1': {
        '0 Water': '1c0dff',
        '1 Evergreen needleleaf forest': '05450a',
        '2 Evergreen broadleaf forest': '086a10',
        '3 Deciduous needleleaf forest': '54a708',
        '4 Deciduous broadleaf forest': '78d203',
        '5 Mixed forest': '009900',
        '6 Closed shrublands': 'c6b044',
        '7 Open shrublands': 'dcd159',
        '8 Woody savannas': 'dade48',
        '9 Savannas': 'fbff13',
        '10 Grasslands': 'b6ff05',
        '11 Permanent wetlands': '27ff87',
        '12 Croplands': 'c24f44',
        '13 Urban and built-up': 'a5a5a5',
        '14 Cropland/natural vegetation mosaic': 'ff6d4c',
        '15 Snow and ice': '69fff8',
        '16 Barren or sparsely vegetated': 'f9ffa4',
        '254 Unclassified': 'ffffff'
    },
    # GlobCover: Global Land Cover Map https://developers.google.com/earth-engine/datasets/catalog/ESA_GLOBCOVER_L4_200901_200912_V2_3
    'GLOBCOVER': {
        '11 Post-flooding or irrigated croplands': 'aaefef',
        '14 Rainfed croplands': 'ffff63',
        '20 Mosaic cropland (50-70%) / vegetation (grassland, shrubland, forest) (20-50%)': 'dcef63',
        '30 Mosaic vegetation (grassland, shrubland, forest) (50-70%) / cropland (20-50%)': 'cdcd64',
        '40 Closed to open (>15%) broadleaved evergreen and/or semi-deciduous forest (>5m)': '006300',
        '50 Closed (>40%) broadleaved deciduous forest (>5m)': '009f00',
        '60 Open (15-40%) broadleaved deciduous forest (>5m)': 'aac700',
        '70 Closed (>40%) needleleaved evergreen forest (>5m)': '003b00',
        '90 Open (15-40%) needleleaved deciduous or evergreen forest (>5m)': '286300',
        '100 Closed to open (>15%) mixed broadleaved and needleleaved forest (>5m)': '788300',
        '110 Mosaic forest-shrubland (50-70%) / grassland (20-50%)': '8d9f00',
        '120 Mosaic grassland (50-70%) / forest-shrubland (20-50%)': 'bd9500',
        '130 Closed to open (>15%) shrubland (<5m)': '956300',
        '140 Closed to open (>15%) grassland': 'ffb431',
        '150 Sparse (>15%) vegetation (woody vegetation, shrubs, grassland)': 'ffebae',
        '160 Closed (>40%) broadleaved forest regularly flooded - Fresh water': '00785a',
        '170 Closed (>40%) broadleaved semi-deciduous and/or evergreen forest regularly flooded - saline water': '009578',
        '180 Closed to open (>15%) vegetation (grassland, shrubland, woody vegetation) on regularly flooded or waterlogged soil - fresh, brackish or saline water': '00dc83',
        '190 Artificial surfaces and associated areas (urban areas >50%) GLOBCOVER 2009': 'c31300',
        '200 Bare areas': 'fff5d6',
        '210 Water bodies': '0046c7',
        '220 Permanent snow and ice': 'ffffff',
        '230 Unclassified': '743411'
    },
    # Global PALSAR-2/PALSAR Forest/Non-Forest Map https://developers.google.com/earth-engine/datasets/catalog/JAXA_ALOS_PALSAR_YEARLY_FNF
    'JAXA/PALSAR': {
        '1 Forest': '006400',
        '2 Non-Forest': 'FEFF99',
        '3 Water': '0000FF'
    },
    # MCD12Q1.006 MODIS Land Cover Type Yearly Global 500m https://developers.google.com/earth-engine/datasets/catalog/MODIS_006_MCD12Q1
    'MODIS/006/MCD12Q1': {
        '1 Evergreen Needleleaf Forests: dominated by evergreen conifer trees (canopy >2m). Tree cover >60%.': '05450a',
        '2 Evergreen Broadleaf Forests: dominated by evergreen broadleaf and palmate trees (canopy >2m). Tree cover >60%.': '086a10',
        '3 Deciduous Needleleaf Forests: dominated by deciduous needleleaf (larch) trees (canopy >2m). Tree cover >60%.': '54a708',
        '4 Deciduous Broadleaf Forests: dominated by deciduous broadleaf trees (canopy >2m). Tree cover >60%.': '78d203',
        '5 Mixed Forests: dominated by neither deciduous nor evergreen (40-60% of each) tree type (canopy >2m). Tree cover >60%.': '009900',
        '6 Closed Shrublands: dominated by woody perennials (1-2m height) >60% cover.': 'c6b044',
        '7 Open Shrublands: dominated by woody perennials (1-2m height) 10-60% cover.': 'dcd159',
        '8 Woody Savannas: tree cover 30-60% (canopy >2m).': 'dade48',
        '9 Savannas: tree cover 10-30% (canopy >2m).': 'fbff13',
        '10 Grasslands: dominated by herbaceous annuals (<2m).': 'b6ff05',
        '11 Permanent Wetlands: permanently inundated lands with 30-60% water cover and >10% vegetated cover.': '27ff87',
        '12 Croplands: at least 60% of area is cultivated cropland.': 'c24f44',
        '13 Urban and Built-up Lands: at least 30% impervious surface area including building materials, asphalt and vehicles.': 'a5a5a5',
        '14 Cropland/Natural Vegetation Mosaics: mosaics of small-scale cultivation 40-60% with natural tree, shrub, or herbaceous vegetation.': 'ff6d4c',
        '15 Permanent Snow and Ice: at least 60% of area is covered by snow and ice for at least 10 months of the year.': '69fff8',
        '16 Barren: at least 60% of area is non-vegetated barren (sand, rock, soil) areas with less than 10% vegetation.': 'f9ffa4',
        '17 Water Bodies: at least 60% of area is covered by permanent water bodies.': '1c0dff'
    },
    # Oxford MAP: Malaria Atlas Project Fractional International Geosphere-Biosphere Programme Landcover https://developers.google.com/earth-engine/datasets/catalog/Oxford_MAP_IGBP_Fractional_Landcover_5km_Annual
    'Oxford': {
        '0 Water': '032f7e',
        '1 Evergreen_Needleleaf_Fores': '02740b',
        '2 Evergreen_Broadleaf_Forest': '02740b',
        '3 Deciduous_Needleleaf_Forest': '8cf502',
        '4 Deciduous_Broadleaf_Forest': '8cf502',
        '5 Mixed_Forest': 'a4da01',
        '6 Closed_Shrublands': 'ffbd05',
        '7 Open_Shrublands': 'ffbd05',
        '8 Woody_Savannas': '7a5a02',
        '9 Savannas': 'f0ff0f',
        '10 Grasslands': '869b36',
        '11 Permanent_Wetlands': '6091b4',
        '12 Croplands': 'ff4e4e',
        '13 Urban_and_Built-up': '999999',
        '14 Cropland_Natural_Vegetation_Mosaic': 'ff4e4e',
        '15 Snow_and_Ice': 'ffffff',
        '16 Barren_Or_Sparsely_Vegetated': 'feffc0',
        '17 Unclassified': '020202'
    },
    # Canada AAFC Annual Crop Inventory https://developers.google.com/earth-engine/datasets/catalog/AAFC_ACI
    'AAFC/ACI': {
        '10 Cloud': '000000',
        '20 Water': '3333ff',
        '30 Exposed Land and Barren': '996666',
        '34 Urban and Developed': 'cc6699',
        '35 Greenhouses': 'e1e1e1',
        '50 Shrubland': 'ffff00',
        '80 Wetland': '993399',
        '110 Grassland': 'cccc00',
        '120 Agriculture (undifferentiated)': 'cc6600',
        '122 Pasture and Forages': 'ffcc33',
        '130 Too Wet to be Seeded': '7899f6',
        '131 Fallow': 'ff9900',
        '132 Cereals': '660000',
        '133 Barley': 'dae31d',
        '134 Other Grains': 'd6cc00',
        '135 Millet': 'd2db25',
        '136 Oats': 'd1d52b',
        '137 Rye': 'cace32',
        '138 Spelt': 'c3c63a',
        '139 Triticale': 'b9bc44',
        '140 Wheat': 'a7b34d',
        '141 Switchgrass': 'b9c64e',
        '142 Sorghum': '999900',
        '145 Winter Wheat': '92a55b',
        '146 Spring Wheat': '809769',
        '147 Corn': 'ffff99',
        '148 Tobacco': '98887c',
        '149 Ginseng': '799b93',
        '150 Oilseeds': '5ea263',
        '151 Borage': '52ae77',
        '152 Camelina': '41bf7a',
        '153 Canola and Rapeseed': 'd6ff70',
        '154 Flaxseed': '8c8cff',
        '155 Mustard': 'd6cc00',
        '156 Safflower': 'ff7f00',
        '157 Sunflower': '315491',
        '158 Soybeans': 'cc9933',
        '160 Pulses': '896e43',
        '162 Peas': '8f6c3d',
        '167 Beans': '82654a',
        '174 Lentils': 'b85900',
        '175 Vegetables': 'b74b15',
        '176 Tomatoes': 'ff8a8a',
        '177 Potatoes': 'ffcccc',
        '178 Sugarbeets': '6f55ca',
        '179 Other Vegetables': 'ffccff',
        '180 Fruits': 'dc5424',
        '181 Berries': 'd05a30',
        '182 Blueberry': 'd20000',
        '183 Cranberry': 'cc0000',
        '185 Other Berry': 'dc3200',
        '188 Orchards': 'ff6666',
        '189 Other Fruits': 'c5453b',
        '190 Vineyards': '7442bd',
        '191 Hops': 'ffcccc',
        '192 Sod': 'b5fb05',
        '193 Herbs': 'ccff05',
        '194 Nursery': '07f98c',
        '195 Buckwheat': '00ffcc',
        '196 Canaryseed': 'cc33cc',
        '197 Hemp': '8e7672',
        '198 Vetch': 'b1954f',
        '199 Other Crops': '749a66',
        '200 Forest (undifferentiated)': '009900',
        '210 Coniferous': '006600',
        '220 Broadleaf': '00cc00',
        '230 Mixedwood': 'cc9900'
    },
    # Copernicus CORINE Land Cover https://developers.google.com/earth-engine/datasets/catalog/COPERNICUS_CORINE_V20_100m
    'COPERNICUS/CORINE/V20/100m': {
        '111 Artificial surfaces > Urban fabric > Continuous urban fabric': 'E6004D',
        '112 Artificial surfaces > Urban fabric > Discontinuous urban fabric': 'FF0000',
        '121 Artificial surfaces > Industrial, commercial, and transport units > Industrial or commercial units': 'CC4DF2',
        '122 Artificial surfaces > Industrial, commercial, and transport units > Road and rail networks and associated land': 'CC0000',
        '123 Artificial surfaces > Industrial, commercial, and transport units > Port areas': 'E6CCCC',
        '124 Artificial surfaces > Industrial, commercial, and transport units > Airports': 'E6CCE6',
        '131 Artificial surfaces > Mine, dump, and construction sites > Mineral extraction sites': 'A600CC',
        '132 Artificial surfaces > Mine, dump, and construction sites > Dump sites': 'A64DCC',
        '133 Artificial surfaces > Mine, dump, and construction sites > Construction sites': 'FF4DFF',
        '141 Artificial surfaces > Artificial, non-agricultural vegetated areas > Green urban areas': 'FFA6FF',
        '142 Artificial surfaces > Artificial, non-agricultural vegetated areas > Sport and leisure facilities': 'FFE6FF',
        '211 Agricultural areas > Arable land > Non-irrigated arable land': 'FFFFA8',
        '212 Agricultural areas > Arable land > Permanently irrigated land': 'FFFF00',
        '213 Agricultural areas > Arable land > Rice fields': 'E6E600',
        '221 Agricultural areas > Permanent crops > Vineyards': 'E68000',
        '222 Agricultural areas > Permanent crops > Fruit trees and berry plantations': 'F2A64D',
        '223 Agricultural areas > Permanent crops > Olive groves': 'E6A600',
        '231 Agricultural areas > Pastures > Pastures': 'E6E64D',
        '241 Agricultural areas > Heterogeneous agricultural areas > Annual crops associated with permanent crops': 'FFE6A6',
        '242 Agricultural areas > Heterogeneous agricultural areas > Complex cultivation patterns': 'FFE64D',
        '243 Agricultural areas > Heterogeneous agricultural areas > Land principally occupied by agriculture, with significant areas of natural vegetation': 'E6CC4D',
        '244 Agricultural areas > Heterogeneous agricultural areas > Agro-forestry areas': 'F2CCA6',
        '311 Forest and semi natural areas > Forests > Broad-leaved forest': '80FF00',
        '312 Forest and semi natural areas > Forests > Coniferous forest': '00A600',
        '313 Forest and semi natural areas > Forests > Mixed forest': '4DFF00',
        '321 Forest and semi natural areas > Scrub and/or herbaceous vegetation associations > Natural grasslands': 'CCF24D',
        '322 Forest and semi natural areas > Scrub and/or herbaceous vegetation associations > Moors and heathland': 'A6FF80',
        '323 Forest and semi natural areas > Scrub and/or herbaceous vegetation associations > Sclerophyllous vegetation': 'A6E64D',
        '324 Forest and semi natural areas > Scrub and/or herbaceous vegetation associations > Transitional woodland-shrub': 'A6F200',
        '331 Forest and semi natural areas > Open spaces with little or no vegetation > Beaches, dunes, sands': 'E6E6E6',
        '332 Forest and semi natural areas > Open spaces with little or no vegetation > Bare rocks': 'CCCCCC',
        '333 Forest and semi natural areas > Open spaces with little or no vegetation > Sparsely vegetated areas': 'CCFFCC',
        '334 Forest and semi natural areas > Open spaces with little or no vegetation > Burnt areas': '000000',
        '335 Forest and semi natural areas > Open spaces with little or no vegetation > Glaciers and perpetual snow': 'A6E6CC',
        '411 Wetlands > Inland wetlands > Inland marshes': 'A6A6FF',
        '412 Wetlands > Inland wetlands > Peat bogs': '4D4DFF',
        '421 Wetlands > Maritime wetlands > Salt marshes': 'CCCCFF',
        '422 Wetlands > Maritime wetlands > Salines': 'E6E6FF',
        '423 Wetlands > Maritime wetlands > Intertidal flats': 'A6A6E6',
        '511 Water bodies > Inland waters > Water courses': '00CCF2',
        '512 Water bodies > Inland waters > Water bodies': '80F2E6',
        '521 Water bodies > Marine waters > Coastal lagoons': '00FFA6',
        '522 Water bodies > Marine waters > Estuaries': 'A6FFE6',
        '523 Water bodies > Marine waters > Sea and ocean': 'E6F2FF'
    },
    # Copernicus Global Land Cover Layers: CGLS-LC100 collection 2 https://developers.google.com/earth-engine/datasets/catalog/COPERNICUS_Landcover_100m_Proba-V_Global
    'COPERNICUS/Landcover/100m/Proba-V/Global': {
        '0 Unknown': '282828',
        '20 Shrubs. Woody perennial plants with persistent and woody stems and without any defined main stem being less than 5 m tall. The shrub foliage can be either evergreen or deciduous.': 'FFBB22',
        '30 Herbaceous vegetation. Plants without persistent stem or shoots above ground and lacking definite firm structure. Tree and shrub cover is less than 10 %.': 'FFFF4C',
        '40 Cultivated and managed vegetation / agriculture. Lands covered with temporary crops followed by harvest and a bare soil period (e.g., single and multiple cropping systems). Note that perennial woody crops will be classified as the appropriate forest or shrub land cover type.': 'F096FF',
        '50 Urban / built up. Land covered by buildings and other man-made structures.': 'FA0000',
        '60 Bare / sparse vegetation. Lands with exposed soil, sand, or rocks and never has more than 10 % vegetated cover during any time of the year.': 'B4B4B4',
        '70 Snow and ice. Lands under snow or ice cover throughout the year.': 'F0F0F0',
        '80 Permanent water bodies. Lakes, reservoirs, and rivers. Can be either fresh or salt-water bodies.': '0032C8',
        '90 Herbaceous wetland. Lands with a permanent mixture of water and herbaceous or woody vegetation. The vegetation can be present in either salt, brackish, or fresh water.': '0096A0',
        '100 Moss and lichen.': 'FAE6A0',
        '111 Closed forest, evergreen needle leaf. Tree canopy >70 %, almost all needle leaf trees remain green all year. Canopy is never without green foliage.': '58481F',
        '112 Closed forest, evergreen broad leaf. Tree canopy >70 %, almost all broadleaf trees remain green year round. Canopy is never without green foliage.': '009900',
        '113 Closed forest, deciduous needle leaf. Tree canopy >70 %, consists of seasonal needle leaf tree communities with an annual cycle of leaf-on and leaf-off periods.': '70663E',
        '114 Closed forest, deciduous broad leaf. Tree canopy >70 %, consists of seasonal broadleaf tree communities with an annual cycle of leaf-on and leaf-off periods.': '00CC00',
        '115 Closed forest, mixed.': '4E751F',
        '116 Closed forest, not matching any of the other definitions.': '007800',
        '121 Open forest, evergreen needle leaf. Top layer- trees 15-70 % and second layer- mixed of shrubs and grassland, almost all needle leaf trees remain green all year. Canopy is never without green foliage.': '666000',
        '122 Open forest, evergreen broad leaf. Top layer- trees 15-70 % and second layer- mixed of shrubs and grassland, almost all broadleaf trees remain green year round. Canopy is never without green foliage.': '8DB400',
        '123 Open forest, deciduous needle leaf. Top layer- trees 15-70 % and second layer- mixed of shrubs and grassland, consists of seasonal needle leaf tree communities with an annual cycle of leaf-on and leaf-off periods.': '8D7400',
        '124 Open forest, deciduous broad leaf. Top layer- trees 15-70 % and second layer- mixed of shrubs and grassland, consists of seasonal broadleaf tree communities with an annual cycle of leaf-on and leaf-off periods.': 'A0DC00',
        '125 Open forest, mixed.': '929900',
        '126 Open forest, not matching any of the other definitions.': '648C00',
        '200 Oceans, seas. Can be either fresh or salt-water bodies.': '000080'
    },
    # USDA NASS Cropland Data Layers https://developers.google.com/earth-engine/datasets/catalog/USDA_NASS_CDL
    'USDA/NASS/CDL': {
        '1 Corn': 'ffd300',
        '2 Cotton': 'ff2626',
        '3 Rice': '00a8e2',
        '4 Sorghum': 'ff9e0a',
        '5 Soybeans': '267000',
        '6 Sunflower': 'ffff00',
        '10 Peanuts': '70a500',
        '11 Tobacco': '00af49',
        '12 Sweet Corn': 'dda50a',
        '13 Pop or Orn Corn': 'dda50a',
        '14 Mint': '7cd3ff',
        '21 Barley': 'e2007c',
        '22 Durum Wheat': '896054',
        '23 Spring Wheat': 'd8b56b',
        '24 Winter Wheat': 'a57000',
        '25 Other Small Grains': 'd69ebc',
        '26 Dbl Crop WinWht/Soybeans': '707000',
        '27 Rye': 'aa007c',
        '28 Oats': 'a05989',
        '29 Millet': '700049',
        '30 Speltz': 'd69ebc',
        '31 Canola': 'd1ff00',
        '32 Flaxseed': '7c99ff',
        '33 Safflower': 'd6d600',
        '34 Rape Seed': 'd1ff00',
        '35 Mustard': '00af49',
        '36 Alfalfa': 'ffa5e2',
        '37 Other Hay/Non Alfalfa': 'a5f28c',
        '38 Camelina': '00af49',
        '39 Buckwheat': 'd69ebc',
        '41 Sugarbeets': 'a800e2',
        '42 Dry Beans': 'a50000',
        '43 Potatoes': '702600',
        '44 Other Crops': '00af49',
        '45 Sugarcane': 'af7cff',
        '46 Sweet Potatoes': '702600',
        '47 Misc Vegs & Fruits': 'ff6666',
        '48 Watermelons': 'ff6666',


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2014 Daniel Standage <daniel.standage@gmail.com>
# Copyright (c) 2008 Sascha Steinbiss <steinbiss@zbh.uni-hamburg.de>
# Copyright (c) 2008 Center for Bioinformatics, University of Hamburg
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#

from gt.dlload import gtlib
from gt.annotationsketch.rec_map import RecMap
import math

class ImageInfo:

    def __init__(self):
        self.ii = gtlib.gt_image_info_new()
        self._as_parameter_ = self.ii
        self.hotspots = None

    def __del__(self):
        try:
            gtlib.gt_image_info_delete(self.ii)
        except AttributeError:
            pass

    def from_param(cls, obj):
        if not (isinstance(obj, ImageInfo) or obj == None):
            raise TypeError, "argument must be an ImageInfo"
        if obj == None:
            return None
        return obj._as_parameter_

    from_param = classmethod(from_param)

    def get_height(self):
        return gtlib.gt_image_info_get_height(self.ii)

    def num_of_rec_maps(self):
        return gtlib.gt_image_info_num_of_rec_maps(self.ii)

    def compare_hotspots(cls, hs1, hs2):
        if hs1[2] - hs1[0] + 1 > hs2[2] - hs2[0] + 1:
            return 1
        elif hs1[2] - hs1[0] + 1 == hs2[2] - hs2[0] + 1:
            if hs1[3] > hs2[3]:
                return 1
            elif hs1[3] == hs2[3]:
                return 0
            else:
                return -1
        else:
            return -1

    compare_hotspots = classmethod(compare_hotspots)

    def each_hotspot(self):
        if not self.hotspots:
            self.hotspots = []
            for i in range(self.num_of_rec_maps()):
                rm = RecMap(gtlib.gt_image_info_get_rec_map(self.ii, i))
                self.hotspots.append([math.floor(rm.get_northwest_x()),
                        math.floor(rm.get_northwest_y()), math.floor(rm.get_southeast_x()),
                        math.floor(rm.get_southeast_y()), rm.get_genome_feature()])
            self.hotspots.sort(ImageInfo.compare_hotspots)
        for hs in self.hotspots:
            yield (hs[0], hs[1], hs[2], hs[3], hs[4])

    def register(cls, gtlib):
        from ctypes import c_void_p, c_ulong, c_uint
        gtlib.gt_image_info_delete.restype = None
        gtlib.gt_image_info_delete.argtypes = [c_void_p]
        gtlib.gt_image_info_get_rec_map.restype = c_void_p
        gtlib.gt_image_info_get_rec_map.argtypes = [c_void_p, c_ulong]
        gtlib.gt_image_info_num_of_rec_maps.restype = c_ulong
        gtlib.gt_image_info_num_of_rec_maps.argtypes = [c_void_p]
        gtlib.gt_image_info_get_height.restype = c_uint
        gtlib.gt_image_info_get_height.argtypes = [c_void_p]
        gtlib.gt_image_info_new.restype = c_void_p
        gtlib.gt_image_info_new.argtypes = []

    register = classmethod(register)


text:
# -*- coding: utf-8 -*-
from __future__ import print_function
import argparse
import aerospike
from aerospike import exception as e

try:
    from aerospike_helpers.operations import map_operations as mh
except:
    pass  # Needs Aerospike client >= 3.4.0
import datetime
import pprint
import random
import sys
import time

argparser = argparse.ArgumentParser(add_help=False)
argparser.add_argument(
    "--help", dest="help", action="store_true", help="Displays this message."
)
argparser.add_argument(
    "-U",
    "--username",
    dest="username",
    metavar="<USERNAME>",
    help="Username to connect to database.",
)
argparser.add_argument(
    "-P",
    "--password",
    dest="password",
    metavar="<PASSWORD>",
    help="Password to connect to database.",
)
argparser.add_argument(
    "-h",
    "--host",
    dest="host",
    default="127.0.0.1",
    metavar="<ADDRESS>",
    help="Address of Aerospike server.",
)
argparser.add_argument(
    "-p",
    "--port",
    dest="port",
    type=int,
    default=3000,
    metavar="<PORT>",
    help="Port of the Aerospike server.",
)
argparser.add_argument(
    "-n",
    "--namespace",
    dest="namespace",
    default="test",
    metavar="<NS>",
    help="Port of the Aerospike server.",
)
argparser.add_argument(
    "-s",
    "--set",
    dest="set",
    default="profiles",
    metavar="<SET>",
    help="Port of the Aerospike server.",
)
argparser.add_argument(
    "-i",
    "--interactive",
    dest="interactive",
    action="store_true",
    help="Interactive Mode",
)
options = argparser.parse_args()
if options.help:
    argparser.print_help()
    print()
    sys.exit(1)

def version_tuple(version):
    return tuple(int(i) for i in version.split("."))

def pause():
    input("Hit return to continue")

if options.namespace and options.namespace != "None":
    namespace = options.namespace
else:
    namespace = None
set = options.set if options.set and options.set != "None" else None

config = {"hosts": [(options.host, options.port)]}
try:
    client = aerospike.client(config).connect(options.username, options.password)
    policy = {"key": aerospike.POLICY_KEY_SEND}
except e.ClientError as e:
    if not options.quiet:
        print("Error: {0} [{1}]".format(e.msg, e.code))
    sys.exit(2)

version = client.info_all("version")
release = list(version.values())[0][1].split(" ")[-1]
if version_tuple(aerospike.__version__) < version_tuple("3.4.0") or version_tuple(
    release
) < version_tuple("4.6"):
    print(
        "\nPlease use Python client >= 3.4.0, ",
        "Aerospike database >= 4.6 for this example.",
    )
    sys.exit(3)

pp = pprint.PrettyPrinter(indent=2)
spacer = "=" * 30
epoch = datetime.datetime(2019, 1, 1)
now = datetime.datetime.now()
try:
    # Find all segments whose TTL is before this hour
    key = (namespace, set, "u3")
    current_hour = int((now - epoch).total_seconds() / 3600)
    print("\nCurrent hour is {} hours since epoch".format(current_hour))
    if options.interactive:
        pause()

    ops = [
        mh.map_get_by_value_range(
            "u",
            [0, aerospike.null()],
            [current_hour - 1, aerospike.null()],
            aerospike.MAP_RETURN_KEY,
            False,
        ),
        mh.map_size("u"),
    ]
    _, _, b = client.operate_ordered(key, ops)
    stale_segments, total_segments = b
    print("This user has a total of {} segments".format(total_segments[1]))
    print(
        "Of those, a total of {} segments should be cleaned".format(
            len(stale_segments[1])
        )
    )
    print("Show all segments with a segment TTL before the current hour:")
    print(stale_segments)
    print(spacer)

    # Clean up the stale segments using a background scan with a transaction
    # attached to it
    print("Clean the stale segments from the entire namespace")
    if options.interactive:
        pause()
    ops = [
        mh.map_remove_by_value_range(
            "u",
            [0, aerospike.null()],
            [current_hour - 1, aerospike.null()],
            aerospike.MAP_RETURN_NONE,
            False,
        )
    ]
    # _, _, _ = client.operate_ordered(key, ops)
    scan = client.scan(namespace, set)
    scan.add_ops(ops)
    job_id = scan.execute_background()
    # wait for job to finish
    while True:
        response = client.job_info(job_id, aerospike.JOB_SCAN)
        if response["status"] != aerospike.JOB_STATUS_INPROGRESS:
            break
        time.sleep(0.25)

    ops = [
        mh.map_get_by_value_range(
            "u",
            [0, aerospike.null()],
            [current_hour - 1, aerospike.null()],
            aerospike.MAP_RETURN_KEY,
            False,
        ),
        mh.map_size("u"),
    ]
    _, _, b = client.operate_ordered(key, ops)
    stale_segments, total_segments = b
    print("This user now has a total of {} segments".format(total_segments[1]))
    print(
        "Of those, a total of {} segments should be cleaned".format(
            len(stale_segments[1])
        )
    )
    print("Show all segments with a segment TTL before the current hour:")
    print(stale_segments)
    print(spacer)

except Exception as e:
    print("Error: {0} [{1}]".format(e.msg, e.code))
client.close()


text:
from ParseHandler import ParseHandler
from PathHandler import PathHandler
import paths

parser = ParseHandler()
pather = PathHandler()

# match subdirectories in both folders
pather.build_matching_subdir(paths.TOY_RAW, paths.TOY_CLEAN)

# get paths to folders in raw/ directory
dir_names = pather.get_dir_names(paths.TOY_RAW)
raw_dir_paths = pather.get_dir_paths(paths.TOY_RAW)
clean_dir_paths = pather.get_dir_paths(paths.TOY_CLEAN)

# iterate through the contents of each folder
for raw_dir_path, clean_dir_path in zip(raw_dir_paths, clean_dir_paths):

    # get raw file paths from each subdir
    file_names = pather.get_file_names(raw_dir_path)
    raw_file_paths = pather.get_file_paths(raw_dir_path)
    clean_file_paths = [clean_dir_path + file_name for file_name in file_names]

    # parse each raw_file into the clean_file
    for raw_file_path, clean_file_path in zip(raw_file_paths, clean_file_paths):
        parser.parse(raw_file_path, clean_file_path)


text:
#!/usr/bin/env python3
# Copyright (c) 2014-2016 The Bitcoin Core developers
# Copyright (c) 2017 The Californiacoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test running californiacoind with the -rpcbind and -rpcallowip options."""

import socket
import sys

from test_framework.test_framework import CaliforniacoinTestFramework, SkipTest
from test_framework.util import *
from test_framework.netutil import *

class RPCBindTest(CaliforniacoinTestFramework):
    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 1

    def setup_network(self):
        self.add_nodes(self.num_nodes, None)

    def run_bind_test(self, allow_ips, connect_to, addresses, expected):
        '''
        Start a node with requested rpcallowip and rpcbind parameters,
        then try to connect, and check if the set of bound addresses
        matches the expected set.
        '''
        self.log.info("Bind test for %s" % str(addresses))
        expected = [(addr_to_hex(addr), port) for (addr, port) in expected]
        base_args = ['-disablewallet', '-nolisten']
        if allow_ips:
            base_args += ['-rpcallowip=' + x for x in allow_ips]
        binds = ['-rpcbind='+addr for addr in addresses]
        self.nodes[0].rpchost = connect_to
        self.start_node(0, base_args + binds)
        pid = self.nodes[0].process.pid
        assert_equal(set(get_bind_addrs(pid)), set(expected))
        self.stop_nodes()

    def run_allowip_test(self, allow_ips, rpchost, rpcport):
        '''
        Start a node with rpcallow IP, and request getnetworkinfo
        at a non-localhost IP.
        '''
        self.log.info("Allow IP test for %s:%d" % (rpchost, rpcport))
        base_args = ['-disablewallet', '-nolisten'] + ['-rpcallowip='+x for x in allow_ips]
        self.nodes[0].rpchost = None
        self.start_nodes([base_args])
        # connect to node through non-loopback interface
        node = get_rpc_proxy(rpc_url(get_datadir_path(self.options.tmpdir, 0), 0, "%s:%d" % (rpchost, rpcport)), 0, coveragedir=self.options.coveragedir)
        node.getnetworkinfo()
        self.stop_nodes()

    def run_test(self):
        # due to OS-specific network stats queries, this test works only on Linux
        if not sys.platform.startswith('linux'):
            raise SkipTest("This test can only be run on linux.")
        # find the first non-loopback interface for testing
        non_loopback_ip = None
        for name,ip in all_interfaces():
            if ip != '127.0.0.1':
                non_loopback_ip = ip
                break
        if non_loopback_ip is None:
            raise SkipTest("This test requires at least one non-loopback IPv4 interface.")
        try:
            s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
            s.connect(("::1",1))
            s.close
        except OSError:
            raise SkipTest("This test requires IPv6 support.")

        self.log.info("Using interface %s for testing" % non_loopback_ip)

        defaultport = rpc_port(0)

        # check default without rpcallowip (IPv4 and IPv6 localhost)
        self.run_bind_test(None, '127.0.0.1', [],
            [('127.0.0.1', defaultport), ('::1', defaultport)])
        # check default with rpcallowip (IPv6 any)
        self.run_bind_test(['127.0.0.1'], '127.0.0.1', [],
            [('::0', defaultport)])
        # check only IPv4 localhost (explicit)
        self.run_bind_test(['127.0.0.1'], '127.0.0.1', ['127.0.0.1'],
            [('127.0.0.1', defaultport)])
        # check only IPv4 localhost (explicit) with alternative port
        self.run_bind_test(['127.0.0.1'], '127.0.0.1:32171', ['127.0.0.1:32171'],
            [('127.0.0.1', 32171)])
        # check only IPv4 localhost (explicit) with multiple alternative ports on same host
        self.run_bind_test(['127.0.0.1'], '127.0.0.1:32171', ['127.0.0.1:32171', '127.0.0.1:32172'],
            [('127.0.0.1', 32171), ('127.0.0.1', 32172)])
        # check only IPv6 localhost (explicit)
        self.run_bind_test(['[::1]'], '[::1]', ['[::1]'],
            [('::1', defaultport)])
        # check both IPv4 and IPv6 localhost (explicit)
        self.run_bind_test(['127.0.0.1'], '127.0.0.1', ['127.0.0.1', '[::1]'],
            [('127.0.0.1', defaultport), ('::1', defaultport)])
        # check only non-loopback interface
        self.run_bind_test([non_loopback_ip], non_loopback_ip, [non_loopback_ip],
            [(non_loopback_ip, defaultport)])

        # Check that with invalid rpcallowip, we are denied
        self.run_allowip_test([non_loopback_ip], non_loopback_ip, defaultport)
        assert_raises_rpc_error(-342, "non-JSON HTTP response with '403 Forbidden' from server", self.run_allowip_test, ['1.1.1.1'], non_loopback_ip, defaultport)

if __name__ == '__main__':
    RPCBindTest().main()


text:
from .convert_index_to_indices import convert_index_to_indices as index_to_indices
from .convert_indices_to_index import convert_indices_to_index as indices_to_index
from .convert_indices_to_zip_type import convert_indices_to_zip_type as indices_to_zip_type
from .convert_zip_type_to_indices import convert_zip_type_to_indices as zip_type_to_indices


text:
# coding: utf-8

from __future__ import absolute_import
from __future__ import print_function

import warnings

from ruamel.yaml.error import MarkedYAMLError, ReusedAnchorWarning
from ruamel.yaml.compat import utf8

from ruamel.yaml.events import (
    StreamStartEvent, StreamEndEvent, MappingStartEvent, MappingEndEvent,
    SequenceStartEvent, SequenceEndEvent, AliasEvent, ScalarEvent,
)
from ruamel.yaml.nodes import (
    MappingNode, ScalarNode, SequenceNode,
)

__all__ = ['Composer', 'ComposerError']

class ComposerError(MarkedYAMLError):
    pass

class Composer(object):
    def __init__(self):
        self.anchors = {}

    def check_node(self):
        # Drop the STREAM-START event.
        if self.check_event(StreamStartEvent):
            self.get_event()

        # If there are more documents available?
        return not self.check_event(StreamEndEvent)

    def get_node(self):
        # Get the root node of the next document.
        if not self.check_event(StreamEndEvent):
            return self.compose_document()

    def get_single_node(self):
        # Drop the STREAM-START event.
        self.get_event()

        # Compose a document if the stream is not empty.
        document = None
        if not self.check_event(StreamEndEvent):
            document = self.compose_document()

        # Ensure that the stream contains no more documents.
        if not self.check_event(StreamEndEvent):
            event = self.get_event()
            raise ComposerError(
                "expected a single document in the stream",
                document.start_mark, "but found another document",
                event.start_mark)

        # Drop the STREAM-END event.
        self.get_event()

        return document

    def compose_document(self):
        # Drop the DOCUMENT-START event.
        self.get_event()

        # Compose the root node.
        node = self.compose_node(None, None)

        # Drop the DOCUMENT-END event.
        self.get_event()

        self.anchors = {}
        return node

    def compose_node(self, parent, index):
        if self.check_event(AliasEvent):
            event = self.get_event()
            alias = event.anchor
            if alias not in self.anchors:
                raise ComposerError(
                    None, None, "found undefined alias %r"
                    % utf8(alias), event.start_mark)
            return self.anchors[alias]
        event = self.peek_event()
        anchor = event.anchor
        if anchor is not None:  # have an anchor
            if anchor in self.anchors:
                # raise ComposerError(
                #     "found duplicate anchor %r; first occurence"
                #     % utf8(anchor), self.anchors[anchor].start_mark,
                #     "second occurence", event.start_mark)
                ws = "\nfound duplicate anchor {!r}\nfirst occurence {}\nsecond occurence "\
                     "{}".format(
                         (anchor), self.anchors[anchor].start_mark, event.start_mark)
                warnings.warn(ws, ReusedAnchorWarning)
        self.descend_resolver(parent, index)
        if self.check_event(ScalarEvent):
            node = self.compose_scalar_node(anchor)
        elif self.check_event(SequenceStartEvent):
            node = self.compose_sequence_node(anchor)
        elif self.check_event(MappingStartEvent):
            node = self.compose_mapping_node(anchor)
        self.ascend_resolver()
        return node

    def compose_scalar_node(self, anchor):
        event = self.get_event()
        tag = event.tag
        if tag is None or tag == u'!':
            tag = self.resolve(ScalarNode, event.value, event.implicit)
        node = ScalarNode(tag, event.value,
                          event.start_mark, event.end_mark, style=event.style,
                          comment=event.comment)
        if anchor is not None:
            self.anchors[anchor] = node
        return node

    def compose_sequence_node(self, anchor):
        start_event = self.get_event()
        tag = start_event.tag
        if tag is None or tag == u'!':
            tag = self.resolve(SequenceNode, None, start_event.implicit)
        node = SequenceNode(tag, [],
                            start_event.start_mark, None,
                            flow_style=start_event.flow_style,
                            comment=start_event.comment, anchor=anchor)
        if anchor is not None:
            self.anchors[anchor] = node
        index = 0
        while not self.check_event(SequenceEndEvent):
            node.value.append(self.compose_node(node, index))
            index += 1
        end_event = self.get_event()
        if node.flow_style is True and end_event.comment is not None:
            if node.comment is not None:
                print('Warning: unexpected end_event commment in sequence '
                      'node {}'.format(node.flow_style))
            node.comment = end_event.comment
        node.end_mark = end_event.end_mark
        self.check_end_doc_comment(end_event, node)
        return node

    def compose_mapping_node(self, anchor):
        start_event = self.get_event()
        tag = start_event.tag
        if tag is None or tag == u'!':
            tag = self.resolve(MappingNode, None, start_event.implicit)
        node = MappingNode(tag, [],
                           start_event.start_mark, None,
                           flow_style=start_event.flow_style,
                           comment=start_event.comment, anchor=anchor)
        if anchor is not None:
            self.anchors[anchor] = node
        while not self.check_event(MappingEndEvent):
            # key_event = self.peek_event()
            item_key = self.compose_node(node, None)
            # if item_key in node.value:
            #     raise ComposerError("while composing a mapping",
            #             start_event.start_mark,
            #             "found duplicate key", key_event.start_mark)
            item_value = self.compose_node(node, item_key)
            # node.value[item_key] = item_value
            node.value.append((item_key, item_value))
        end_event = self.get_event()
        if node.flow_style is True and end_event.comment is not None:
            node.comment = end_event.comment
        node.end_mark = end_event.end_mark
        self.check_end_doc_comment(end_event, node)
        return node

    def check_end_doc_comment(self, end_event, node):
        if end_event.comment and end_event.comment[1]:
            # pre comments on an end_event, no following to move to
            if node.comment is None:
                node.comment = [None, None]
            assert not isinstance(node, ScalarEvent)
            # this is a post comment on a mapping node, add as third element
            # in the list
            node.comment.append(end_event.comment[1])
            end_event.comment[1] = None


text:
# -*- coding: utf-8 -*-
from datetime import date, time

import pytest
from django.contrib.admin import site as admin_site

from resources.admin.period_inline import PeriodModelForm, prefix_weekday
from resources.models import Period, Resource
from resources.models.unit import Unit
from resources.tests.utils import assert_response_contains, get_form_data

@pytest.mark.django_db
@pytest.mark.parametrize("commit", (False, True))
def test_period_model_form(space_resource, commit):
    period = Period(resource=space_resource, start=date(2015, 8, 1), end=date(2015, 11, 1), name="plop")
    period.full_clean()
    period.save()
    for wd in range(7):
        period.days.create(weekday=wd, opens=time(9, wd * 2), closes=time(12 + wd))

    pmf = PeriodModelForm(instance=period)
    data = get_form_data(pmf, prepared=True)
    # Make every day open at 06, set closed on wednesdays
    for key in list(data.keys()):
        if key.startswith(prefix_weekday(2, "")):
            data[key] = ""
        elif key.endswith("opens"):
            data[key] = "06:00"
    pmf = PeriodModelForm(instance=period, data=data)
    assert pmf.is_valid()
    period = pmf.save(commit=commit)
    if not commit:
        period.save()
        pmf.save_m2m()

    assert all(day.opens.hour == 6 for day in period.days.all())
    assert not period.days.filter(weekday=2).exists()  # Weekdays _got_ closed, yeah?

@pytest.mark.django_db
@pytest.mark.parametrize("model", (Resource, Unit))
def test_period_inline_containing_admins_work(rf, admin_user, model, space_resource, test_unit):
    if model is Resource:
        instance = space_resource
    elif model is Unit:
        instance = test_unit
    else:
        raise NotImplementedError("Unexpected parametrization")

    admin = admin_site._registry[model]  # Sorry for accessing a private member :(
    request = rf.get("/")
    request.user = admin_user
    response = admin.change_view(request, instance.pk)
    assert_response_contains(response, prefix_weekday(2, "opens"))  # should have a weekday field