# Copyright (c) 2015, System Engineering Software Society
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of the System Engineering Software Society nor the
#       names of its contributors may be used to endorse or promote products
#       derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.
# IN NO EVENT SHALL SYSTEM ENGINEERING SOFTWARE SOCIETY BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import functools
import collections
import copy
import weakref

import numpy as np

import data_manager
from icons import SvgIcon
import plugins
import editor_type
import scales
from . import patterns
from sympathy.utils import uuid_generator

COLOR_SCALES = ['cubehelix', 'rainbow', 'blues', 'yellow-green-blue', 'summer',
                'pink', 'brown and blue', 'red and blue']

class NodeTags(object):
    """
    Tags for giving nodes different properties. Used for updating bounded
    things.
    """

    data_reference = 0
    root = 1
    editable = 2
    is_container = 3
    is_rearrangable = 4
    is_deletable = 5

def reference_setter(root, property_, value):
    """
    Setter function for ID which needs to be updated.
    :param root:  Root node.
    :param value: Parameter node.
    """
    old_value = property_.editor.get()

    def update_references(node):
        if (NodeTags.data_reference in node.tags and
                node.editor.get() == old_value):
            node.editor.set(value)
        for child in node.children:
            update_references(child)

    property_.set(value)
    update_references(root)

def remove_node(node):
    """
    Remove node from its parent.
    :param node: Node to be removed.
    """
    if node.parent is not None:
        node.parent.remove_child(node)

def insert_node(node, parent_node, position=None):
    """
    Insert node with the given parent. An exception is thrown if the node type
    is not allowed as a child of the parent node.
    :param node: Node to be added as child.
    :param parent_node: Parent-to-be of node.
    :param position: Position to insert node at.
    """
    if node.__class__ not in parent_node.valid_children:
        raise TypeError('{} is not valid child of {}'.
                        format(node.__class__, parent_node.__class__))
    if position is None:
        parent_node.insert_child(len(parent_node.children), node)
    else:
        parent_node.insert_child(position, node)
    node.parent = parent_node

def move_node(node, new_parent_node, position=None):
    """
    Move node to new parent node.
    :param node: Node to move.
    :param new_parent_node: New parent node.
    :param position: Child position at new parent.
    """
    remove_node(node)
    insert_node(node, new_parent_node, position)

def is_parent_same_node(parent, node):
    """
    Check if the parent, or any of its parents, is equal to the given node.
    :param parent: Parent to start checking for.
    :param node: Node to check for equality.
    :return: True if there is a parent which is equal to the node.
    """
    if parent is None:
        return False
    if parent == node:
        return True
    return is_parent_same_node(parent.parent, node)

# TODO(stefan): These functions should probably be related to the default
#               data structure of each class instead.
def create_empty_data():
    """
    Create empty data structure.
    :return: Dictionary containing empty data structure.
    """
    s = {
        'type': 'root',
        'sytype': 'report',
        'version': 1,
        'signals': [],
        'scales': [],
        'pages': []
    }
    return s

def create_empty_scale(model_node):
    """
    Create empty scale.
    :param model_node: A model node to access model.
    """
    # Find a free unique scale name.
    scale_list = list_of_scales(model_node, filter_type_compatibility=False)
    n = 1
    scale_id = 'scale1'
    while scale_id in scale_list:
        n += 1
        scale_id = 'scale{}'.format(n)

    s = {
        'id': scale_id,
        'type': 'linear',
        'domain': [-1, 1],
        'extent': False,
        'range': [-1, 1],
        'invalid': ''
    }
    return s

def list_of_scales(property_node, filter_type_compatibility=True):
    """
    Generate a list of available scales.
    :param property_node: A node in the data model, must be Property to filter.
    :param filter_type_compatibility: Keep items which are compatible.
    :return: List of strings with name of scales.
    """
    if filter_type_compatibility:
        assert (isinstance(property_node, Property))
    scale_node_list = property_node.find_all_nodes_with_class(RootScale)
    scale_list = []
    if filter_type_compatibility:
        property_is_color = isinstance(
            property_node.editor, (editor_type.Color, editor_type.ColorScale))
    for scale_node in scale_node_list:
        properties = scale_node.properties_as_dict()
        range_value = properties['range'].get()
        if filter_type_compatibility:
            try:
                is_color = patterns.re_color.match(range_value[0]) is not None
            except TypeError:
                is_color = False
            if ((property_is_color and is_color) or
                    not (property_is_color or is_color)):
                scale_list.append(properties['id'].get())
        else:
            scale_list.append(properties['id'].get())
    return scale_list

def compress_signals(data_dict):
    """
    Compress signal list by removing entries in the signals list which cannot
    be found in the rest of the document.
    :param data_dict: data dict
    :return: compressed list of signals
    """
    found_signals = set()

    def search(signal_name, node):
        if isinstance(node, basestring):
            if node == signal_name:
                found_signals.add(signal_name)
        elif isinstance(node, collections.Sequence):
            for item in node:
                search(signal_name, item)
        elif isinstance(node, collections.Mapping):
            for value in node.itervalues():
                search(signal_name, value)

    for signal in data_dict['signals']:
        search(signal, data_dict['pages'])

    return list(found_signals)

class AbstractNode(object):
    """Abstract base for all nodes."""

    icon = SvgIcon.blank
    # TODO(stefan): Replace this with some kind of schema for flexibility.
    valid_children = frozenset()
    default_data = None
    weak_parent = None

    def __init__(self, data, parent=None):
        self.data = data
        self.parent = parent
        self.children = []
        self.properties = []
        self.tags = set()
        self.init()

    def init(self):
        pass

    @property
    def parent(self):
        return self.weak_parent() if self.weak_parent is not None else None

    @parent.setter
    def parent(self, value):
        self.weak_parent = weakref.ref(value) if value is not None else None

    @classmethod
    def create_empty_instance(cls, parent=None):
        if cls.default_data is not None:
            return cls(copy.deepcopy(cls.default_data), parent=parent)

    def find_node_by_python_id(self, python_id):
        """
        Find object instance of node given its Python id.
        :param python_id: Python id.
        :return: Object if found, None otherwise.
        """
        current_node = self.root_node()
        if id(current_node) == python_id:
            return current_node

        def search(x):
            for n in x.children:
                if id(n) == python_id:
                    return n
                else:
                    result = search(n)
                    if result is not None:
                        return result
            return None

        return search(current_node)

    def root_node(self):
        """
        Find root node.
        :return: Root node.
        """
        if NodeTags.root in self.tags:
            return self
        else:
            return self.parent.root_node()

    def root_data(self):
        """
        Find data of root node.
        :return: Root node data.
        """
        return self.root_node().data

    def find_node(self, node_class, node_id, parent_node=None):
        """
        Search recursively for a node with the given class and ID.
        :param node_class: Class of node.
        :param node_id: String containing ID to search for.
        :param parent_node: Node to search from.
        :return: Found node or None.
        """
        if parent_node is None:
            parent_node = self.root_node()

        for child in parent_node.children:
            if isinstance(child, node_class) and child.data['id'] == node_id:
                return child
            if child.has_children():
                sub_child = self.find_node(node_class, node_id, child)
                if sub_child is not None:
                    return sub_child
        return None

    def find_all_nodes_with_class(self, node_class, parent_node=None):
        """
        Search recursively for all nodes of a given class.
        :param node_class: Class of node.
        :param parent_node: Node to search from.
        :return: List of found nodes.
        """
        found = []
        if parent_node is None:
            parent_node = self.root_node()

        for child in parent_node.children:
            if isinstance(child, node_class):
                found.append(child)
            if child.has_children():
                sub_child = self.find_all_nodes_with_class(node_class, child)
                found.extend(sub_child)
        return found

    def find_child_property(self, property_id):
        """
        Search for child property among children of this node.
        :param property_id: Id of property.
        :return: Property object or None.
        """
        for p in self.properties:
            if p.property == property_id:
                return p
        return None

    def properties_as_dict(self):
        """
        Return all properties which are direct children as a dict.
        :return: Dict with properties.
        """
        property_dict = collections.OrderedDict()
        for p in self.properties:
            property_dict[p.property] = p
        return property_dict

    def remove_child(self, node):
        raise NotImplementedError()

    def insert_child(self, position, node):
        raise NotImplementedError()

    @property
    def label(self):
        """Return label to be visualized in tree."""
        raise NotImplementedError

    def has_children(self):
        """Return True if the node has any children."""
        return self.row_count() > 0

    def row_count(self):
        """Return the number of children."""
        return len(self.children)

    def child(self, row):
        """Return child number."""
        return self.children[row]

    def index(self, child):
        """Return index of child."""
        return self.children.index(child)

class AbstractLeaf(AbstractNode):
    """Convenience class describing a leaf node."""

    def row_count(self):
        return 0

class Property(AbstractLeaf):
    """A general property which can be edited."""

    def __init__(self, parameters, parent=None):
        """
        Property constructor.
        :param parameters: Dictionary containing label, property, icon,
                           editor, data, getter_function and setter_function.
                           'label' visual label for property.
                           'property' is property name.
                           'icon' is icon of property.
                           'editor' is editor specification for property.
                           'data' is pointer to parent object in data
                           structure where property can be found.
                           'getter' is getter function of value (optional).
                           'setter' is setter function of value (optional).
                           'scale_bindable' is True if it is possible to bind
                           the property to a scale (optional).
                           'scale_binding' is a scales.ScaleBinding object
                           (optional).
        :param parent: Parent object.
        """
        super(Property, self).__init__(parameters['data'], parent)
        self._label = parameters['label']
        self.property = parameters['property']
        self.icon = parameters['icon']
        self.getter = parameters.get('getter', None)
        self.setter = parameters.get('setter', None)
        self.tags.add(NodeTags.editable)

        self.scale_binding = parameters.get('scale_binding', None)
        self.scale_bindable = parameters.get('scale_bindable', False)
        # Used as a memory if enabling a scale again.
        self.last_scale_binding = self.scale_binding

        # Editors has to be initialized last since they depend on consistent
        # data in this object.
        self.editor = parameters['editor']
        self.editor.property_object = self
        self.editor.init()

    @property
    def label(self):
        return self._label

    def get(self):
        if self.is_bindable:
            return (self.data[self.property]['value']
                    if self.getter is None else self.getter())
        return (self.data.get(self.property, None)
                if self.getter is None else self.getter())

    def set(self, value):
        if self.setter is None:
            if self.is_bindable:
                self.data[self.property]['value'] = value
            else:
                self.data[self.property] = value
        else:
            self.setter(value)

    def set_scale_binding(self, data_name, scale_name):
        if not self.scale_bindable:
            raise RuntimeError('Binding a scale is not possible on this '
                               'property.')
        # TODO(stefan): Check that the scale is valid.
        self.scale_binding = scales.ScaleBinding(data_name, scale_name)
        self.last_scale_binding = self.scale_binding

        self.data[self.property]['binding'] = (
            None if self.scale_binding is None else
            self.scale_binding.as_dict())

    def clear_scale_binding(self):
        self.scale_binding = None
        self.data[self.property]['binding'] = None

    def has_valid_binding(self):
        # TODO(stefan): Add type constraint (numeric or color).
        if self.scale_binding is None:
            return False
        data_list = data_manager.data_source.signal_list()
        if self.scale_binding.data_id not in data_list:
            return False
        scale_list = list_of_scales(self)
        if self.scale_binding.scale_id not in scale_list:
            return False
        return True

    @property
    def is_bindable(self):
        return self.scale_bindable

    @property
    def has_binding(self):
        return self.scale_binding is not None

    def invalidate_scale(self, scale_name):
        """Invalidate a scale by e.g. removing a reference to it."""
        if (self.scale_binding is not None and
                self.scale_binding.scale_id == scale_name):
            self.clear_scale_binding()
            self.last_scale_binding = None
        else:
            # TODO(stefan): This could be dangerous.
            if self.get() == scale_name:
                self.set('')

class Root(AbstractNode):
    """Root node."""

    def init(self):
        self.valid_children = {RootScales, Pages}
        self.tags.add(NodeTags.root)
        self.children = []

        # Build up children member gradually
        if self.data:
            root_scales = RootScales(self.data['scales'], self)
            self.children = (root_scales,)
            pages = Pages(self.data['pages'], self)
            self.children = (root_scales, pages)

class RootScales(AbstractNode):
    """Container of all defined scales in root."""

    icon = SvgIcon.scales

    def init(self):
        self.valid_children = {RootScale}
        self.children = [RootScale(data, self) for data in self.data]

    @property
    def label(self):
        return 'Scales'

    def remove_child(self, node):
        self.data.remove(node.data)
        self.children.remove(node)
        node.parent = None

        def search_and_replace(value, n):
            for p in n.properties:
                p.invalidate_scale(value)
            for child in n.children:
                search_and_replace(value, child)

        # Also find all instances of the scale in the data tree and clear them.
        scale_id = node.data['id']
        search_and_replace(scale_id, self.root_node())

    def insert_child(self, position, node):
        self.data.insert(position, node.data)
        self.children.insert(position, node)

class RootScale(AbstractNode):
    """Definition of scale in Root -> Scales -> ..."""

    icon = SvgIcon.scales

    SCALE_PROPERTIES = (
        ('Id', 'id', SvgIcon.text, editor_type.String),
        ('Type', 'type', SvgIcon.config, editor_type.ImmutableList),
        ('Domain', 'domain', SvgIcon.ruler, editor_type.String),
        ('Range', 'range', SvgIcon.ruler, editor_type.String),
        ('Extent', 'extent', SvgIcon.blank, editor_type.Boolean),
        ('Invalid', 'invalid', SvgIcon.blank, editor_type.String)
    )

    def init(self):
        self.tags.add(NodeTags.is_deletable)

        for label_, property_, icon, editor in self.SCALE_PROPERTIES:
            if property_ == 'type':
                # TODO(stefan): This data should not be hard coded.
                options = lambda: scales.SCALE_TYPES
            else:
                options = lambda: None

            prop = Property({'label': label_,
                             'property': property_,
                             'icon': icon,
                             'editor': editor(options),
                             'data': self.data},
                            parent=self)

            if property_ == 'id':
                prop.editor.set = functools.partial(
                    reference_setter, self.root_node(), prop)
                prop.editor.tags.add(
                    editor_type.EditorTags.force_update_after_edit)
            elif property_ in ('domain', 'range', 'extent', 'invalid'):
                prop.editor.tags.add(
                    editor_type.EditorTags.force_rebuild_after_edit)

            self.properties.append(prop)

    @property
    def label(self):
        return 'Scale ({})'.format(self.data['id'])

    def create_scale(self, extent_data=None):
        p = self.properties_as_dict()
        if p['extent'].get() and extent_data is None:
            raise ValueError('extent_data must be provided when extent is '
                             'enabled.')
        if not p['extent'].get():
            return scales.create_scale(p['type'].get(),
                                       p['domain'].get(),
                                       p['range'].get(),
                                       p['invalid'].get())
        else:
            domain = np.linspace(extent_data.min(),
                                 extent_data.max(),
                                 len(p['range'].get()))
            return scales.create_scale(p['type'].get(),
                                       domain,
                                       p['range'].get(),
                                       p['invalid'].get())

class Pages(AbstractNode):
    """Root node for all pages. Root -> Pages"""

    icon = SvgIcon.pages

    def init(self):
        self.valid_children = {Page}
        self.tags.add(NodeTags.is_container)

        for page in self.data:
            self.children.append(
                VIEW_TO_CLASS[page['type']](page, parent=self))

    @property
    def label(self):
        return 'Pages'

    def remove_child(self, node):
        self.data.remove(node.data)
        self.children.remove(node)

    def insert_child(self, position, node):
        self.data.insert(position, node.data)
        self.children.insert(position, node)

class Page(AbstractNode):
    """Page definition. Root -> Pages -> ..."""

    icon = SvgIcon.page
    # Check rebuild_widgets in gui.py for comments regarding adding a unique
    # identifier to each page.
    default_data = {
        'type': 'page',
        'title': 'New Page',
        'content': None
    }

    def init(self):
        # Make sure that the page has a unique identifier.
        self.data.setdefault('uuid', uuid_generator.generate_uuid())

        # Storage for page thumbnail.
        self.data.setdefault('thumbnail')

        self.tags.add(NodeTags.is_deletable)

        self.valid_children = {Layout}
        content = self.data['content']
        if content is not None:
            self.children.append(
                VIEW_TO_CLASS[content[0]['type']](content[0], parent=self))
        self.tags.add(NodeTags.is_container)
        self.tags.add(NodeTags.is_rearrangable)

        prop = Property({'label': 'Title',
                         'property': 'title',
                         'icon': SvgIcon.label,
                         'editor': editor_type.String(),
                         'data': self.data},
                        parent=self)
        # this line only updates the page tree view
        prop.editor.tags.add(editor_type.EditorTags.force_update_after_edit)
        self.properties.append(prop)

    @property
    def uuid(self):
        return self.data['uuid']

    @property
    def label(self):
        return self.data['title']

    @property
    def thumbnail(self):
        return self.data['thumbnail']

    @thumbnail.setter
    def thumbnail(self, value):
        self.data['thumbnail'] = value

    def insert_child(self, position, node):
        if len(self.children) == 1:
            raise RuntimeError('A page may only have one child.')
        self.data['content'] = [node.data]
        self.children.insert(position, node)

    def remove_child(self, node):
        self.data['content'] = None
        self.children.remove(node)

class Layout(AbstractNode):
    """Definition of layout in a page."""

    icon = SvgIcon.layout
    default_data = {
        'type': 'layout',
        'kind': 'horizontal',
        'items': None
    }
    layout_kinds = [
        'horizontal',
        'vertical'
    ]

    def init(self):
        self.tags.add(NodeTags.is_rearrangable)
        self.tags.add(NodeTags.is_deletable)

        self.valid_children = {Layout, Graph, TextBox, Image}

        if self.data['items'] is None:
            self.data['items'] = []

        editor = editor_type.ImmutableList(lambda: self.layout_kinds)
        editor.tags.add(editor_type.EditorTags.force_rebuild_after_edit)

        self.properties.append(Property({'label': 'Kind',
                                         'property': 'kind',
                                         'icon': SvgIcon.blank,
                                         'editor': editor,
                                         'data': self.data},
                                        parent=self))
        for item in self.data['items']:
            self.children.append(VIEW_TO_CLASS[item['type']](item, self))
        self.tags.add(NodeTags.is_container)
        self.tags.add(NodeTags.is_rearrangable)

    @property
    def label(self):
        return 'Layout ({})'.format(self.data['kind'])

    def remove_child(self, node):
        self.data['items'].remove(node.data)
        self.children.remove(node)

    def insert_child(self, position, node):
        self.data['items'].insert(position, node.data)
        self.children.insert(position, node)

class TextBox(AbstractNode):
    """Free text box."""

    icon = SvgIcon.text
    default_data = {
        'type': 'textbox',
        'id': 'new-textbox',
        'text': '<Replace me>',
        'halign': 'center',
        'valign': 'center',
        'width': 0,
        'height': 0
    }

    TEXTBOX_NODE_PROPERTIES = (
        ('Id', 'id', SvgIcon.text, editor_type.String),
        ('Text (HTML allowed)', 'text', SvgIcon.text, editor_type.String),
        ('Horizontal Alignment', 'halign', SvgIcon.blank,
         editor_type.ImmutableList),
        ('Vertical Alignment', 'valign', SvgIcon.blank,
         editor_type.ImmutableList),
        ('Width', 'width', SvgIcon.text, editor_type.Integer),
        ('Height', 'height', SvgIcon.text, editor_type.Integer)
    )

    def init(self):
        self.tags.add(NodeTags.is_deletable)
        self.tags.add(NodeTags.is_rearrangable)

        for label, property_, icon, editor_class in (
                self.TEXTBOX_NODE_PROPERTIES):
            if property_ in ('halign',):
                editor = editor_class(lambda: ('left', 'center', 'right'))
            elif property_ in ('valign',):
                editor = editor_class(lambda: ('top', 'center', 'bottom'))
            else:
                editor = editor_class()
            prop = Property({'label': label,
                             'property': property_,
                             'icon': icon,
                             'editor': editor,
                             'data': self.data})
            if property_ in ('id', 'text'):
                editor.tags.add(editor_type.EditorTags.force_update_after_edit)
            if property_ in ('width', 'height'):
                editor.value_range = {'min': 0, 'max': 100000, 'step': 10}
            self.properties.append(prop)

    @property
    def label(self):
        return 'Textbox ({})'.format(self.data['id'])

class Image(AbstractNode):
    """Static image node."""

    icon = SvgIcon.picture
    default_data = {
        'type': 'image',
        'id': 'new-image',
        'image': ''
    }

    IMAGE_NODE_PROPERTIES = (
        ('Id', 'id', SvgIcon.text, editor_type.String),
        ('Image', 'image', SvgIcon.picture, editor_type.Image)
    )

    def init(self):
        self.tags.add(NodeTags.is_deletable)
        self.tags.add(NodeTags.is_rearrangable)

        for label, property_, icon, editor_class in (
                self.IMAGE_NODE_PROPERTIES):
            editor = editor_class()
            prop = Property({'label': label,
                             'property': property_,
                             'icon': icon,
                             'editor': editor,
                             'data': self.data})
            if property_ in ('id',):
                editor.tags.add(editor_type.EditorTags.force_update_after_edit)
            if property_ in ('image',):
                editor.tags.add(
                    editor_type.EditorTags.force_rebuild_after_edit)
            self.properties.append(prop)

    @property
    def label(self):
        return 'Image ({})'.format(self.data['id'])

class Graph(AbstractNode):
    """Definition of a graph in a page."""

    icon = SvgIcon.plot
    default_data = {
        'type': 'graph',
        'id': 'new-graph',
        'title': 'New Graph',
        'width': 400,
        'height': 400,
        'grid': False,
        'projection': 'cartesian',
        'dimensions': [
            [
                {
                    'id': 'x-axis',
                    'title': '',
                    'extent': True,
                    'min': 0.0,
                    'max': 1.0,
                    'scale_type': 'linear'
                }
            ],
            [
                {
                    'id': 'y-axis',
                    'title': '',
                    'extent': True,
                    'min': 0.0,
                    'max': 1.0,
                    'scale_type': 'linear'
                }
            ]
        ],
        'layers': []
    }
    projection_options = [
        'cartesian',
        'polar'
    ]

    GRAPH_NODE_PROPERTIES = (
        ('Id', 'id', SvgIcon.text, editor_type.String),
        ('Title', 'title', SvgIcon.text, editor_type.String),
        ('Width', 'width', SvgIcon.width, editor_type.Integer),
        ('Height', 'height', SvgIcon.height, editor_type.Integer),
        ('Grid', 'grid', SvgIcon.grid, editor_type.Boolean),
        ('Projection', 'projection', SvgIcon.projection,
         editor_type.ImmutableList)
    )

    def init(self):
        self.valid_children = {GraphLayers, GraphDimensions}
        self.tags.add(NodeTags.is_rearrangable)
        self.tags.add(NodeTags.is_deletable)

        for label, property_, icon, editor_class in self.GRAPH_NODE_PROPERTIES:
            if property_ == 'projection':
                # TODO(stefan): Extract this data out of code.
                options = lambda: self.projection_options
            else:
                options = lambda: None
            editor = editor_class(options)
            if property_ in ('projection',):
                editor.tags.add(
                    editor_type.EditorTags.force_rebuild_after_edit)
            if property_ in ('id',):
                editor.tags.add(editor_type.EditorTags.force_update_after_edit)
            prop = Property({'label': label,
                             'property': property_,
                             'icon': icon,
                             'editor': editor,
                             'data': self.data},
                            parent=self)
            if property_ in ('width', 'height'):
                editor.value_range = {'min': 0, 'max': 100000, 'step': 10}
            self.properties.append(prop)

        self.children.append(
            GraphDimensions(self.data['dimensions'], parent=self))
        self.children.append(GraphLayers(self.data['layers'], parent=self))

    @property
    def label(self):
        return 'Graph ({})'.format(self.data['id'])

    def layers(self):
        """Return a list of all layer objects."""
        layer_parent = None
        for child in self.children:
            if isinstance(child, GraphLayers):
                layer_parent = child

        if layer_parent is None:
            return None

        layers = []

        for child in layer_parent.children:
            if isinstance(child, GraphLayer):
                layers.append(child)

        return layers

class GraphDimensions(AbstractNode):
    """Container for dimensions of graph."""

    icon = SvgIcon.coordinates

    def init(self):
        self.valid_children = {GraphDimension}
        self.children = [GraphDimension(dimension, data, parent=self)
                         for dimension, data in enumerate(self.data)]

    @property
    def label(self):
        return 'Dimensions'

    def remove_child(self, node):
        self.data.remove(node.data)
        self.children.remove(node)

    def insert_child(self, position, node):
        self.data.insert(position, node.data)
        self.children.insert(position, node)

class GraphDimension(AbstractNode):
    """Definition of a graph dimension."""

    def __init__(self, dimension, data, parent):
        super(GraphDimension, self).__init__(data, parent)
        self.dimension_number = dimension
        dim = (SvgIcon.x_axis, SvgIcon.y_axis, SvgIcon.z_axis)
        self.icon = (dim[dimension] if dimension < 3 else SvgIcon.n_axis)

    def init(self):
        self.valid_children = {GraphAxis}
        self.children = [GraphAxis(data, parent=self) for data in self.data]

    @property
    def label(self):
        return 'Dimension {}'.format(self.dimension_number + 1)

class GraphAxis(AbstractNode):
    icon = SvgIcon.ruler

    AXIS_NODE_PROPERTIES = (
        ('Id', 'id', SvgIcon.text, editor_type.String),
        ('Title', 'title', SvgIcon.text, editor_type.String),
        ('Fit to data', 'extent', SvgIcon.blank, editor_type.Boolean),
        ('Min', 'min', SvgIcon.scales, editor_type.Float),
        ('Max', 'max', SvgIcon.scales, editor_type.Float),
        ('Scale', 'scale_type', SvgIcon.blank, editor_type.ImmutableList)
    )

    def __init__(self, data, parent):
        # Update old data model to newer model:
        if 'scale' in data:
            scale_id = data['scale']
            del data['scale']
            scale_model = parent.root_node().find_node(RootScale, scale_id)
            if scale_model:
                data['min'] = scale_model.data['domain'][0]
                data['max'] = scale_model.data['domain'][-1]
                data['extent'] = scale_model.data.get('extent', True)
                data['scale_type'] = scale_model.data['type']
            else:
                data['min'] = 0.0
                data['max'] = 1.0
                data['extent'] = True
                data['scale_type'] = 'linear'

        super(GraphAxis, self).__init__(data, parent)

    def init(self):
        for label, property_, icon, editor in self.AXIS_NODE_PROPERTIES:
            if property_ == 'scale_type':
                options = lambda: ['linear', 'log']
            else:
                options = lambda: None
            prop = Property({'label': label,
                             'property': property_,
                             'icon': icon,
                             'editor': editor(options),
                             'data': self.data},
                            parent=self)
            if property_ == 'id':
                # An id-change here should only propagate to stuff within
                # the graph, not the root.
                prop.editor.set = functools.partial(
                    reference_setter, self.parent.parent.parent, prop)
                prop.editor.tags.add(
                    editor_type.EditorTags.force_update_after_edit)

            self.properties.append(prop)

    @property
    def label(self):
        return 'Axis ({})'.format(self.data['id'])

class GraphLayers(AbstractNode):
    """Container for layers in graph."""

    icon = SvgIcon.layers

    def init(self):
        self.valid_children = {GraphLayer}
        self.children = [GraphLayer(data, parent=self) for data in self.data]
        self.tags.add(NodeTags.is_container)

    @property
    def label(self):
        return 'Layers'

    def remove_child(self, node):
        self.data.remove(node.data)
        self.children.remove(node)

    def insert_child(self, position, node):
        self.data.insert(position, node.data)
        self.children.insert(position, node)

class GraphLayer(AbstractNode):
    """Container for a layer."""

    icon = SvgIcon.layer
    layer = None
    # default_data = None   This is set before instantiated.

    def init(self):
        self.valid_children = {GraphLayerData}
        self.tags.add(NodeTags.is_rearrangable)
        self.tags.add(NodeTags.is_deletable)

        # First add all data specification nodes to the children.
        self.children = [GraphLayerData(self.data['data'], parent=self)]

        layer_type = self.data['type']
        try:
            # Fetch Layer class using the plugin framework for the given
            # layer type.
            layer = plugins.layer_modules[layer_type].layer.Layer()
        except KeyError:
            raise ValueError('Unknown layer type {}'.format(layer_type))
        layer_properties = layer.create_properties(layer_model=self,
                                                   property_class=Property)
        self.properties.extend(layer_properties.values())

    @property
    def label(self):
        return '{} ({})'.format(
            self.data.get('name', '---'), self.data['type'])

    def extract_data_and_properties(self):
        """Convenience method for extracting data and properties from layer
        model.
        """
        # Extract all data source properties. Assuming one source per
        # dimension.
        data_source_properties = []
        # Loop through all children.
        for child_ in self.children:
            # If we have data...
            if isinstance(child_, GraphLayerData):
                # ...go through all dimensions...
                for data_dimension in child_.children:
                    # ...and all its properties...
                    for data_dimension_property in data_dimension.properties:
                        # ...and extract all data sources.
                        if data_dimension_property.property == 'source':
                            data_source_properties.append(
                                data_dimension_property)
        try:
            data = [data_manager.data_source.data(x.get())
                    for x in data_source_properties]
        except KeyError:
            data = [np.array([]) for _ in data_source_properties]
        if len(data_source_properties) == 0:
            data_source_properties = None
        return data, data_source_properties

    def extent_of_layer_data(self, dimension):
        """
        Extract extent (data range) of layer data for the given dimension.
        :param dimension: Dimension to extract extent for.
        :return: Tuple with (min, max). (0, 1) if the extent couldn't be
                 calculated.
        """
        for c1 in self.children:
            if isinstance(c1, GraphLayerData):
                try:
                    c_dim = c1.children[dimension]
                except IndexError:
                    return 0, 1
                properties = c_dim.properties_as_dict()
                data_source = properties['source'].get()
                data = data_manager.data_source.data(data_source)
                try:
                    min_, max_ = np.nanmin(data), np.nanmax(data)
                except (ValueError, TypeError):
                    pass
                else:
                    if min_ is not None and max_ is not None:
                        if min_ == max_:
                            # zero length axes are no fun, so add a little bit
                            # wiggle room.
                            min_ = min_ - 0.5
                            max_ = max_ + 0.5
                        return min_, max_

        # Arbitrary default values to avoid breaking calculations.
        return 0, 1

class GraphLayerData(AbstractNode):
    """
    Data container containing information regarding what data is represented
    in a layer.
    """

    icon = SvgIcon.data

    def init(self):
        self.valid_children = {GraphLayerDataDimension}
        self.children = [GraphLayerDataDimension(dimension, data, parent=self)
                         for dimension, data in enumerate(self.data)]

    @property
    def label(self):
        return 'Data'

class GraphLayerDataDimension(AbstractNode):
    """Dimension binding for data in a layer."""

    BINDING_PROPERTIES = (
        ('Data Source', 'source', 'link', editor_type.DataSource),
        ('Axis', 'axis', 'ruler', editor_type.ImmutableList)
    )

    def __init__(self, dimension, data, parent):
        self.dimension = dimension
        super(GraphLayerDataDimension, self).__init__(data, parent)
        dim = (SvgIcon.x_axis, SvgIcon.y_axis, SvgIcon.z_axis)
        self.icon = (dim[dimension] if dimension < 3 else SvgIcon.n_axis)

    def init(self):
        for label, property_, icon, editor in self.BINDING_PROPERTIES:
            if property_ == 'source':
                options = lambda: [x['id'] for x in self.root_data()['data']]
            elif property_ == 'axis':
                options = lambda: [x['id'] for x in
                                   self.parent.parent.parent.parent.data[
                                       'dimensions'][self.dimension]]
            else:
                options = lambda: None
            prop = Property({'label': label,
                             'property': property_,
                             'icon': icon,
                             'editor': editor(options),
                             'data': self.data},
                            parent=self)
            if property_ == 'source':
                prop.tags.add(NodeTags.data_reference)
                # Update is done by backend.
            elif property_ == 'axis':
                # Default to first available axis for this dimension
                options_list = options()
                if options_list and not prop.get():
                    prop.editor.set(options_list[0])

                prop.tags.add(NodeTags.data_reference)
                prop.editor.tags.add(
                    editor_type.EditorTags.force_rebuild_after_edit)
            self.properties.append(prop)

    @property
    def label(self):
        return 'Dimension {}'.format(
            self.dimension + 1)

class Label(AbstractLeaf):
    icon = SvgIcon.label

    @property
    def label(self):
        return 'Label ({})'.format(self.data['id'])

VIEW_TO_CLASS = {
    'layout': Layout,
    'pages': Pages,
    'page': Page,
    'textbox': TextBox,
    'image': Image,
    'graph': Graph
}


#!/usr/bin/python
# Copyright (c) 2017 Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

ANSIBLE_METADATA = {'metadata_version': '1.1',
                    'status': ['preview'],
                    'supported_by': 'community'}

DOCUMENTATION = '''
---
module: luks_device

short_description: Manage encrypted (LUKS) devices

version_added: "2.8"

description:
    - "Module manages L(LUKS,https://en.wikipedia.org/wiki/Linux_Unified_Key_Setup)
      on given device. Supports creating, destroying, opening and closing of
      LUKS container and adding or removing new keys."

options:
    device:
        description:
            - "Device to work with (e.g. C(/dev/sda1)). Needed in most cases.
              Can be omitted only when I(state=closed) together with I(name)
              is provided."
        type: str
    state:
        description:
            - "Desired state of the LUKS container. Based on its value creates,
              destroys, opens or closes the LUKS container on a given device."
            - "I(present) will create LUKS container unless already present.
              Requires I(device) and I(keyfile) options to be provided."
            - "I(absent) will remove existing LUKS container if it exists.
              Requires I(device) or I(name) to be specified."
            - "I(opened) will unlock the LUKS container. If it does not exist
              it will be created first.
              Requires I(device) and I(keyfile) to be specified. Use
              the I(name) option to set the name of the opened container.
              Otherwise the name will be generated automatically and returned
              as a part of the result."
            - "I(closed) will lock the LUKS container. However if the container
              does not exist it will be created.
              Requires I(device) and I(keyfile) options to be provided. If
              container does already exist I(device) or I(name) will suffice."
        type: str
        default: present
        choices: [present, absent, opened, closed]
    name:
        description:
            - "Sets container name when I(state=opened). Can be used
              instead of I(device) when closing the existing container
              (i.e. when I(state=closed))."
        type: str
    keyfile:
        description:
            - "Used to unlock the container and needed for most
              of the operations. Parameter value is the path
              to the keyfile with the passphrase."
            - "BEWARE that working with keyfiles in plaintext is dangerous.
              Make sure that they are protected."
        type: path
    new_keyfile:
        description:
            - "Adds additional key to given container on I(device).
              Needs I(keyfile) option for authorization. LUKS container
              supports up to 8 keys. Parameter value is the path
              to the keyfile with the passphrase."
            - "NOTE that adding additional keys is I(not idempotent).
              A new keyslot will be used even if another keyslot already
              exists for this keyfile."
            - "BEWARE that working with keyfiles in plaintext is dangerous.
              Make sure that they are protected."
        type: path
    remove_keyfile:
        description:
            - "Removes given key from the container on I(device). Does not
              remove the keyfile from filesystem.
              Parameter value is the path to the keyfile with the passphrase."
            - "NOTE that removing keys is I(not idempotent). Trying to remove
              a key which no longer exists results in an error."
            - "NOTE that to remove the last key from a LUKS container, the
              I(force_remove_last_key) option must be set to C(yes)."
            - "BEWARE that working with keyfiles in plaintext is dangerous.
              Make sure that they are protected."
        type: path
    force_remove_last_key:
        description:
            - "If set to C(yes), allows removing the last key from a container."
            - "BEWARE that when the last key has been removed from a container,
              the container can no longer be opened!"
        type: bool
        default: no

requirements:
    - "cryptsetup"
    - "wipefs"
    - "lsblk"

author:
    "Jan Pokorny (@japokorn)"
'''

EXAMPLES = '''

- name: create LUKS container (remains unchanged if it already exists)
  luks_device:
    device: "/dev/loop0"
    state: "present"
    keyfile: "/vault/keyfile"

- name: (create and) open the LUKS container; name it "mycrypt"
  luks_device:
    device: "/dev/loop0"
    state: "opened"
    name: "mycrypt"
    keyfile: "/vault/keyfile"

- name: close the existing LUKS container "mycrypt"
  luks_device:
    state: "closed"
    name: "mycrypt"

- name: make sure LUKS container exists and is closed
  luks_device:
    device: "/dev/loop0"
    state: "closed"
    keyfile: "/vault/keyfile"

- name: create container if it does not exist and add new key to it
  luks_device:
    device: "/dev/loop0"
    state: "present"
    keyfile: "/vault/keyfile"
    new_keyfile: "/vault/keyfile2"

- name: add new key to the LUKS container (container has to exist)
  luks_device:
    device: "/dev/loop0"
    keyfile: "/vault/keyfile"
    new_keyfile: "/vault/keyfile2"

- name: remove existing key from the LUKS container
  luks_device:
    device: "/dev/loop0"
    remove_keyfile: "/vault/keyfile2"

- name: completely remove the LUKS container and its contents
  luks_device:
    device: "/dev/loop0"
    state: "absent"
'''

RETURN = '''
name:
    description:
        When I(state=opened) returns (generated or given) name
        of LUKS container. Returns None if no name is supplied.
    returned: success
    type: str
    sample: "luks-c1da9a58-2fde-4256-9d9f-6ab008b4dd1b"
'''

import os
import re
import stat

from ansible.module_utils.basic import AnsibleModule

RETURN_CODE = 0
STDOUT = 1
STDERR = 2

# used to get <luks-name> out of lsblk output in format 'crypt <luks-name>'
# regex takes care of any possible blank characters
LUKS_NAME_REGEX = re.compile(r'\s*crypt\s+([^\s]*)\s*')
# used to get </luks/device> out of lsblk output
# in format 'device: </luks/device>'
LUKS_DEVICE_REGEX = re.compile(r'\s*device:\s+([^\s]*)\s*')

class Handler(object):

    def __init__(self, module):
        self._module = module
        self._lsblk_bin = self._module.get_bin_path('lsblk', True)

    def _run_command(self, command):
        return self._module.run_command(command)

    def generate_luks_name(self, device):
        ''' Generate name for luks based on device UUID ('luks-<UUID>').
            Raises ValueError when obtaining of UUID fails.
        '''
        result = self._run_command([self._lsblk_bin, '-n', device, '-o', 'UUID'])

        if result[RETURN_CODE] != 0:
            raise ValueError('Error while generating LUKS name for %s: %s'
                             % (device, result[STDERR]))
        dev_uuid = result[STDOUT].strip()
        return 'luks-%s' % dev_uuid

class CryptHandler(Handler):

    def __init__(self, module):
        super(CryptHandler, self).__init__(module)
        self._cryptsetup_bin = self._module.get_bin_path('cryptsetup', True)

    def get_container_name_by_device(self, device):
        ''' obtain LUKS container name based on the device where it is located
            return None if not found
            raise ValueError if lsblk command fails
        '''
        result = self._run_command([self._lsblk_bin, device, '-nlo', 'type,name'])
        if result[RETURN_CODE] != 0:
            raise ValueError('Error while obtaining LUKS name for %s: %s'
                             % (device, result[STDERR]))

        m = LUKS_NAME_REGEX.search(result[STDOUT])

        try:
            name = m.group(1)
        except AttributeError:
            name = None
        return name

    def get_container_device_by_name(self, name):
        ''' obtain device name based on the LUKS container name
            return None if not found
            raise ValueError if lsblk command fails
        '''
        # apparently each device can have only one LUKS container on it
        result = self._run_command([self._cryptsetup_bin, 'status', name])
        if result[RETURN_CODE] != 0:
            return None

        m = LUKS_DEVICE_REGEX.search(result[STDOUT])
        device = m.group(1)
        return device

    def is_luks(self, device):
        ''' check if the LUKS container does exist
        '''
        result = self._run_command([self._cryptsetup_bin, 'isLuks', device])
        return result[RETURN_CODE] == 0

    def run_luks_create(self, device, keyfile):
        # create a new luks container; use batch mode to auto confirm
        result = self._run_command([self._cryptsetup_bin, 'luksFormat',
                                    '-q', device, keyfile])
        if result[RETURN_CODE] != 0:
            raise ValueError('Error while creating LUKS on %s: %s'
                             % (device, result[STDERR]))

    def run_luks_open(self, device, keyfile, name):
        result = self._run_command([self._cryptsetup_bin, '--key-file', keyfile,
                                    'open', '--type', 'luks', device, name])
        if result[RETURN_CODE] != 0:
            raise ValueError('Error while opening LUKS container on %s: %s'
                             % (device, result[STDERR]))

    def run_luks_close(self, name):
        result = self._run_command([self._cryptsetup_bin, 'close', name])
        if result[RETURN_CODE] != 0:
            raise ValueError('Error while closing LUKS container %s' % (name))

    def run_luks_remove(self, device):
        wipefs_bin = self._module.get_bin_path('wipefs', True)

        name = self.get_container_name_by_device(device)
        if name is not None:
            self.run_luks_close(name)
        result = self._run_command([wipefs_bin, '--all', device])
        if result[RETURN_CODE] != 0:
            raise ValueError('Error while wiping luks container %s: %s'
                             % (device, result[STDERR]))

    def run_luks_add_key(self, device, keyfile, new_keyfile):
        ''' Add new key to given 'device'; authentication done using 'keyfile'
            Raises ValueError when command fails
        '''
        result = self._run_command([self._cryptsetup_bin, 'luksAddKey', device,
                                    new_keyfile, '--key-file', keyfile])
        if result[RETURN_CODE] != 0:
            raise ValueError('Error while adding new LUKS key to %s: %s'
                             % (device, result[STDERR]))

    def run_luks_remove_key(self, device, keyfile, force_remove_last_key=False):
        ''' Remove key from given device
            Raises ValueError when command fails
        '''
        if not force_remove_last_key:
            result = self._run_command([self._cryptsetup_bin, 'luksDump', device])
            if result[RETURN_CODE] != 0:
                raise ValueError('Error while dumping LUKS header from %s'
                                 % (device, ))
            keyslot_count = 0
            keyslot_area = False
            keyslot_re = re.compile(r'^Key Slot [0-9]+: ENABLED')
            for line in result[STDOUT].splitlines():
                if line.startswith('Keyslots:'):
                    keyslot_area = True
                elif line.startswith('  '):
                    # LUKS2 header dumps use human-readable indented output.
                    # Thus we have to look out for 'Keyslots:' and count the
                    # number of indented keyslot numbers.
                    if keyslot_area and line[2] in '0123456789':
                        keyslot_count += 1
                elif line.startswith('\t'):
                    pass
                elif keyslot_re.match(line):
                    # LUKS1 header dumps have one line per keyslot with ENABLED
                    # or DISABLED in them. We count such lines with ENABLED.
                    keyslot_count += 1
                else:
                    keyslot_area = False
            if keyslot_count < 2:
                self._module.fail_json(msg="LUKS device %s has less than two active keyslots. "
                                           "To be able to remove a key, please set "
                                           "`force_remove_last_key` to `yes`." % device)

        result = self._run_command([self._cryptsetup_bin, 'luksRemoveKey', device,
                                    '-q', '--key-file', keyfile])
        if result[RETURN_CODE] != 0:
            raise ValueError('Error while removing LUKS key from %s: %s'
                             % (device, result[STDERR]))

class ConditionsHandler(Handler):

    def __init__(self, module, crypthandler):
        super(ConditionsHandler, self).__init__(module)
        self._crypthandler = crypthandler

    def luks_create(self):
        return (self._module.params['device'] is not None and
                self._module.params['keyfile'] is not None and
                self._module.params['state'] in ('present',
                                                 'opened',
                                                 'closed') and
                not self._crypthandler.is_luks(self._module.params['device']))

    def opened_luks_name(self):
        ''' If luks is already opened, return its name.
            If 'name' parameter is specified and differs
            from obtained value, fail.
            Return None otherwise
        '''
        if self._module.params['state'] != 'opened':
            return None

        # try to obtain luks name - it may be already opened
        name = self._crypthandler.get_container_name_by_device(
            self._module.params['device'])

        if name is None:
            # container is not open
            return None

        if self._module.params['name'] is None:
            # container is already opened
            return name

        if name != self._module.params['name']:
            # the container is already open but with different name:
            # suspicious. back off
            self._module.fail_json(msg="LUKS container is already opened "
                                   "under different name '%s'." % name)

        # container is opened and the names match
        return name

    def luks_open(self):
        if (self._module.params['device'] is None or
                self._module.params['keyfile'] is None or
                self._module.params['state'] != 'opened'):
            # conditions for open not fulfilled
            return False

        name = self.opened_luks_name()

        if name is None:
            return True
        return False

    def luks_close(self):
        if ((self._module.params['name'] is None and
                self._module.params['device'] is None) or
                self._module.params['state'] != 'closed'):
            # conditions for close not fulfilled
            return False

        if self._module.params['device'] is not None:
            name = self._crypthandler.get_container_name_by_device(
                self._module.params['device'])
            # successfully getting name based on device means that luks is open
            luks_is_open = name is not None

        if self._module.params['name'] is not None:
            device = self._crypthandler.get_container_device_by_name(
                self._module.params['name'])
            # successfully getting device based on name means that luks is open
            luks_is_open = device is not None

        return luks_is_open

    def luks_add_key(self):
        if (self._module.params['device'] is None or
                self._module.params['keyfile'] is None or
                self._module.params['new_keyfile'] is None):
            # conditions for adding a key not fulfilled
            return False

        if self._module.params['state'] == 'absent':
            self._module.fail_json(msg="Contradiction in setup: Asking to "
                                   "add a key to absent LUKS.")

        return True

    def luks_remove_key(self):
        if (self._module.params['device'] is None or
                self._module.params['remove_keyfile'] is None):
            # conditions for removing a key not fulfilled
            return False

        if self._module.params['state'] == 'absent':
            self._module.fail_json(msg="Contradiction in setup: Asking to "
                                   "remove a key from absent LUKS.")

        return True

    def luks_remove(self):
        return (self._module.params['device'] is not None and
                self._module.params['state'] == 'absent' and
                self._crypthandler.is_luks(self._module.params['device']))

def run_module():
    # available arguments/parameters that a user can pass
    module_args = dict(
        state=dict(type='str', default='present', choices=['present', 'absent', 'opened', 'closed']),
        device=dict(type='str'),
        name=dict(type='str'),
        keyfile=dict(type='path'),
        new_keyfile=dict(type='path'),
        remove_keyfile=dict(type='path'),
        force_remove_last_key=dict(type='bool', default=False),
    )

    # seed the result dict in the object
    result = dict(
        changed=False,
        name=None
    )

    module = AnsibleModule(argument_spec=module_args,
                           supports_check_mode=True)

    if module.params['device'] is not None:
        try:
            statinfo = os.stat(module.params['device'])
            mode = statinfo.st_mode
            if not stat.S_ISBLK(mode) and not stat.S_ISCHR(mode):
                raise Exception('{0} is not a device'.format(module.params['device']))
        except Exception as e:
            module.fail_json(msg=str(e))

    crypt = CryptHandler(module)
    conditions = ConditionsHandler(module, crypt)

    # The conditions are in order to allow more operations in one run.
    # (e.g. create luks and add a key to it)

    # luks create
    if conditions.luks_create():
        if not module.check_mode:
            try:
                crypt.run_luks_create(module.params['device'],
                                      module.params['keyfile'])
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        result['changed'] = True
        if module.check_mode:
            module.exit_json(**result)

    # luks open

    name = conditions.opened_luks_name()
    if name is not None:
        result['name'] = name

    if conditions.luks_open():
        name = module.params['name']
        if name is None:
            try:
                name = crypt.generate_luks_name(module.params['device'])
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        if not module.check_mode:
            try:
                crypt.run_luks_open(module.params['device'],
                                    module.params['keyfile'],
                                    name)
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        result['name'] = name
        result['changed'] = True
        if module.check_mode:
            module.exit_json(**result)

    # luks close
    if conditions.luks_close():
        if module.params['device'] is not None:
            try:
                name = crypt.get_container_name_by_device(
                    module.params['device'])
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        else:
            name = module.params['name']
        if not module.check_mode:
            try:
                crypt.run_luks_close(name)
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        result['changed'] = True
        if module.check_mode:
            module.exit_json(**result)

    # luks add key
    if conditions.luks_add_key():
        if not module.check_mode:
            try:
                crypt.run_luks_add_key(module.params['device'],
                                       module.params['keyfile'],
                                       module.params['new_keyfile'])
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        result['changed'] = True
        if module.check_mode:
            module.exit_json(**result)

    # luks remove key
    if conditions.luks_remove_key():
        if not module.check_mode:
            try:
                crypt.run_luks_remove_key(module.params['device'],
                                          module.params['remove_keyfile'],
                                          force_remove_last_key=module.params['force_remove_last_key'])
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        result['changed'] = True
        if module.check_mode:
            module.exit_json(**result)

    # luks remove
    if conditions.luks_remove():
        if not module.check_mode:
            try:
                crypt.run_luks_remove(module.params['device'])
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        result['changed'] = True
        if module.check_mode:
            module.exit_json(**result)

    # Success - return result
    module.exit_json(**result)

def main():
    run_module()

if __name__ == '__main__':
    main()


from __future__ import absolute_import, division, print_function, unicode_literals

import threading

from echomesh.base import Config
from echomesh.util import Log
from echomesh.util import Importer
from echomesh.util.thread.Runnable import Runnable
from echomesh.util.thread import Lock

LOGGER = Log.logger(__name__)

MAIN_THREAD = threading.current_thread()

_TYPE_MAP = {
  'client': 'echomesh.light.ExternalLightBank',
  'spi': 'echomesh.light.SpiLightBank'
  }

class LightSingleton(Runnable):
  def __init__(self):
    super(LightSingleton, self).__init__()
    self.lights = None
    self.lock = Lock.Lock()
    self.owner_count = 0
    self.start()

  def _on_pause(self):
    if self.lights:
      self.lights.pause()
      self.lights = None

  def add_owner(self):
    self.owner_count += 1
    self._make_lights()

  def remove_owner(self):
    self.owner_count -= 1
    self._stop_lights()

  def add_client(self, client):
    self.lights.add_client(client)
    self._make_lights()

  def remove_client(self, client):
    with self.lock:
      if self.lights:
        self.lights.remove_client(client)
      self._stop_lights()

  def _make_lights(self):
    with self.lock:
      if self.lights:
        return
      classpath = _TYPE_MAP[Config.get('light', 'visualizer', 'type')]
      self.lights = Importer.imp(classpath, defer_failure=False)()
    self.lights.start()

  def _stop_lights(self):
    with self.lock:
      if not self.lights or self.lights.has_clients() or self.owner_count > 0:
        return
      lights, self.lights = self.lights, None
    lights.pause()

_SINGLETON = LightSingleton()
add_client = _SINGLETON.add_client
remove_client = _SINGLETON.remove_client
add_owner = _SINGLETON.add_owner
remove_owner = _SINGLETON.remove_owner
stop = _SINGLETON.pause


import sys

class Bank(object):
    def __init__(self):
        self.regs = {}

    def inc(self, reg, value):
        self.regs[reg] = self.regs.get(reg, 0) + value

    def dec(self, reg, value):
        self.regs[reg] = self.regs.get(reg, 0) - value

    def check(self, reg, op, value):
        reg_value = self.regs.get(reg, 0)
        if op == "==":
            return reg_value == value
        elif op == "!=":
            return reg_value != value
        elif op == ">=":
            return reg_value >= value
        elif op == "<=":
            return reg_value <= value
        elif op == ">":
            return reg_value > value
        elif op == "<":
            return reg_value < value
        else:
            raise ValueError("The operation '%s' is not supported" % op)

def main(lines):
    b = Bank()
    ins = extract_instructions(lines)
    for i in ins:
        cond = i[1]
        if b.check(cond[0], cond[1], cond[2]):
            op = i[0]
            if op[1] == 'inc':
                b.inc(op[0], op[2])
            elif op[1] == 'dec':
                b.dec(op[0], op[2])
            else:
                raise ValueError("The registers are limited to 'inc' and 'dec'. '%s' is not supported" % op[1])
    print(max([value for _, value in b.regs.items()]))

def extract_instructions(lines):
    l = []
    for line in lines:
        inc_dec, cond = line.strip().split(' if ')
        regadd, addi, valadd = inc_dec.strip().split(' ')
        regcmp, opcmp, valcmp = cond.strip().split(' ')
        l.append([[regadd, addi, int(valadd)],[regcmp, opcmp, int(valcmp)]])
    return l

if __name__ == "__main__":
    main(open(sys.argv[1], 'r').readlines())


from users.models import EmailVerifyCode
from random import choice
from django.core.mail import send_mail
from GradDesign.settings import EMAIL_FROM

# 获取随机验证码函数
def get_random_code(code_length):
    # 码源
    code_source = '1234567890qewrtyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM'
    code = ''
    for i in range(code_length):
        # 随机选择一个字符

        code += choice(code_source)
        # code += code_source[randint]
    return code

def send_email_code(email, send_type):
    # 1.创建邮箱验证码对象，保存数据，用来做对比
    code = get_random_code(6)
    a = EmailVerifyCode()
    a.email = email
    a.send_type = send_type
    # 验证码
    a.code = code
    a.save()

    # 2.正式的发送邮件功能
    send_title = ''
    send_body = ''
    if send_type == 1:
        send_title = "数字图书馆注册邮箱激活："
        send_body = "请点击以下链接激活您的账号: \n http://127.0.0.1:8000/users/user_active/"+code
        send_mail(send_title, send_body, EMAIL_FROM, [email])
    elif send_type == 2:
        send_title = "数字图书馆重置密码确认:"
        send_body = "请点击以下链接进行密码重置: \n http://127.0.0.1:8000/users/user_reset/"+code
        send_mail(send_title, send_body, EMAIL_FROM, [email])


import sys
sys.stdin = open("input.txt","r")

pay = int(input())
ans = 0
rest = 1000 - pay

coins = [500, 100, 50, 10, 5, 1]

for coin in coins:
    if rest >= coin:
        ans += rest // coin
        rest %= coin

print(ans)


# -*- coding=utf-8 -*-
"""
#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
from __future__ import print_function
import os
import time
import numpy as np
import logging
import argparse

import paddle
import paddle.fluid as fluid
from paddle.fluid import profiler

from args import print_arguments, parse_args
from infer_network import TdmInferNet
from dataset_generator import TDMDataset

logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger("fluid")
logger.setLevel(logging.INFO)

def to_tensor(data, place):
    """
    Convert data to paddle tensor
    """
    flattened_data = np.concatenate(data, axis=0).astype("float32")
    flattened_data = flattened_data.reshape([-1, 768])
    res = fluid.Tensor()
    res.set(flattened_data, place)
    return res

def data2tensor(data, place):
    """
    Dataset prepare
    """
    input_emb = to_tensor([x[0] for x in data], place)

    return input_emb

def run_infer(args, model_path):
    """run infer"""
    logger.info("Infer Begin")
    file_list = [
        str(args.test_files_path) + "/%s" % x
        for x in os.listdir(args.test_files_path)
    ]

    tdm_model = TdmInferNet(args)
    inputs = tdm_model.input_data()
    res_item = tdm_model.infer_net(inputs)
    test_reader = TDMDataset().infer_reader(file_list, args.batch_size)

    place = fluid.CPUPlace()
    exe = fluid.Executor(place)

    path = os.path.join(args.model_files_path, model_path)
    fluid.io.load_persistables(
        executor=exe,
        dirname=path,
        main_program=fluid.default_main_program())

    logger.info("Load persistables from \"{}\"".format(path))

    if args.save_init_model:
        logger.info("Begin Save infer model.")
        model_path = (str(args.model_files_path) + "/" + "infer_model")
        fluid.io.save_inference_model(executor=exe, dirname=model_path,
                                      feeded_var_names=[
                                          'input_emb', 'first_layer_node', 'first_layer_node_mask'],
                                      target_vars=[res_item])
        logger.info("End Save infer model.")

    first_layer_node = tdm_model.first_layer_node
    first_layer_nums = len(first_layer_node)
    first_layer_node = np.array(first_layer_node)
    first_layer_node = first_layer_node.reshape((1, -1)).astype('int64')
    first_layer_node = first_layer_node.repeat(args.batch_size, axis=0)
    # 在demo中，假设infer起始层的节点都不是叶子节点，mask=0
    # 若真实的起始层含有叶子节点，则对应位置的 mask=1
    first_layer_mask = (
        np.zeros((args.batch_size, first_layer_nums))).astype('int64')

    for batch_id, data in enumerate(test_reader()):
        input_emb = data2tensor(data, place)
        item_res = exe.run(fluid.default_main_program(),
                           feed={"input_emb": input_emb,
                                 "first_layer_node": first_layer_node,
                                 "first_layer_node_mask": first_layer_mask},
                           fetch_list=[res_item])
        logger.info("TEST --> batch: {} infer_item {}".format(
            batch_id, item_res))
    logger.info("Inference complete!")

if __name__ == "__main__":
    args = parse_args()
    print_arguments(args)
    # 在此处指定infer模型所在的文件夹
    path = "epoch_0"
    run_infer(args, path)


from django.db.models import query
from django.shortcuts import render
from rest_framework.generics import GenericAPIView
from rest_framework.mixins import(ListModelMixin,CreateModelMixin,
                                  UpdateModelMixin,DestroyModelMixin,RetrieveModelMixin)
from api.models import Student
from api.serializers import StudentModelSerializer

from rest_framework.generics import (ListAPIView,CreateAPIView,
                                     UpdateAPIView,RetrieveAPIView,DestroyAPIView,
                                     RetrieveDestroyAPIView, RetrieveUpdateAPIView)

from rest_framework.throttling import ScopedRateThrottle

# Create your views here.
class StudentListMixin(GenericAPIView,ListModelMixin):
    queryset = Student.objects.all()
    # print("yesss")
    # breakpoint()
    serializer_class = StudentModelSerializer

    def get(self,request,*args,**kwargs):
        return self.list(request,*args,**kwargs)

class StudentCreateMixin(GenericAPIView,CreateModelMixin):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer

    def post(self,request,*args,**kwargs):
        return self.create(request,*args,**kwargs)

class StudentRetriveMixin(GenericAPIView,RetrieveModelMixin):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer

    def get(self,request,pk,*args,**kwargs):
        return self.retrieve(request,pk,*args,**kwargs)

class StudentUpdateMixin(GenericAPIView,UpdateModelMixin):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer

    def post(self,request,pk,*args,**kwargs):
        return self.update(request,pk,*args,**kwargs)

class StudentDestroyMixin(GenericAPIView,DestroyModelMixin):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer

    def delete(self,request,pk,*args,**kwargs):
        return self.destroy(request,pk,*args,**kwargs)

class StudentListAPIView(ListAPIView):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer
    throttle_classes = [ScopedRateThrottle]
    throttle_scope = 'viewstu'

class StudentCreateAPIView(CreateAPIView):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer
    throttle_classes = [ScopedRateThrottle]

class StudentUpdateAPIView(UpdateAPIView):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer
    throttle_classes = [ScopedRateThrottle]
    throttle_scope ='modifystu'

class StudentRetriveAPIView(RetrieveAPIView):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer
    throttle_classes = [ScopedRateThrottle]

class StudentDestoryAPIView(DestroyAPIView):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer

class StudentRetriveDestoryAPIView(RetrieveDestroyAPIView):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer
    throttle_classes = [ScopedRateThrottle]

class StudentRetriveUpdateAPIView(RetrieveUpdateAPIView):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer
    throttle_classes = [ScopedRateThrottle]


import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from os import listdir
from os.path import isfile, join, isdir
import argparse
#Get files from exp directory
templates = [
            '(ROOT(S(NP)(VP)(.)))' ,

            '(ROOT(S(VP)(.)))' ,

            '(ROOT(NP(NP)(.)))' ,

            '(ROOT(FRAG(SBAR)(.)))' ,

            '(ROOT(S(S)(,)(CC)(S)(.)))' ,

            '(ROOT(SBARQ(WHADVP)(SQ)(.)))' ,

            '(ROOT(S(PP)(,)(NP)(VP)(.)))' ,

            '(ROOT(S(ADVP)(NP)(VP)(.)))' ,

            '(ROOT(S(SBAR)(,)(NP)(VP)(.)))'
        ]
def get_values(df):
    df1=df.query("Inc==1")
    #df2=df.query("Inc==0")
    index=df1.index

    print(index)
    X=[]
    Y=[]
    #s= df1["ID"].sum()
    for i in range(len(index)):
        x1=index[i][0]
        x2=index[i][1]
        df2=df.query("Dif=="+str(x1))
        s = df2["ID"].sum ()
        print(s)
        y=(df1.loc[x1,x2]["ID"])

        X.append(x1)
        Y.append(y/s)
    return X,Y

def inc_pars(args):
    mypath = args.log_path
    out_path=args.out_path
    dirs = [dI for dI in listdir (mypath) if isdir (join (mypath , dI))]
    datasets = ["AMAZON_PRODUCTS" , "NEWS_HEADLINES" , "STANFORD_TREEBANK"]
    tools = ["CHAR_CNN" , "CNN_TEXT" , "SENTICNET" , "SENTIWORDNET" , "STANFORD" , "VADER"]
    fig = plt.figure ()
    # sns.set (style="whitegrid")
    sns.set (font_scale=0.6)

    # sns.palplot (sns.hls_palette (8 , l=.3 , s=.8))
    numfile = 0
    clusters_tab=[]
    for dir in dirs:
        path_dir = join (mypath , dir)
        onlyfiles = [f for f in listdir (path_dir) if isfile (join (path_dir , f))]
        numfile = numfile + 1
        tool_cluster=pd.DataFrame();
        i=0;
        for f in onlyfiles:
            print(f)

            path_file=join(path_dir,f)
            df= pd.read_csv(path_file,sep=";")
            #df=df.loc[df["Dif"]<20]
            #df["Dif"]= df["Dif"].apply(lambda x: round(x,2))
            #clusters=df.groupby(["Parse"]).agg("count")
            clusters = df.groupby (["Parse"], as_index=False)["Inc"].mean()
            clusters.columns=["Parse","Inc"]
            tool_cluster = clusters
            """
            if i==0:
                tool_cluster= clusters
            else:
                tool_cluster=pd.merge (tool_cluster , clusters,how="inner",on="Parse")

                #tool_cluster=pd.merge(tool_cluster,clusters,how="inner", on=["Parse"])
                #tool_cluster=tool_cluster.dropna (inplace=True)

            i=i+1
        tool_cluster.index.names=["Parse"]

        """

        df_new= pd.DataFrame(columns=tool_cluster.columns)
        j=0;
        for i in range(len(tool_cluster)):

            if tool_cluster.iloc[i]["Parse"] in templates:
                df_new.loc[j]=tool_cluster.iloc[i]
                j=j+1

        df_new.to_csv(join(out_path,f),sep=";")

        """
        X,Y=get_values (clusters)
        #sns.scatterplot (X , Y ,ax=ax)            #clusters= list(clusters)
        #y1=df["Inc"].apply(lambda x: round(x,2))
        ax = fig.add_subplot (1, 6 , numfile)
        #sns.lineplot (X,Y , ax=ax)
        sns.scatterplot (X , Y , ax=ax)
        j=j+1
        #print(X)
        #print(y1)
        """
        #print(clusters.loc[clusters["Id"]>10])

        """
        for cluster in clusters:

          print(cluster[1])
        """
        """
        ax.set_ylabel ("nb_inconsistencies" , fontsize=10)
        ax.set_xlabel ("diff in words" , fontsize=10)
        ax.set_title (dir , fontsize=10)
        """

    #plt.figlegend ( datasets , loc='lower center' , ncol=3 , labelspacing=0.)
    #plt.show()
    #fig.savefig (args.out_path)

if __name__ == '__main__':
    ##model args
    parser = argparse.ArgumentParser (description='Syntactically Controlled Paraphrase Transformer')

    parser.add_argument ('--log_path' , type=str , default="D:/Users/wissam/Documents/These/these/papers_material/VLDB_submission/experiments/logs/Intool_inc_sim/sim_inc" ,
                         help='input path of logs')

    parser.add_argument ('--out_path' , type=str , default='./data/sentiment_dataset_sentic.csv' ,
                         help='output of plots')

    args = parser.parse_args ()
    inc_pars(args)


from fairseq.models import FairseqEncoder

import torch
import torch.nn as nn
import math

from fairseq import options, utils

from fairseq.models.fairseq_encoder import EncoderOut
from fairseq.modules import (
    AdaptiveSoftmax,
    FairseqDropout,
    LayerDropModuleList,
    LayerNorm,
    PositionalEmbedding,
    SinusoidalPositionalEmbedding,
    TransformerEncoderLayer,
)

from fairseq.modules.quant_noise import quant_noise as apply_quant_noise_
from torch import Tensor

class SrcFactorEncoder(FairseqEncoder):
    """
    Based on TransformerEncoder but adds an additional
    embedding for src_factor.
    Changes have comments
    """
    def __init__(self, args, dictionary, embed_tokens, src_factor_tokens):
        super().__init__(dictionary)

        self.register_buffer("version", torch.Tensor([3]))

        self.dropout_module = FairseqDropout(args.dropout, module_name=self.__class__.__name__)
        self.encoder_layerdrop = args.encoder_layerdrop
        """
        change the embedding dim to account for src_factor
        """
        embed_dim = embed_tokens.embedding_dim + src_factor_tokens.embedding_dim

        self.padding_idx = embed_tokens.padding_idx
        self.max_source_positions = args.max_source_positions

        self.embed_tokens = embed_tokens
        """
        save embedding
        """
        self.src_factor_tokens = src_factor_tokens

        self.embed_scale = 1.0 if args.no_scale_embedding else math.sqrt(embed_dim)

        self.embed_positions = (
            PositionalEmbedding(
                args.max_source_positions,
                embed_dim,
                self.padding_idx,
                learned=args.encoder_learned_pos,
            )
            if not args.no_token_positional_embeddings
            else None
        )

        if not args.adaptive_input and args.quant_noise_pq > 0:
            self.quant_noise = apply_quant_noise_(
                nn.Linear(embed_dim, embed_dim, bias=False),
                args.quant_noise_pq,
                args.quant_noise_pq_block_size,
            )
        else:
            self.quant_noise = None

        if self.encoder_layerdrop > 0.0:
            self.layers = LayerDropModuleList(p=self.encoder_layerdrop)
        else:
            self.layers = nn.ModuleList([])
        self.layers.extend(
            [self.build_encoder_layer(args) for i in range(args.encoder_layers)]
        )
        self.num_layers = len(self.layers)

        if args.encoder_normalize_before:
            self.layer_norm = LayerNorm(embed_dim)
        else:
            self.layer_norm = None
        if getattr(args, "layernorm_embedding", False):
            self.layernorm_embedding = LayerNorm(embed_dim)
        else:
            self.layernorm_embedding = None

    def forward_embedding(self, src_tokens, src_factor_tokens):
        """
        concat before passing to the rest
        """
        x = self.embed_tokens(src_tokens)
        x = torch.cat([self.src_factor_tokens(src_factor_tokens),x],-1)

        x = embed = self.embed_scale * x
        if self.embed_positions is not None:
            x = embed + self.embed_positions(src_tokens)
        if self.layernorm_embedding is not None:
            x = self.layernorm_embedding(x)
        x = self.dropout_module(x)
        if self.quant_noise is not None:
            x = self.quant_noise(x)
        return x, embed

    def build_encoder_layer(self, args):
        return TransformerEncoderLayer(args)

    def forward(self, src_tokens, src_lengths, src_factor_tokens, src_factor_lengths, return_all_hiddens: bool = False):
        """
        forward embedding signature
        """
        x, encoder_embedding = self.forward_embedding(src_tokens, src_factor_tokens)

        # B x T x C -> T x B x C
        x = x.transpose(0, 1)
        # compute padding mask
        encoder_padding_mask = src_tokens.eq(self.padding_idx)

        encoder_states = [] if return_all_hiddens else None

        # encoder layers
        for layer in self.layers:
            x = layer(x, encoder_padding_mask)
            if return_all_hiddens:
                assert encoder_states is not None
                encoder_states.append(x)

        if self.layer_norm is not None:
            x = self.layer_norm(x)

        return EncoderOut(
            encoder_out=x,  # T x B x C
            encoder_padding_mask=encoder_padding_mask,  # B x T
            encoder_embedding=encoder_embedding,  # B x T x C
            encoder_states=encoder_states,  # List[T x B x C]
            src_tokens=None,
            src_lengths=None,
        )

    @torch.jit.export
    def reorder_encoder_out(self, encoder_out: EncoderOut, new_order):
        """
        Reorder encoder output according to *new_order*.

        Args:
            encoder_out: output from the ``forward()`` method
            new_order (LongTensor): desired order

        Returns:
            *encoder_out* rearranged according to *new_order*
        """
        """
        Since encoder_padding_mask and encoder_embedding are both of type
        Optional[Tensor] in EncoderOut, they need to be copied as local
        variables for Torchscript Optional refinement
        """
        encoder_padding_mask: Optional[Tensor] = encoder_out.encoder_padding_mask
        encoder_embedding: Optional[Tensor] = encoder_out.encoder_embedding

        new_encoder_out = (
            encoder_out.encoder_out
            if encoder_out.encoder_out is None
            else encoder_out.encoder_out.index_select(1, new_order)
        )
        new_encoder_padding_mask = (
            encoder_padding_mask
            if encoder_padding_mask is None
            else encoder_padding_mask.index_select(0, new_order)
        )
        new_encoder_embedding = (
            encoder_embedding
            if encoder_embedding is None
            else encoder_embedding.index_select(0, new_order)
        )
        src_tokens = encoder_out.src_tokens
        if src_tokens is not None:
            src_tokens = src_tokens.index_select(0, new_order)

        src_lengths = encoder_out.src_lengths
        if src_lengths is not None:
            src_lengths = src_lengths.index_select(0, new_order)

        encoder_states = encoder_out.encoder_states
        if encoder_states is not None:
            for idx, state in enumerate(encoder_states):
                encoder_states[idx] = state.index_select(1, new_order)

        return EncoderOut(
            encoder_out=new_encoder_out,  # T x B x C
            encoder_padding_mask=new_encoder_padding_mask,  # B x T
            encoder_embedding=new_encoder_embedding,  # B x T x C
            encoder_states=encoder_states,  # List[T x B x C]
            src_tokens=src_tokens,  # B x T
            src_lengths=src_lengths,  # B x 1
        )

    def max_positions(self):
        """Maximum input length supported by the encoder."""
        if self.embed_positions is None:
            return self.max_source_positions
        return min(self.max_source_positions, self.embed_positions.max_positions)

    def upgrade_state_dict_named(self, state_dict, name):
        """Upgrade a (possibly old) state dict for new versions of fairseq."""
        if isinstance(self.embed_positions, SinusoidalPositionalEmbedding):
            weights_key = "{}.embed_positions.weights".format(name)
            if weights_key in state_dict:
                print("deleting {0}".format(weights_key))
                del state_dict[weights_key]
            state_dict[
                "{}.embed_positions._float_tensor".format(name)
            ] = torch.FloatTensor(1)
        for i in range(self.num_layers):
            # update layer norms
            self.layers[i].upgrade_state_dict_named(
                state_dict, "{}.layers.{}".format(name, i)
            )

        version_key = "{}.version".format(name)
        if utils.item(state_dict.get(version_key, torch.Tensor([1]))[0]) < 2:
            # earlier checkpoints did not normalize after the stack of layers
            self.layer_norm = None
            self.normalize = False
            state_dict[version_key] = torch.Tensor([1])
        return state_dict


""" Compiled: 2020-09-18 10:38:52 """

#__src_file__ = "extensions/frtb/./etc/FRTBSAExport.py"
import FRTBCommon
import FRTBExport

ael_variables = FRTBExport.getDistributeCalculation()
exporters = FRTBCommon.getExporters(group_name='sa')
ael_variables.extend(FRTBExport.getRiskFactorAelVariables(
    risk_classes='enum(cl_FRTB_SA_Risk_Class)')
)

ael_variables.extend(FRTBExport.getPositionAelVariables())
ael_variables = FRTBExport.createAelVariables(
    ael_vars_list=ael_variables, exporters=exporters,
    log_filename='frtb_sa_export.log'
)

def ael_main(parameters):
    return FRTBCommon.aelMain(parameters, exporters)


#!/usr/bin/env python

# Copyright (C) 2010 Red Hat, Inc.
#
# This is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this software; if not, write to the Free
# Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA, or see the FSF site: http://www.fsf.org.

"""
A module containing the functions for loading the configuration
and preparing the environment for the tests.
"""
import os
import sys
import threading
import traceback
import time
import gc
import logging
import yaml
import collections
from jinja2 import Environment, FileSystemLoader

ART_CONFIG = {}
GE = {}

# garbage collector interval in seconds
GC_INTERVAL = 600

def set_cmd_line_params_in_dict(cmd_line):

    cmd_line_args = {}

    vars_to_be_treated_as_list = ['engines', 'storages']

    for param in cmd_line:
        try:
            key, value = param.split('=', 1)
        except ValueError:
            raise Exception("Expected '=' sign somewhere in '%s'." % param)
        section, var = key.split('.')
        if var in vars_to_be_treated_as_list:
            value = value.split(',')
        update_dict(cmd_line_args, {section: {var: value}})

    return cmd_line_args

def update_dict(master, update):
    for k, v in update.iteritems():
        if isinstance(v, collections.Mapping):
            r = update_dict(master.get(k, {}), v)
            master[k] = r
        else:
            master[k] = update[k]

    return master

def get_ge_yaml(cmd_line_params):
    ge_yaml = cmd_line_params.get('RUN').get('golden_environment')

    assert os.path.exists(ge_yaml)

    return ge_yaml

def generate_ge_description(ge_yaml):
    env = Environment(loader=FileSystemLoader('/'))

    runtime_yaml = 'runtime.yaml'
    template = env.get_template(ge_yaml)
    with open(ge_yaml, 'r') as f:
        context = yaml.load(f)

    rendered_yaml = template.render(context)

    with open(runtime_yaml, 'w') as f:
        f.write(rendered_yaml)

    with open(runtime_yaml, 'r') as f:
        return yaml.load(f)

    return None

def get_vds_n_passwords():

    vds_passwords = []
    vds = []

    for host in GE['hosts']:
        vds_passwords.append(host.get('password'))
        vds.append(host.get('address'))

    return vds, vds_passwords

def create_runtime_config(path_to_defaults, art_define_args):

    global ART_CONFIG
    global GE

    with open(path_to_defaults, 'r') as fh:
        defaults = yaml.load(fh)

    context = {}
    update_dict(context, defaults)

    cmd_line = set_cmd_line_params_in_dict(art_define_args)
    ge_yaml = get_ge_yaml(cmd_line)

    update_dict(context, cmd_line)

    ART_CONFIG.update(context)
    GE.update(generate_ge_description(ge_yaml))

    ART_CONFIG['DEFAULT']['PRODUCT'] = GE['product']
    ART_CONFIG['DEFAULT']['VERSION'] = GE['version']
    ART_CONFIG['REST_CONNECTION']['host'] = GE['engine_fqdn']
    ART_CONFIG['REST_CONNECTION']['uri'] = (
        ART_CONFIG['REST_CONNECTION']['uri'] % ART_CONFIG['REST_CONNECTION']
    )
    if not ART_CONFIG['REST_CONNECTION']['urisuffix']:
        ART_CONFIG['REST_CONNECTION']['uri'] = (
            ART_CONFIG['REST_CONNECTION']['uri'].replace('None', '')
        )

    vds, vds_paswords = get_vds_n_passwords()

    ART_CONFIG['PARAMETERS']['vds'] = vds
    ART_CONFIG['PARAMETERS']['vds_password'] = vds_paswords

    GE['mac_ranges'] = GE.get('mac_pools')[0].get(
        'mac_pool_ranges'
    )[0].replace(
        ',', '-'
    )

def dump_stacks(signal, frame):
    """
    In case of ART get stuck we can run kill sig command and get the
    stack traceback of each thread.
    like:
        kill -SIGUSR1 <ART PID>

    __author__ : khakimi
    :param signal: the signal number
    :type signal: int
    :param frame: the interrupted stack frame
    :type frame: frame object
    """
    id2name = dict((th.ident, th.name) for th in threading.enumerate())
    for threadId, stack in sys._current_frames().items():
        print("\nThread: {0}({1})".format(id2name[threadId], threadId))
        traceback.print_stack(f=stack)

def stuck_handler():
    """
    Check MainThread every 4 minutes if stuck.
    """
    mt = threading.current_thread().ident
    t = threading.Thread(target=stuck_check, args=(mt,))
    t.daemon = True
    t.start()

def stuck_check(main_thread):
    t = [None for i in range(5)]
    logger = logging.getLogger("stuck_handler")
    while True:
        time.sleep(240)
        t.pop(0)
        try:
            tmp = sys._current_frames()[main_thread]
        except Exception as ex:
            logger.warning(
                "sys._current_frames failed with exception: %s\n", ex
            )
            break
        t.append(traceback.format_stack(f=tmp))
        if not [x for x in t if t[0] != x]:
            logger.warn(
                "There is possiblity that MainThread is stucked. "
                "Check debug log to see traceback where it is stucked on."
            )
            logger.debug(''.join(t[-1]))

class MonitorGC(object):

    logger = logging.getLogger('art_monitor_gc')

    def __init__(self, interval=GC_INTERVAL):
        self.interval = interval
        self.thread = threading.Thread(target=self.run, name='monitor_gc')
        self.thread.daemon = True
        self.thread.start()

    def run(self):
        self.logger.info("monitor Garbage Collector started")
        saved_flags = gc.get_debug()
        gc.set_debug(0)
        try:
            while True:
                time.sleep(GC_INTERVAL)
                self.collect_gc()
        finally:
            gc.set_debug(saved_flags)
            self.logger.debug("monitor GC stopped")

    def collect_gc(self):
        try:
            collected = gc.collect()
            self.logger.debug("Collected %d objects from GC", collected)
            # Copy garbage so it is not modified while iterate over it.
            uncollectable = gc.garbage[:]
            if uncollectable:
                uncollectable = [
                    self.saferepr(obj) for obj in uncollectable
                    ]
                self.logger.warning(
                    "Found %d uncollectable objects: %s",
                    len(uncollectable), uncollectable
                )
        except Exception as exc:
            self.logger.exception("Error checking GC: %s", exc)

    def saferepr(self, obj):
        """
        Some objects from standard library fail in repr because of buggy
        __repr__ implementation. Try the builtin repr() and if it fails,
        warn and fallback to simple repr.
        """
        try:
            return repr(obj)
        except Exception as e:
            simple_repr = "<%s at 0x%x>" % (type(obj), id(obj))
            self.logger.warning("repr() failed for %s: %s", simple_repr, e)
            return simple_repr


# Copyright (C) Nial Peters 2009
#
# This file is part of PASKIL.
#
# PASKIL is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
#(at your option) any later version.
#
# PASKIL is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with PASKIL.  If not, see <http://www.gnu.org/licenses/>.
"""
Module containing miscellaneous statistical functions used internally by PASKIL
"""
import math
#
# def hist_variance_mean(hist):
#    """
#    Function returns the variance and mean of a histogram
#    """
#    sum_x = 0.0
#    sum_sqd_x = 0.0
#    for i in range(len(hist)):
#        sum_x = sum_x + (i*hist[i])
#        sum_sqd_x=sum_sqd_x+(i*hist[i]*hist[i])
#
#    mean = sum_x/float(sum(hist))
#    mean_of_squares = sum_sqd_x/float(sum(hist))
#
#    return (mean_of_squares-(mean*mean), mean)

def variance_mean(data):
    """
    Function returns the variance and mean of the data in a list
    """
    sum_x = 0.0
    sum_sqd_x = 0.0
    for i in range(len(data)):
        sum_x = sum_x + data[i]
        sum_sqd_x = sum_sqd_x + data[i] * data[i]

    mean = sum_x / float(len(data))
    mean_of_squares = sum_sqd_x / float(len(data))

    return (mean_of_squares - (mean * mean), mean)

def variance(data):
    """
    Function returns the variance of the data in a list
    """
    sum_x = 0.0
    sum_sqd_x = 0.0
    for i in range(len(data)):
        sum_x = sum_x + data[i]
        sum_sqd_x = sum_sqd_x + data[i] * data[i]

    mean = sum_x / float(len(data))
    mean_of_squares = sum_sqd_x / float(len(data))

    return mean_of_squares - (mean * mean)

def median(data):
    """
    Function returns the median value of the data in a list
    """
    # copy data list
    data_copy = []
    for i in range(len(data)):
        data_copy.append(data[i])

    data_copy.sort()

    if len(data_copy) % 2 == 0:
        return (data_copy[len(data_copy) // 2 - 1] + data_copy[len(data_copy) // 2]) / 2.0
    return data_copy[len(data_copy) // 2]

def mean(data):
    """
    Returns the mean of a list of data.
    """
    return float(sum(data)) / float(len(data))

def stdDev(data):
    """
    Function returns standard deviation of data in a list
    """
    return math.sqrt(variance(data))