# Copyright (c) 2017 The Khronos Group Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import division, print_function, absolute_import

import copy
import typing
from functools import partial

import numpy as np
import six

from nnef_tools.conversion import converter
from nnef_tools.conversion import transforms
from nnef_tools.core import graph_utils
from nnef_tools.core import utils
from nnef_tools.io.nnef.nnef_graph import *
from nnef_tools.io.nnef.parser_config import NNEFParserConfig
from nnef_tools.io.onnx.onnx_graph import *
from nnef_tools.io.onnx.onnx_io import NumpyDTypeByONNXDType
from nnef_tools.shape_inference import shape_inference as infer

_onnx_dtype_by_numpy_dtype = {str(np.dtype(v)): k
                              for k, v in six.iteritems(NumpyDTypeByONNXDType)
                              if v is not None}

_onnx_dtype_by_nnef_dtype = {
    'scalar': 'FLOAT',
    'integer': 'INT64',
    'logical': 'BOOL',
}

_numpy_dtype_by_nnef_dtype = {
    'scalar': 'float32',
    'integer': 'int64',
    'logical': 'bool',
}

_onnx_pad_mode_by_nnef_border = {'constant': 'constant',
                                 'reflect': 'reflect',
                                 'replicate': 'edge'}

class Converter(converter.Converter[NNEFTensor, NNEFOperation, NNEFGraph,
                                    ONNXTensor, ONNXOperation, ONNXGraph]):

    def __init__(self,
                 custom_converter_by_op_name=None,
                 enable_default_conversion=False,
                 enable_imprecise_image_resize=False):
        converters = {}
        converters.update(_DefaultConverters)
        if custom_converter_by_op_name is not None:
            converters.update(custom_converter_by_op_name)
        default_op_converter = convert_default if enable_default_conversion else None

        super(Converter, self).__init__(op_converter_by_name=converters,
                                        default_op_converter=default_op_converter)

        self._enable_imprecise_image_resize = enable_imprecise_image_resize

    def create_graph(self, source_graph):
        # type: (NNEFGraph)->ONNXGraph

        return ONNXGraph(name=source_graph.name)

    def convert_tensor(self, source_tensor, target_graph):
        # type: (NNEFTensor, ONNXGraph)->ONNXTensor
        # TODO handle constants here
        return ONNXTensor(graph=target_graph,
                          name=source_tensor.name,
                          shape=list(source_tensor.shape),
                          dtype=(self.numpy_dtype_to_onnx_dtype(source_tensor.data.dtype)
                                 if isinstance(source_tensor.data, np.ndarray)
                                 else self.nnef_dtype_to_onnx_dtype(source_tensor.dtype)),
                          data=copy.copy(source_tensor.data))

    def convert_graph(self, source_graph):
        graph_utils.remove_unreachable(source_graph)
        target_graph = super(Converter, self).convert_graph(source_graph)  # type: NNEFGraph
        graph_utils.remove_unreachable(target_graph)
        target_graph.generate_missing_names()
        return target_graph

    @property
    def enable_imprecise_image_resize(self):
        return self._enable_imprecise_image_resize

    @staticmethod
    def numpy_dtype_to_onnx_dtype(numpy_dtype):
        return _onnx_dtype_by_numpy_dtype[str(np.dtype(numpy_dtype))]

    @staticmethod
    def nnef_dtype_to_onnx_dtype(nnef_dtype):
        return _onnx_dtype_by_nnef_dtype[nnef_dtype]

    @staticmethod
    def nnef_dtype_to_numpy_dtype(nnef_dtype):
        return np.dtype(_numpy_dtype_by_nnef_dtype[nnef_dtype])

    @staticmethod
    def add_squeeze(onnx_graph, onnx_tensor, axes, squeezed_tensor=None):
        # type: (ONNXGraph, ONNXTensor, typing.List[int], typing.Optional[ONNXTensor])->ONNXTensor
        if squeezed_tensor is None:
            squeezed_tensor = ONNXTensor(graph=onnx_graph,
                                         shape=transforms.squeezed_shape(onnx_tensor.shape, axes),
                                         dtype=onnx_tensor.dtype)
        return ONNXOperation(graph=onnx_graph,
                             name="Squeeze",
                             inputs=onnx_tensor,
                             attribs=dict(axes=axes),
                             outputs=squeezed_tensor).output

    @staticmethod
    def add_unsqueeze(onnx_graph, onnx_tensor, axes, unsqueezed_tensor=None):
        # type: (ONNXGraph, ONNXTensor, typing.List[int], typing.Optional[ONNXTensor])->ONNXTensor
        if unsqueezed_tensor is None:
            unsqueezed_tensor = ONNXTensor(graph=onnx_graph,
                                           shape=transforms.unsqueezed_shape(onnx_tensor.shape, axes),
                                           dtype=onnx_tensor.dtype)
        return ONNXOperation(graph=onnx_graph,
                             name="Unsqueeze",
                             inputs=onnx_tensor,
                             attribs=dict(axes=axes),
                             outputs=unsqueezed_tensor).output

    @staticmethod
    def add_expand(onnx_graph, onnx_tensor, shape):
        # type: (ONNXGraph, ONNXTensor, typing.List[int])->ONNXTensor
        return ONNXOperation(graph=onnx_graph,
                             name="Expand",
                             inputs=(onnx_tensor,
                                     ONNXTensor(graph=onnx_graph,
                                                shape=[4],
                                                dtype='INT64',
                                                data=list(shape))),
                             outputs=ONNXTensor(graph=onnx_graph,
                                                shape=list(shape),
                                                dtype=onnx_tensor.dtype)).output

    @staticmethod
    def onnx_pads(nnef_padding):
        # type: (typing.List[typing.Tuple[int, int]])->typing.List[int]
        return utils.concat_lists(utils.zip_inverse(2, nnef_padding))

    @staticmethod
    def onnx_pad_mode(nnef_border):
        assert nnef_border in _onnx_pad_mode_by_nnef_border
        return _onnx_pad_mode_by_nnef_border[nnef_border]

    @staticmethod
    def constant_1d_tensor(graph, list_, dtype):
        # type:(ONNXGraph,  typing.List[typing.Any], str)->ONNXTensor
        return ONNXTensor(graph=graph, shape=[len(list_)], dtype=dtype, data=list(list_))

    @staticmethod
    def constant_0d_tensor(graph, value, dtype):
        # type:(ONNXGraph,  typing.Any, str)->ONNXTensor
        return ONNXTensor(graph=graph, shape=[], dtype=dtype, data=[value])

def convert_default(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    print("Warning: Converter of {} is not implemented, doing default conversion.".format(nnef_op.name))

    def flatten(x):
        return utils.flatten(x) if isinstance(x, (list, tuple)) else x

    ONNXOperation(graph=onnx_graph,
                  name=nnef_op.name,
                  inputs=converter.converted_tensors(nnef_op.inputs),
                  attribs={k: flatten(v) for k, v in six.iteritems(nnef_op.attribs)},
                  outputs=converter.converted_tensors(nnef_op.outputs))

def UNSUPPORTED(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    if converter.default_op_converter is not None:
        converter.default_op_converter(converter, nnef_op, onnx_graph)
    else:
        raise utils.NNEFToolsException('NNEF to ONNX: Unsupported op: {}'.format(nnef_op.name))

def UNNEEDED(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    assert False, "This should not be called!"

def NONATOMIC(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    assert False, "This should not be called!"

def generic_convert_binary(converter, nnef_op, onnx_graph, target_name, broadcast_workaround=False):
    # type: (Converter, NNEFOperation, ONNXGraph, str, bool)->None

    if broadcast_workaround:
        x, y = converter.converted_tensors(nnef_op.inputs)
        z = converter.converted_tensor(nnef_op.output)

        # Caffe2 onnx backend cannot broadcast in Pow
        if x.shape != z.shape:
            x = converter.add_expand(onnx_graph, x, z.shape)
        if y.shape != z.shape:
            y = converter.add_expand(onnx_graph, y, z.shape)

        ONNXOperation(
            graph=onnx_graph,
            name=target_name,
            inputs=(x, y),
            outputs=z)
    else:
        x, y = converter.converted_tensors(nnef_op.inputs)
        z = converter.converted_tensor(nnef_op.output)

        ONNXOperation(
            graph=onnx_graph,
            name=target_name,
            inputs=(converter.add_unsqueeze(onnx_graph, x, list(range(x.rank, y.rank))) if 0 < x.rank < y.rank else x,
                    converter.add_unsqueeze(onnx_graph, y, list(range(y.rank, x.rank))) if 0 < y.rank < x.rank else y),
            outputs=z)

def generic_convert_unary(converter, nnef_op, onnx_graph, target_name, copy_attribs=None):
    # type: (Converter, NNEFOperation, ONNXGraph, str, typing.List[str])->None
    onnx_op = ONNXOperation(graph=onnx_graph,
                            name=target_name,
                            inputs=converter.converted_tensor(nnef_op.input),
                            outputs=converter.converted_tensor(nnef_op.output))
    if copy_attribs:
        for attr in copy_attribs:
            onnx_op.attribs[attr] = copy.deepcopy(nnef_op.attribs[attr])

def convert_lrn(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    input, output = converter.converted_tensors((nnef_op.input, nnef_op.output))
    nnef_size = nnef_op.attribs['size']

    assert len(nnef_size) >= 2 and all(dim == 1 or i == 1 for i, dim in enumerate(nnef_size)), \
        'Only channel LRN is supported'

    ONNXOperation(graph=onnx_graph,
                  name='LRN',
                  inputs=input,
                  outputs=output,
                  attribs=dict(size=nnef_size[1],
                               alpha=nnef_op.attribs['alpha'],
                               beta=nnef_op.attribs['beta'],
                               bias=nnef_op.attribs['bias']))

def convert_matmul(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None
    A, B = converter.converted_tensors(nnef_op.inputs)
    C = converter.converted_tensor(nnef_op.output)

    assert A.rank <= 2 and B.rank <= 2, "Batch matmul is unsupported in ONNX"

    ONNXOperation(graph=onnx_graph,
                  name='Gemm',
                  inputs=(A, B, converter.constant_0d_tensor(graph=onnx_graph, value=0.0, dtype=C.dtype)),
                  outputs=C,
                  attribs=dict(transA=1 if nnef_op.attribs['transposeA'] else 0,
                               transB=1 if nnef_op.attribs['transposeB'] else 0))

def convert_linear(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None
    A, B, C = converter.converted_tensors(nnef_op.inputs)
    D = converter.converted_tensor(nnef_op.output)

    assert A.rank <= 2 and B.rank <= 2 and C.rank <= 2, "Batch matmul is unsupported in ONNX"

    ONNXOperation(graph=onnx_graph,
                  name='Gemm',
                  inputs=(A, B, C),
                  outputs=D,
                  attribs=dict(transA=0, transB=1))

def partial_convert_conv_deconv(converter, nnef_op, onnx_graph, is_deconv, input, filter, bias, output):
    # type: (Converter, NNEFOperation, ONNXGraph, bool, ONNXTensor, ONNXTensor, ONNXTensor, ONNXTensor)->None

    strides = list(nnef_op.attribs['stride'])
    if not strides:
        strides = [1] * (input.rank - 2)

    dilations = nnef_op.attribs['dilation']
    if not dilations:
        dilations = [1] * (input.rank - 2)

    groups = nnef_op.attribs.get('groups', 1)  # default needed because box does not have groups
    if groups == 0:
        groups = input.shape[1]

    assert nnef_op.attribs['border'] == 'constant'

    if is_deconv:
        pads = nnef_op.attribs['padding']
        if not pads:  # auto pad
            calc_output_size = [i * s for i, s in (input.shape[2:], strides)]
            pads = infer.same_padding(upscaled_input=calc_output_size,
                                      filter=filter.shape[2:],
                                      stride=strides,
                                      dilation=dilations)
        else:
            calc_output_size = infer.conv(input=input.shape,
                                          filter=filter.shape[2:],
                                          padding=pads,
                                          stride=strides,
                                          dilation=dilations,
                                          groups=groups,
                                          output_channels=output.shape[1],
                                          format=infer.Format.NCHW,
                                          deconv=True)[2:]
        output_size = output.shape[2:]
        output_padding = [o - c for o, c in zip(output_size, calc_output_size)]
    else:
        pads = nnef_op.attribs['padding']
        if not pads:
            pads = infer.same_padding(upscaled_input=input.shape[2:],
                                      filter=filter.shape[2:],
                                      stride=strides,
                                      dilation=dilations)
        output_padding = [0] * len(pads)

    pads = converter.onnx_pads(pads)

    if bias.is_constant and bias.data == [0.0]:
        inputs = (input, filter)
    else:
        if bias.rank == 2:
            assert bias.shape[0] == 1
            bias = converter.add_squeeze(onnx_graph=onnx_graph, onnx_tensor=bias, axes=[0])
        inputs = (input, filter, bias)

    op = ONNXOperation(graph=onnx_graph,
                       name='ConvTranspose' if is_deconv else 'Conv',
                       inputs=inputs,
                       attribs=dict(
                           kernel_shape=filter.shape[2:],  # Not mandatory, but Caffe2 fails without this
                           strides=strides,
                           dilations=dilations,
                           pads=pads,
                           group=groups),
                       outputs=output)

    if is_deconv:
        op.attribs['output_padding'] = output_padding

def generic_convert_conv_deconv(converter, nnef_op, onnx_graph, is_deconv):
    # type: (Converter, NNEFOperation, ONNXGraph, bool)->None

    input, filter, bias = converter.converted_tensors(nnef_op.inputs)
    output = converter.converted_tensor(nnef_op.output)
    partial_convert_conv_deconv(converter, nnef_op, onnx_graph, is_deconv, input, filter, bias, output)

def convert_box(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    input, output = converter.converted_tensors((nnef_op.input, nnef_op.output))

    if nnef_op.attribs['size'] == [1] * input.rank:
        onnx_op = ONNXOperation(graph=onnx_graph,
                                name='Pad',
                                inputs=input,
                                attribs=dict(mode=converter.onnx_pad_mode(nnef_op.attribs['border']),
                                             pads=converter.onnx_pads(nnef_op.attribs['padding'])),
                                outputs=output)

        if onnx_op.attribs['mode'] == 'constant':
            onnx_op.attribs['value'] = 0.0
        return

    if nnef_op.attribs['normalize']:
        partial_convert_pool(converter, nnef_op, onnx_graph,
                             target_name='AveragePool', input=input, outputs=(output,))
    else:
        temporary = ONNXTensor(graph=onnx_graph, shape=list(output.shape), dtype=output.dtype)
        partial_convert_pool(converter, nnef_op, onnx_graph,
                             target_name='AveragePool', input=input, outputs=(temporary,), force_constant=True)
        ONNXOperation(
            graph=onnx_graph,
            name='Mul',
            inputs=(temporary,
                    converter.constant_0d_tensor(onnx_graph, float(np.product(nnef_op.attribs['size'])), 'FLOAT')),
            outputs=output)

def generic_convert_pool(converter, nnef_op, onnx_graph, target_name):
    # type: (Converter, NNEFOperation, ONNXGraph, str)->None

    input = converter.converted_tensor(nnef_op.input)
    outputs = converter.converted_tensors(nnef_op.outputs)
    partial_convert_pool(converter, nnef_op, onnx_graph, target_name, input, outputs)

def partial_convert_pool(converter, nnef_op, onnx_graph, target_name, input, outputs, force_constant=False):
    # type: (Converter, NNEFOperation, ONNXGraph, str, ONNXTensor, typing.Tuple[ONNXTensor, ...], bool)->None
    if nnef_op.name == 'argmax_pool':
        outputs = (ONNXTensor(graph=onnx_graph, shape=list(outputs[0].shape), dtype=input.dtype),) + tuple(outputs)

    strides = list(nnef_op.attribs['stride'])
    if not strides:
        strides = [1] * input.rank
    dilations = nnef_op.attribs['dilation']
    if not dilations:
        dilations = [1] * input.rank

    assert nnef_op.attribs['border'] in ['constant', 'ignore']

    pads = nnef_op.attribs['padding']
    if not pads:
        pads = infer.same_padding(upscaled_input=input.shape[2:],
                                  filter=nnef_op.attribs['size'][2:],
                                  stride=strides[2:],
                                  dilation=dilations[2:])
    else:
        assert pads[:2] == [(0, 0), (0, 0)], "Padding in batch and channel dimensions ({}) is not supported in ONNX".format(pads)
        pads = pads[2:]
    pads = converter.onnx_pads(pads)

    assert nnef_op.attribs['size'][:2] == strides[:2] == dilations[:2] == [1, 1], \
        'Pooling in batch and channel dimensions is not supported in ONNX'
    strides = strides[2:]
    dilations = dilations[2:]

    assert all(d == 1 for d in dilations), 'Dilation is not supported for pooling in ONNX'

    onnx_op = ONNXOperation(graph=onnx_graph,
                            name=target_name,
                            inputs=input,
                            attribs=dict(kernel_shape=nnef_op.attribs['size'][2:],
                                         pads=pads,
                                         strides=strides),
                            outputs=outputs)

    if target_name == 'AveragePool':
        onnx_op.attribs['count_include_pad'] = 1 if (nnef_op.attribs['border'] == 'constant' or force_constant) else 0

def convert_desample(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    input, indices = converter.converted_tensors(nnef_op.inputs)
    output = converter.converted_tensor(nnef_op.output)

    strides = list(nnef_op.attribs['stride'])
    if not strides:
        strides = [1] * input.rank
    dilations = nnef_op.attribs['dilation']
    if not dilations:
        dilations = [1] * input.rank

    assert nnef_op.attribs['border'] in ['constant', 'ignore']

    pads = nnef_op.attribs['padding']
    if not pads:  # auto pad
        calc_output_size = [i * s for i, s in (input.shape[2:], strides)]
        pads = infer.same_padding(upscaled_input=calc_output_size,
                                  filter=nnef_op.attribs['size'],
                                  stride=strides,
                                  dilation=dilations)
    else:
        assert pads[:2] == [(0, 0), (0, 0)], "Padding in batch and channel dimensions ({}) is not supported in ONNX".format(pads[:2])
        pads = pads[2:]
    pads = converter.onnx_pads(pads)

    assert nnef_op.attribs['size'][:2] == strides[:2] == dilations[:2] == [1, 1], \
        'Pooling in batch and channel dimensions is not supported in ONNX'
    strides = strides[2:]
    dilations = dilations[2:]

    assert all(d == 1 for d in dilations), 'Dilation is not supported for pooling in ONNX'

    ONNXOperation(graph=onnx_graph,
                  name='MaxUnpool',
                  inputs=(input, indices),
                  attribs=dict(kernel_shape=nnef_op.attribs['size'][2:],
                               pads=pads,
                               strides=strides),
                  outputs=output)

def convert_reshape(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    input, output = converter.converted_tensors((nnef_op.input, nnef_op.output))

    shape = nnef_op.attribs['shape']
    axis_start = nnef_op.attribs['axis_start']
    axis_count = nnef_op.attribs['axis_count']
    if axis_count == -1:
        axis_count = input.rank - axis_start

    onnx_shape = input.shape[:axis_start] + shape + input.shape[axis_start + axis_count:]

    ONNXOperation(graph=onnx_graph,
                  name='Reshape',
                  inputs=(input, converter.constant_1d_tensor(graph=onnx_graph,
                                                              list_=onnx_shape,
                                                              dtype='INT64')),
                  outputs=output)

def convert_transpose(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    input, output = converter.converted_tensors((nnef_op.input, nnef_op.output))

    if not nnef_op.attribs['axes']:
        ONNXOperation(graph=onnx_graph, name='Identity', inputs=input, outputs=output)
        return

    perm = list(nnef_op.attribs['axes'])
    if len(perm) < input.rank:
        perm += list(range(len(perm), input.rank))

    ONNXOperation(graph=onnx_graph,
                  name='Transpose',
                  inputs=input,
                  attribs=dict(perm=perm),
                  outputs=output)

def convert_concat(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    inputs = converter.converted_tensors(nnef_op.inputs)
    output = converter.converted_tensor(nnef_op.output)

    ONNXOperation(graph=onnx_graph,
                  name='Concat',
                  inputs=inputs,
                  attribs=dict(axis=nnef_op.attribs['axis']),
                  outputs=output)

def convert_split(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    input = converter.converted_tensor(nnef_op.input)
    outputs = converter.converted_tensors(nnef_op.outputs)

    axis = nnef_op.attribs['axis']
    ratios = nnef_op.attribs['ratios']
    assert input.shape[axis] % sum(ratios) == 0
    unit = input.shape[axis] // sum(ratios)

    ONNXOperation(graph=onnx_graph,
                  name='Split',
                  inputs=input,
                  attribs=dict(axis=axis,
                               split=[ratio * unit for ratio in ratios]),
                  outputs=outputs)

def convert_stack(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    inputs = converter.converted_tensors(nnef_op.inputs)
    output = converter.converted_tensor(nnef_op.output)

    inputs = [converter.add_unsqueeze(onnx_graph, input, [nnef_op.attribs['axis']]) for input in inputs]

    ONNXOperation(graph=onnx_graph,
                  name='Concat',
                  inputs=inputs,
                  attribs=dict(axis=nnef_op.attribs['axis']),
                  outputs=output)

def convert_unstack(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    input = converter.converted_tensor(nnef_op.input)
    outputs = converter.converted_tensors(nnef_op.outputs)

    axis = nnef_op.attribs['axis']
    num_parts = input.shape[axis]

    def unsqueeze(shape, axis):
        shape = list(shape)
        shape.insert(axis, 1)
        return shape

    intermediates = [ONNXTensor(graph=onnx_graph, shape=unsqueeze(output.shape, axis), dtype=output.dtype)
                     for output in outputs]

    ONNXOperation(graph=onnx_graph,
                  name='Split',
                  inputs=input,
                  attribs=dict(axis=axis, split=[1] * num_parts),
                  outputs=intermediates)

    for intermediate, output in zip(intermediates, outputs):
        converter.add_unsqueeze(onnx_graph, intermediate, axis, unsqueezed_tensor=output)

def convert_slice(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    input, output = converter.converted_tensors((nnef_op.input, nnef_op.output))

    if not nnef_op.attribs['axes']:
        ONNXOperation(graph=onnx_graph, name='Identity', inputs=input, outputs=output)
        return

    axes = list(nnef_op.attribs['axes'])
    starts = list(nnef_op.attribs['begin'])

    ends = list(utils.INT32_MAX if e == 0 else e for e in nnef_op.attribs['end'])

    ONNXOperation(graph=onnx_graph,
                  name='Slice',
                  inputs=input,
                  attribs=dict(axes=axes, starts=starts, ends=ends),
                  outputs=output)

def generic_convert_binary1_or_binary2(converter, nnef_op, onnx_graph, target_name1, target_name2):
    # type: (Converter, NNEFOperation, ONNXGraph, str, str)->None

    x, y = converter.converted_tensors(nnef_op.inputs)
    z = converter.converted_tensor(nnef_op.output)

    binary1 = ONNXOperation(
        graph=onnx_graph,
        name=target_name1,
        inputs=(converter.add_unsqueeze(onnx_graph, x, list(range(x.rank, y.rank))) if 0 < x.rank < y.rank else x,
                converter.add_unsqueeze(onnx_graph, y, list(range(y.rank, x.rank))) if 0 < y.rank < x.rank else y),
        outputs=ONNXTensor(graph=onnx_graph, shape=list(z.shape), dtype=z.dtype))

    binary2 = ONNXOperation(
        graph=onnx_graph,
        name=target_name2,
        inputs=(converter.add_unsqueeze(onnx_graph, x, list(range(x.rank, y.rank))) if 0 < x.rank < y.rank else x,
                converter.add_unsqueeze(onnx_graph, y, list(range(y.rank, x.rank))) if 0 < y.rank < x.rank else y),
        outputs=ONNXTensor(graph=onnx_graph, shape=list(z.shape), dtype=z.dtype))

    ONNXOperation(
        graph=onnx_graph,
        name="Or",
        inputs=(binary1.output, binary2.output),
        outputs=z)

def convert_ne(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    x, y = converter.converted_tensors(nnef_op.inputs)
    z = converter.converted_tensor(nnef_op.output)

    equal = ONNXOperation(
        graph=onnx_graph,
        name='Equal',
        inputs=(converter.add_unsqueeze(onnx_graph, x, list(range(x.rank, y.rank))) if 0 < x.rank < y.rank else x,
                converter.add_unsqueeze(onnx_graph, y, list(range(y.rank, x.rank))) if 0 < y.rank < x.rank else y),
        outputs=ONNXTensor(graph=onnx_graph, shape=list(z.shape), dtype=z.dtype))

    ONNXOperation(
        graph=onnx_graph,
        name="Not",
        inputs=equal.output,
        outputs=z)

def convert_round(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    input, output = converter.converted_tensors((nnef_op.input, nnef_op.output))

    add = ONNXOperation(graph=onnx_graph,
                        name='Add',
                        inputs=(input, converter.constant_0d_tensor(graph=onnx_graph, value=0.5, dtype=input.dtype)),
                        outputs=ONNXTensor(graph=onnx_graph, shape=list(output.shape), dtype=output.dtype))

    ONNXOperation(graph=onnx_graph,
                  name='Floor',
                  inputs=add.output,
                  outputs=output)

def convert_select(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    cond, true_value, false_value = converter.converted_tensors(nnef_op.inputs)
    output = converter.converted_tensor(nnef_op.output)

    max_rank = max(t.rank for t in [cond, true_value, false_value])

    def broadcast(t):
        return converter.add_unsqueeze(onnx_graph, t, list(range(t.rank, max_rank))) if 0 < t.rank < max_rank else t

    ONNXOperation(
        graph=onnx_graph,
        name='Where',
        inputs=(broadcast(cond), broadcast(true_value), broadcast(false_value)),
        outputs=output)

def convert_clamp(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    x, a, b = converter.converted_tensors(nnef_op.inputs)
    y = converter.converted_tensor(nnef_op.output)

    if a.is_constant and a.shape == [] and b.is_constant and b.shape == []:
        ONNXOperation(
            graph=onnx_graph,
            name='Clip',
            inputs=x,
            attribs=dict(min=a.data[0], max=b.data[0]),
            outputs=y)
    else:
        max_rank = max(t.rank for t in [x, a, b])

        def broadcast(t):
            return converter.add_unsqueeze(onnx_graph, t, list(range(t.rank, max_rank))) if 0 < t.rank < max_rank else t

        min = ONNXOperation(
            graph=onnx_graph,
            name='Min',
            inputs=(broadcast(x), broadcast(b)),
            outputs=ONNXTensor(graph=onnx_graph, shape=list(y.shape), dtype=y.dtype),
        )

        ONNXOperation(
            graph=onnx_graph,
            name='Max',
            inputs=(min.output, broadcast(a)),
            outputs=y,
        )

def convert_multilinear_upsample(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None
    input, output = converter.converted_tensors((nnef_op.input, nnef_op.output))

    if nnef_op.attribs['method'] != 'symmetric':
        if converter.enable_imprecise_image_resize:
            print("Warning: method={} is unsupported in multilinear_upsample, "
                  "using symmetric, because enable_imprecise_image_resize was True"
                  .format(nnef_op.attribs["method"]))
        else:
            assert False, "Error: method={} is unsupported in multilinear_upsample. " \
                          "Use enable_imprecise_image_resize=True, to suppress this error." \
                .format(nnef_op.attribs["method"])

    if nnef_op.attribs['border'] != 'replicate':
        if converter.enable_imprecise_image_resize:
            print("Warning: border={} is unsupported in multilinear_upsample, "
                  "using replicate, because enable_imprecise_image_resize was True"
                  .format(nnef_op.attribs["border"]))
        else:
            assert False, "Error: border={} is unsupported in multilinear_upsample. " \
                          "Use enable_imprecise_image_resize=True, to suppress this error." \
                .format(nnef_op.attribs["border"])

    scales = [float(f) for f in [1, 1] + nnef_op.attribs['factor']]

    ONNXOperation(graph=onnx_graph,
                  name='Upsample',
                  inputs=(input, converter.constant_1d_tensor(graph=onnx_graph, list_=scales, dtype='FLOAT')),
                  attribs=dict(mode='linear'),
                  outputs=output)

def generic_convert_reduce(converter, nnef_op, onnx_graph, target_name, target_name_if_normalized="", one_axis=False):
    # type: (Converter, NNEFOperation, ONNXGraph, str, str, bool)->None
    input, output = converter.converted_tensors((nnef_op.input, nnef_op.output))

    if not nnef_op.attribs['axes']:
        ONNXOperation(graph=onnx_graph, name='Identity', inputs=input, outputs=output)
        return

    if one_axis:
        assert len(nnef_op.attribs['axes']) == 1, "{} supports only one axis in ONNX".format(target_name)

    onnx_op = ONNXOperation(graph=onnx_graph,
                            name=(target_name_if_normalized
                                  if target_name_if_normalized and nnef_op.attribs['normalize']
                                  else target_name),
                            inputs=input,
                            attribs=dict(keepdims=1),
                            outputs=output)

    if one_axis:
        onnx_op.attribs['axis'] = nnef_op.attribs['axes'][0]
    else:
        onnx_op.attribs['axes'] = list(nnef_op.attribs['axes'])

def convert_prelu(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None
    input, slope = converter.converted_tensors(nnef_op.inputs)
    output = converter.converted_tensor(nnef_op.output)

    ONNXOperation(graph=onnx_graph,
                  name='PRelu',
                  inputs=(input, converter.add_squeeze(onnx_graph, slope, [0])),
                  outputs=output)

def convert_batch_normalization(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    input, mean, variance, offset, scale = converter.converted_tensors(nnef_op.inputs)
    output = converter.converted_tensor(nnef_op.output)

    if input.rank < 3:  # Caffe2 BatchNorm works only for rank >= 3
        axes = list(range(input.rank, 3))
        input = converter.add_unsqueeze(onnx_graph=onnx_graph, onnx_tensor=input, axes=axes)
        final_output = output
        output = ONNXTensor(graph=onnx_graph, shape=list(input.shape), dtype=input.dtype)
        converter.add_squeeze(onnx_graph=onnx_graph, onnx_tensor=output, axes=axes, squeezed_tensor=final_output)

    ONNXOperation(graph=onnx_graph,
                  name='BatchNormalization',
                  inputs=(input,
                          converter.add_squeeze(onnx_graph, scale, [0]),
                          converter.add_squeeze(onnx_graph, offset, [0]),
                          converter.add_squeeze(onnx_graph, mean, [0]),
                          converter.add_squeeze(onnx_graph, variance, [0])),
                  attribs=dict(epsilon=nnef_op.attribs['epsilon']),
                  outputs=output)

def convert_copy_n(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    input = converter.converted_tensor(nnef_op.input)
    outputs = converter.converted_tensors(nnef_op.outputs)

    for output in outputs:
        ONNXOperation(graph=onnx_graph,
                      name='Identity',
                      inputs=input,
                      outputs=output)

def convert_nearest_upsample(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    input, output = converter.converted_tensors((nnef_op.input, nnef_op.output))

    scales = [float(f) for f in [1, 1] + nnef_op.attribs['factor']]

    ONNXOperation(graph=onnx_graph,
                  name='Upsample',
                  inputs=(input, converter.constant_1d_tensor(onnx_graph, scales, 'FLOAT')),
                  attribs=dict(mode='nearest'),
                  outputs=output)

def convert_tile(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    input, output = converter.converted_tensors((nnef_op.input, nnef_op.output))

    ONNXOperation(graph=onnx_graph,
                  name='Tile',
                  inputs=(input,
                          converter.constant_1d_tensor(graph=onnx_graph,
                                                       list_=nnef_op.attribs['repeats'],
                                                       dtype='INT64')),
                  outputs=output)

def convert_pad(converter, nnef_op, onnx_graph):
    # type: (Converter, NNEFOperation, ONNXGraph)->None

    input, output = converter.converted_tensors((nnef_op.input, nnef_op.output))

    onnx_op = ONNXOperation(graph=onnx_graph,
                            name='Pad',
                            inputs=input,
                            attribs=dict(mode=converter.onnx_pad_mode(nnef_op.attribs['border']),
                                         pads=converter.onnx_pads(nnef_op.attribs['padding'])),
                            outputs=output)

    if onnx_op.attribs['mode'] == 'constant':
        onnx_op.attribs['value'] = 0.0

_DefaultConverters = {
    'external': UNNEEDED,
    'variable': UNNEEDED,
    'constant': UNNEEDED,
    'update': UNSUPPORTED,
    'reshape': convert_reshape,
    'transpose': convert_transpose,
    'concat': convert_concat,
    'split': convert_split,
    'slice': convert_slice,
    'squeeze': partial(generic_convert_unary, target_name='Squeeze', copy_attribs=['axes']),
    'unsqueeze': partial(generic_convert_unary, target_name='Unsqueeze', copy_attribs=['axes']),
    'stack': convert_stack,
    'unstack': convert_unstack,
    'add': partial(generic_convert_binary, target_name='Add'),
    'sub': partial(generic_convert_binary, target_name='Sub'),
    'mul': partial(generic_convert_binary, target_name='Mul'),
    'div': partial(generic_convert_binary, target_name='Div'),
    'pow': partial(generic_convert_binary, target_name='Pow', broadcast_workaround=True),
    'exp': partial(generic_convert_unary, target_name='Exp'),
    'log': partial(generic_convert_unary, target_name='Log'),
    'abs': partial(generic_convert_unary, target_name='Abs'),
    'sign': partial(generic_convert_unary, target_name='Sign'),
    'rcp': partial(generic_convert_unary, target_name='Reciprocal'),
    'neg': partial(generic_convert_unary, target_name='Neg'),
    'copy': partial(generic_convert_unary, target_name='Identity'),
    'lt': partial(generic_convert_binary, target_name='Less'),
    'gt': partial(generic_convert_binary, target_name='Greater'),
    'le': partial(generic_convert_binary1_or_binary2, target_name1='Less', target_name2='Equal'),
    'ge': partial(generic_convert_binary1_or_binary2, target_name1='Greater', target_name2='Equal'),
    'eq': partial(generic_convert_binary, target_name='Equal'),
    'ne': convert_ne,
    'and': partial(generic_convert_binary, target_name='And'),
    'or': partial(generic_convert_binary, target_name='Or'),
    'not': partial(generic_convert_unary, target_name='Not'),
    'floor': partial(generic_convert_unary, target_name='Floor'),
    'ceil': partial(generic_convert_unary, target_name='Ceil'),
    'round': convert_round,
    'select': convert_select,
    'sqr': NONATOMIC,
    'sqrt': partial(generic_convert_unary, target_name='Sqrt'),
    'rsqr': NONATOMIC,
    'rsqrt': NONATOMIC,
    'log2': NONATOMIC,
    'min': partial(generic_convert_binary, target_name='Min', broadcast_workaround=True),
    'max': partial(generic_convert_binary, target_name='Max', broadcast_workaround=True),
    'clamp': convert_clamp,
    'matmul': convert_matmul,
    'conv': partial(generic_convert_conv_deconv, is_deconv=False),
    'deconv': partial(generic_convert_conv_deconv, is_deconv=True),
    'box': convert_box,
    'debox': UNSUPPORTED,
    'argmax_pool': partial(generic_convert_pool, target_name='MaxPool'),  # not typo
    'sample': UNSUPPORTED,
    'desample': convert_desample,
    'nearest_downsample': NONATOMIC,
    'area_downsample': NONATOMIC,
    'nearest_upsample': convert_nearest_upsample,
    'multilinear_upsample': convert_multilinear_upsample,
    'sum_reduce': partial(generic_convert_reduce, target_name='ReduceSum', target_name_if_normalized='ReduceMean'),
    'max_reduce': partial(generic_convert_reduce, target_name='ReduceMax'),
    'min_reduce': partial(generic_convert_reduce, target_name='ReduceMin'),
    'argmax_reduce': partial(generic_convert_reduce, target_name='ArgMax', one_axis=True),
    'argmin_reduce': partial(generic_convert_reduce, target_name='ArgMin', one_axis=True),
    'mean_reduce': partial(generic_convert_reduce, target_name='ReduceMean'),
    'moments': NONATOMIC,
    'relu': partial(generic_convert_unary, target_name='Relu'),
    'sigmoid': partial(generic_convert_unary, target_name='Sigmoid'),
    'tanh': partial(generic_convert_unary, target_name='Tanh'),
    'softabs': NONATOMIC,
    'softmax': partial(generic_convert_unary, target_name='Softmax'),
    'softplus': partial(generic_convert_unary, target_name='Softplus'),
    'elu': partial(generic_convert_unary, target_name='Elu'),
    'prelu': convert_prelu,
    'leaky_relu': partial(generic_convert_unary, target_name='LeakyRelu', copy_attribs=['alpha']),
    'max_pool_with_index': partial(generic_convert_pool, target_name='MaxPool'),  # not typo
    'max_pool': partial(generic_convert_pool, target_name='MaxPool'),
    'avg_pool': partial(generic_convert_pool, target_name='AveragePool'),
    'rms_pool': NONATOMIC,
    'linear': convert_linear,
    'separable_conv': NONATOMIC,
    'separable_deconv': NONATOMIC,
    'local_response_normalization': convert_lrn,
    'local_mean_normalization': NONATOMIC,
    'local_variance_normalization': NONATOMIC,
    'local_contrast_normalization': NONATOMIC,
    'l1_normalization': NONATOMIC,
    'l2_normalization': NONATOMIC,
    'batch_normalization': convert_batch_normalization,
    'avg_roi_pool': UNSUPPORTED,
    'max_roi_pool': UNSUPPORTED,  # maybe supported
    'roi_resample': UNSUPPORTED,
    'avg_roi_align': UNSUPPORTED,
    'max_roi_align': UNSUPPORTED,
    'linear_quantize': NONATOMIC,
    'logarithmic_quantize': NONATOMIC,
    'copy_n': convert_copy_n,
    'tile': convert_tile,
    'pad': convert_pad,
    'sin': partial(generic_convert_unary, target_name="Sin"),
    'cos': partial(generic_convert_unary, target_name="Cos"),
}

# TODO add to class as static(?) method
# NNEF must be parsed with this before calling nnef_to_tf.Converter on it
ParserConfig = NNEFParserConfig(lowered=[k for k, v in six.iteritems(_DefaultConverters) if v is NONATOMIC])


from xai.brain.wordbase.verbs._yammer import _YAMMER

#calss header
class _YAMMERS(_YAMMER, ):
	def __init__(self,):
		_YAMMER.__init__(self)
		self.name = "YAMMERS"
		self.specie = 'verbs'
		self.basic = "yammer"
		self.jsondata = {}


from pathlib import Path

from ray import tune
from ray.util.sgd.utils import BATCH_SIZE

import workloads.common as com
from fluid.trainer import TorchTrainer
from workloads.common import cifar as workload

DATA_PATH, RESULTS_PATH = com.detect_paths()
EXP_NAME = com.remove_prefix(Path(__file__).stem, "tune_")

def create_grid_search_space(exp_no):
    #     mutations = space.create_grid_space_1()
    method_name = "space.create_grid_space_" + str(exp_no)
    mutations = eval(method_name)()
    return {key: tune.sample_from(val) for key, val in mutations.items()}

def setup_tune_scheduler(exp_no):
    search_space = create_grid_search_space(exp_no)
    sync_to_driver = not RESULTS_PATH.startswith("/nfs")

    return dict(
        config=search_space,
        resources_per_trial={"gpu": 1},
        sync_to_driver=sync_to_driver,
        local_dir=RESULTS_PATH,
        name=EXP_NAME + str(exp_no),
    )

def main():
    exp_no, sd = com.init_ray()
    MyTrainable = TorchTrainer.as_trainable(
        data_creator=workload.data_creator,
        model_creator=workload.model_creator,
        loss_creator=workload.loss_creator,
        optimizer_creator=workload.optimizer_creator,
        config={"seed": sd, BATCH_SIZE: 64, "extra_fluid_trial_resources": {}},
    )

    params = {
        # **com.run_options(__file__),
        # 'stop': workload.create_stopper(),
        **setup_tune_scheduler(exp_no),
    }

    analysis = tune.run(MyTrainable, stop={"training_iteration": 81}, **params)

    dfs = analysis.trial_dataframes
    for logdir, df in dfs.items():
        ld = Path(logdir)
        df.to_csv(ld / "trail_dataframe.csv")

if __name__ == "__main__":
    main()


import os
import warnings

import gym

from sacking.environment import load_env
from sacking.policy import DiscretePolicy
from sacking.q_network import DiscreteQNetwork
from sacking.trainer import train

def main():
    if (os.environ.get('OMP_NUM_THREADS') != '1'
           or os.environ.get('MKL_NUM_THREADS') != '1'):
        warnings.warn('running without OMP_NUM_THREADS=1 MKL_NUM_THREADS=1')

    env = load_env('gym/CartPole-v1')
    valid_env = load_env('gym/CartPole-v1')

    assert isinstance(env.observation_space, gym.spaces.Box)
    assert len(env.observation_space.shape) == 1
    obs_dim = env.observation_space.shape[0]

    assert isinstance(env.action_space, gym.spaces.Discrete)
    assert len(env.action_space.shape) == 0
    action_dim = env.action_space.n

    policy = DiscretePolicy(obs_dim, action_dim,
                            hidden_layers=[64, 64])
    q_network = DiscreteQNetwork(obs_dim, action_dim,
                                 hidden_layers=[64, 64],
                                 num_nets=2)

    train(
        policy, q_network, env,
        batch_size=128,
        learning_rate=0.003,
        num_steps=50000,
        replay_buffer_size=100000,
        rundir='output',
        validation_env=valid_env,
    )

if __name__ == '__main__':
    main()


__author__ = 'christopherdavidson'

import uuid
from common.database import Database
from models.room import Room

####################################
# ROOM MATRIX: Keeps Track of Rooms Available for meetings
####################################

class RoomMatrix(object):
    counter = 2     # shared for all instances

    # CONSTRUCTOR
    def __init__(self, roomNum=None, room_id=None, _id=None):
        RoomMatrix.counter += 1
        self.roomNum = RoomMatrix.counter
        self.room_id = room_id
        self._id = uuid.uuid4().hex if _id is None else _id

    # STORED DATA IN MEETING CLASS
    def json(self):
        return {
            'roomNum': self.roomNum,
            'room_id': self.room_id,
            '_id': self._id
        }

    #### INSTANCE METHODS ####
    def create_room(self):
        room = Room(meetings=None,roomNum=self.roomNum)
        self.room_id = room._id
        room.save_to_mongo()
        Database.insert(collection='office', data=self.json())
        return room._id

    #### CLASS METHODS ####
    @classmethod
    def get_room_number(cls, roomID):
        office = Database.find_one('office', {'room_id': roomID})
        r_number = office['room']
        return r_number

    @classmethod
    def get_rooms(cls):
        return [room for room in Database.find(collection='office', query={})]

    @classmethod
    def get_by_id(cls, m_id):
        office = Database.find_one('office', {'_id': m_id})
        r_id = office['room_id']
        return r_id

    @classmethod
    def delete_room(cls, office_id, room_id):
        if room_id is not None:
            # room.from_mongo(by_id) should return a class object
            # so we can use (.) operator to access members of room
            room = Room.get_from_mongo(room_id)
            # need to make sure we don't delete room and its corresponding matrix_id
            # if there are existing meetings in the room object; else don't delete either object
            if len(room.meetings) < 1:
                if RoomMatrix.counter > 2:
                    RoomMatrix.counter -= 1
                room.delete_room_base(room_id)
                Database.remove_one(collection='office', searchVal=office_id)
                return True
        return False


"""Handle transaction information interface."""

from marshmallow import fields

from ....messaging.models.base_record import (
    BaseExchangeRecord,
    BaseExchangeSchema,
)

from ....config.injection_context import InjectionContext

class TransactionRecord(BaseExchangeRecord):
    """Represents a single transaction record."""

    class Meta:
        """Transaction Record metadata."""

        schema_class = "TransactionRecordSchema"

    RECORD_ID_NAME = "transaction_id"
    CACHE_ENABLED = True
    TAG_NAMES = {"state", "thread_id", "connection_id", "transaction_type"}
    RECORD_TYPE = "transaction"
    STATE_INIT = "init"

    def __init__(
        self,
        *,
        transaction_id: str = None,
        _type: str = None,
        transaction_type:str = None,
        comment: str = None,
        signature_request: list = [],
        signature_response: list = [],
        timing: dict = {},
        formats: list = [],
        messages_attach: list = [],
        thread_id: str = None,
        connection_id: str = None,
        state: str = None,
        **kwargs,
    ):
        """Initialize a new TransactionRecord."""

        super().__init__(transaction_id, state or self.STATE_INIT, **kwargs)
        self._type = _type
        self.transaction_type = transaction_type
        self.comment = comment
        self.signature_request = signature_request
        self.signature_response = signature_response
        self.timing = timing
        self.formats = formats
        self.messages_attach = messages_attach
        self.thread_id = thread_id
        self.connection_id = connection_id

    @property
    def record_value(self) -> dict:
        """Accessor for the JSON record value generated for this credential exchange."""
        return {
            prop: getattr(self, prop)
            for prop in (
                "signature_request",
                "timing",
                "formats",
                "messages_attach",
                "signature_response",
                "transaction_type"
            )
        }

    @classmethod
    async def retrieve_by_connection_and_thread(
        cls, context: InjectionContext, connection_id: str, thread_id: str
    ) -> "TransactionRecord":
        """Retrieve a transaction record by connection and thread ID."""
        cache_key = f"credential_exchange_ctidx::{connection_id}::{thread_id}"
        record_id = await cls.get_cached_key(context, cache_key)
        if record_id:
            record = await cls.retrieve_by_id(context, record_id)
        else:
            record = await cls.retrieve_by_tag_filter(
                context,
                {"thread_id": thread_id},
                {"connection_id": connection_id} if connection_id else None,
            )
            await cls.set_cached_key(context, cache_key, record._id)
        return record

class TransactionRecordSchema(BaseExchangeSchema):
    """Schema to allow serialization/deserialization of transaction records."""

    class Meta:
        """TransactionRecordSchema metadata."""

        model_class = "TransactionRecord"

    _id = fields.Str(
        required=False, description="Connection identifier", example="any_example"
    )
    _type = fields.Str(
        required=False,
        description="Transaction type",
        example="The type of transaction",
    )
    transaction = fields.Str(
        required=False,
        description="The type of transaction (Schema or a Credential Definition"
    )
    signature_request = fields.List(
        fields.Dict(),
        required=False,
    )
    signature_response = fields.List(fields.Dict(), required=False)
    timing = fields.Dict(required=False)
    formats = fields.List(fields.Dict(), required=False)
    messages_attach = fields.List(fields.Dict(), required=False)
    thread_id = fields.Str(required=False, description="Thread Identifier")
    connection_id = fields.Str(
        required=False,
        description="The connection identifier for thie particular transaction record",
    )


"""
https://leetcode.com/problems/add-and-search-word-data-structure-design/

Use Trie to store the added words. Use DFS to search.

Time Comlexity:
addWord -- O(N), N = len(word)
searchWord -- O(26^N), N = len(word)
"""
class TrieNode:
    def __init__(self, char):
        self.val = char
        self.next = {}
        self.end = False

class WordDictionary:

    def __init__(self):
        """
        Initialize your data structure here.
        """
        self.root = TrieNode(None)

    def addWord(self, word: str) -> None:
        """
        Adds a word into the data structure.
        """
        curr = self.root
        for ch in word:
            if ch in curr.next:
                curr = curr.next[ch]
            else:
                curr.next[ch] = TrieNode(ch)
                curr = curr.next[ch]
        curr.end = True

    def search(self, word: str) -> bool:
        """
        Returns if the word is in the data structure. A word could contain the dot character '.' to represent any one letter.
        """
        def dfs(root, word):
            if len(word) == 1:
                if word == '.':
                    if any([root.next[x].end for x in root.next]):
                        return True
                    else:
                        return False
                else:
                    if word in root.next and root.next[word].end:
                        return True
                    else:
                        return False
            else:
                for ch in word:
                    if ch != '.':
                        if ch not in root.next:
                            return False
                        else:
                            return dfs(root.next[ch], word[1:])
                    else:
                        if any([dfs(root.next[x], word[1:]) for x in root.next]):
                            return True
                        else:
                            return False
        return dfs(self.root, word)

# Your WordDictionary object will be instantiated and called as such:
# obj = WordDictionary()
# obj.addWord(word)
# param_2 = obj.search(word)


import numpy as np
import matplotlib.pyplot as plt
import random
import os
from scipy.optimize import curve_fit
from scipy import special

def gaussian(x, mean, amplitude, standard_deviation):
    return amplitude * np.exp( - ((x - mean)**2 / (2*standard_deviation ** 2)))

def make_graphs(tracks, output_dir, title):
    vals, bin_edges = np.histogram(tracks, 100,(1,5))
    #print(vals)
    bin_middles = 0.5*(bin_edges[1:] + bin_edges[:-1])
    plt.errorbar(bin_middles, vals, np.sqrt(vals),  4.0/20.0,fmt='o',color='g')
    plt.title('Hist')
    plt.xlabel('randoms values')
    plt.savefig(f"{output_dir}/{title}.png")
    plt.clf()
def integral_approximation(f, a, b):
    return (b-a)*np.mean(f)

if __name__ == '__main__':

    output_dir = "hist_ana_20210510"
    os.makedirs(f'{output_dir}', exist_ok=True)

    list_cordinate_x = np.random.uniform(0,5,30000)
    list_cordinate_x.sort()
    print('list_cordinate_x',list_cordinate_x)
    print('len of list_cordinate_x is: ',len(list_cordinate_x))

    make_graphs(list_cordinate_x, output_dir, "hist_ran_numbers_01")
    nums = []
    mu = 0
    sigma = 0.2

    for i in range(30000):
        temp = random.gauss(mu, sigma)
        nums.append(temp)

    print('nums',nums)
    print('len of num: ',len(nums))
    list_cordinate_x_err = []
    for p,m in zip(list_cordinate_x,nums):
        list_cordinate_x_er = p + m
        list_cordinate_x_err.append(list_cordinate_x_er)
    print('list_cordinate_x_err',list_cordinate_x_err)
    print('len of list_cordinate_x_err: ',len(list_cordinate_x_err))
    make_graphs(list_cordinate_x_err, output_dir, "hist_ran_numbers_with_err_01")
    bin_heights, bin_edges = np.histogram(list_cordinate_x_err, 100,(0,5))
    #print(vals)
    bin_centers = 0.5*(bin_edges[1:] + bin_edges[:-1])
    plt.errorbar(bin_centers, bin_heights, np.sqrt(bin_heights),  4.0/20.0,fmt='o', ecolor='g', color='blue')

    popt, _ = curve_fit(gaussian, bin_centers, bin_heights, p0=[1., 1., 1.])
    print(popt)
    x_interval_for_fit = np.linspace(bin_edges[0], bin_edges[-1], 30000)
    plt.plot(x_interval_for_fit, gaussian(x_interval_for_fit, *popt), color='r', label='fit: mean=%5.3f, amp=%5.3f, std=%5.3f' % tuple(popt))
    plt.legend()
    plt.title('fitting')
    plt.xlim(-1,5.5)
    plt.xlabel('data_ran_num')
    plt.ylabel('Counts')
    plt.savefig(f"{output_dir}/fitting_ran_data.png")
    plt.show()
    plt.clf()

    plt.plot(x_interval_for_fit, gaussian(x_interval_for_fit, *popt), color='r', label='fit: mean=%5.3f, amp=%5.3f, std=%5.3f' % tuple(popt))
    plt.legend()
    plt.title('fitting')
    plt.xlim(-0.1,2)
    plt.xlabel('data_ran_num')
    plt.ylabel('Counts')
    plt.savefig(f"{output_dir}/fitting_ran_data_sxlim.png")
    plt.show()
    plt.clf()

    # plt.plot(list_cordinate_x_err, special.erf(list_cordinate_x_err))
    # plt.text(1.1,0,f"sigma = {np.std(special.erf(list_cordinate_x_err)):2.2f}", size=10)
    # print('special.erf(list_cordinate_x_err)',np.std(special.erf(list_cordinate_x_err)))
    # plt.xlabel('list_cordinate_vals')
    # plt.ylabel('$erf(x)$')
    # plt.xlim(-1,6)
    # plt.savefig(f"{output_dir}/error_function.png")
    # plt.show()
    # plt.clf()

    # mean = 0
    # standard_deviation = 1
    # a = 0
    # b = 1
    # fx = gaussian(x_interval_for_fit, mu, 30000, sigma)
    # approx = integral_approximation(fx,a,b)
    # print('approx',approx)


def read():
    try:
        with open("Demonetization.txt","r") as dm:
            para = dm.readlines()
            for p in para:
                print(p)
    except(FileNotFoundError):
        print("File has not been created...\n")

def write():
    try:
        with open("Demonetization.txt",'x') as dm:
            string = input("Demonetization : ")
            dm.writelines(string)
    except(FileExistsError):
        print("File already exists...\n")

while True:
    choice = input("Enter a.create and write a file\nb.display contents of file\nc.Exit\n")
    if choice == 'c':
        exit(0)
    elif choice == 'a':
        write()
    else:
        read()


from django.contrib import admin
from django.utils import timezone
from django.urls import path
from django.shortcuts import render
from django import forms
from django.core.validators import URLValidator
from django.core.exceptions import ValidationError
from django.contrib import messages
from rest_framework import generics, permissions, serializers
from rest_framework.reverse import reverse

from collectionjson import services

from .models import PluginMeta, Plugin, ComputeResource
from .serializers import ComputeResourceSerializer, PluginSerializer
from .services.manager import PluginManager

plugin_readonly_fields = [fld.name for fld in Plugin._meta.fields if
                          fld.name != 'compute_resources']

class ComputeResourceAdmin(admin.ModelAdmin):
    readonly_fields = ['creation_date', 'modification_date']
    list_display = ('name', 'compute_url', 'description', 'id')
    list_filter = ['name', 'creation_date', 'modification_date']
    search_fields = ['name', 'description']

    def add_view(self, request, form_url='', extra_context=None):
        """
        Overriden to only show the required fields in the add compute resource page.
        """
        self.fields = ['name', 'compute_url', 'description']
        return admin.ModelAdmin.add_view(self, request, form_url, extra_context)

    def change_view(self, request, object_id, form_url='', extra_context=None):
        """
        Overriden to show all compute resources's fields in the compute resource page.
        """
        self.fields = ['name', 'compute_url', 'description', 'creation_date',
                       'modification_date']
        return admin.ModelAdmin.change_view(self, request, object_id, form_url,
                                            extra_context)

    def save_model(self, request, obj, form, change):
        """
        Overriden to set the modification date.
        """
        if change:
            obj.modification_date = timezone.now()
        super().save_model(request, obj, form, change)

    def delete_model(self, request, obj):
        """
        Overriden to prevent deleting compute resources that would leave orphan
        plugins after the operation.
        """
        try:
            super().delete_model(request, obj)
        except ValidationError as e:
            messages.set_level(request, messages.ERROR)
            messages.error(request, e.message)

    def delete_queryset(self, request, queryset):
        """
        Overriden to prevent deleting compute resources that would leave orphan
        plugins after the operation. This customizes the deletion process for the
        “delete selected objects” action.
        """
        for compute_resource in queryset:
            plg_ids = compute_resource.get_plugins_with_self_as_single_compute_resource()
            if plg_ids:
                messages.set_level(request, messages.ERROR)
                msg = "Can not delete compute resource '%s'. Please do not check it " \
                      "for the delete or otherwise first register its associated " \
                      "plugins with IDs %s with another compute resource."
                messages.error(request, msg % (compute_resource, plg_ids))
                return None
        super().delete_queryset(request, queryset)

class PluginMetaAdmin(admin.ModelAdmin):
    readonly_fields = ['name', 'title', 'stars', 'public_repo', 'license', 'type', 'icon',
                       'category', 'authors', 'documentation', 'creation_date',
                       'modification_date']
    list_display = ('name', 'type', 'id')
    list_filter = ['type', 'creation_date', 'modification_date', 'category']
    search_fields = ['name']

    def has_change_permission(self, request, obj=None):
        """
        Overriden to disable the editing of fields in the view plugin meta page.
        """
        return False

    def has_add_permission(self, request):
        """
        Overriden to remove the add plugin meta button.
        """
        return False

class UploadFileForm(forms.Form):
    file = forms.FileField()

class PluginAdminForm(forms.ModelForm):
    name = forms.CharField(max_length=100, required=False)
    version = forms.CharField(max_length=10, required=False)
    url = forms.URLField(max_length=300, required=False)

    def clean(self):
        """
        Overriden to validate the full set of plugin descriptors and save the newly
        created plugin to the DB.
        """
        if self.instance.pk is None:  # create plugin operation
            pl_manager = PluginManager()
            compute_resources = self.cleaned_data.get('compute_resources')
            if compute_resources is None:
                raise forms.ValidationError('Please choose a compute resource')
            url = self.cleaned_data.pop('url', None)
            cr = compute_resources[0]
            if url:
                try:
                    self.instance = pl_manager.register_plugin_by_url(url, cr.name)
                except Exception as e:
                    raise forms.ValidationError(e)
            else:
                name = self.cleaned_data.get('name')
                if not name:
                    raise forms.ValidationError("A plugin's name or url is required")
                # get user-provided version (can be blank)
                version = self.cleaned_data.get('version')
                try:
                    self.instance = pl_manager.register_plugin(name, version, cr.name)
                except Exception as e:
                    raise forms.ValidationError(e)
            # reset form validated data
            self.cleaned_data['name'] = self.instance.meta.name
            self.cleaned_data['version'] = self.instance.version
            current_compute_resources = list(self.instance.compute_resources.all())
            compute_resources = list(compute_resources) + current_compute_resources
            self.instance.compute_resources.set(compute_resources)
            self.cleaned_data['compute_resources'] = self.instance.compute_resources.all()

class PluginAdmin(admin.ModelAdmin):
    form = PluginAdminForm
    list_display = ('meta', 'version', 'id')
    search_fields = ['meta', 'version']
    list_filter = ['meta', 'creation_date']
    change_form_template = 'admin/plugins/plugin/change_form.html'
    change_list_template = 'admin/plugins/plugin/change_list.html'

    def add_view(self, request, form_url='', extra_context=None):
        """
        Overriden to only show the required fields in the add plugin page.
        """
        self.readonly_fields = []
        self.fieldsets = [
            ('Compute resources', {'fields': ['compute_resources']}),
            ('Identify plugin by name and version', {'fields': [('name', 'version')]}),
            ('Or identify plugin by url', {'fields': ['url']}),
        ]
        return admin.ModelAdmin.add_view(self, request, form_url, extra_context)

    def change_view(self, request, object_id, form_url='', extra_context=None):
        """
        Overriden to show all plugin's fields in the view plugin page.
        """
        self.readonly_fields = [fl for fl in plugin_readonly_fields]
        self.readonly_fields.append('get_registered_compute_resources')
        self.fieldsets = [
            ('Compute resources', {'fields': ['compute_resources',
                                              'get_registered_compute_resources']}),
            ('Plugin properties', {'fields': plugin_readonly_fields}),
        ]
        return admin.ModelAdmin.change_view(self, request, object_id, form_url,
                                            extra_context)

    # def save_model(self, request, obj, form, change):
    #     """
    #     Overriden to set the modification date..
    #     """
    #     if change:
    #         obj.modification_date = timezone.now()
    #     super().save_model(request, obj, form, change)

    def get_urls(self):
        urls = super(PluginAdmin, self).get_urls()
        custom_urls = [
            path('add_plugins/',
                 self.admin_site.admin_view(self.add_plugins_from_file_view),
                 name="add_plugins"),
        ]
        return custom_urls + urls

    def delete_model(self, request, obj):
        """
        Overriden to delete the associated meta if this is the last associated plugin.
        """
        meta = obj.meta
        super().delete_model(request, obj)
        if meta.plugins.count() == 0:
            meta.delete()  # delete the meta if this is the last associated plugin

    def delete_queryset(self, request, queryset):
        """
        Overriden to delete plugin metas if their last associated plugin is deleted.
        This customizes the deletion process for the “delete selected objects” action.
        """
        pl_metas = []
        for plugin in queryset:
            if plugin.meta.plugins.count() == 1:
                pl_metas.append(plugin.meta)
        super().delete_queryset(request, queryset)
        for meta in pl_metas:
            meta.delete()

    def add_plugins_from_file_view(self, request):
        """
        Custom view to handle the add plugins from file page.
        """
        # custom view should return an HttpResponse
        context = dict(
            # Include common variables for rendering the admin template.
            self.admin_site.each_context(request),
            # Anything else you want in the context...
            opts=self.opts,
        )
        if request.method == 'POST':
            file_form = UploadFileForm(request.POST, request.FILES)
            if file_form.is_valid():
                # handle uploaded file (request.FILES['file'])
                summary = self.register_plugins_from_file(request.FILES['file'])
                context['summary'] = summary
                return render(request,
                              'admin/plugins/plugin/add_plugins_from_file_result.html',
                              context)
        else:
            file_form = UploadFileForm()
        context['file_form'] = file_form
        return render(request,
                      'admin/plugins/plugin/add_plugins_from_file.html',
                      context)

    def register_plugins_from_file(self, f):
        """
        Custom method to register plugins from a text file f. The first string of each
        line of the file is the plugin's name or url, the second string is the version
        (can be empty) and the third string is the associated compute environment. Any
        plugin in the file must already be previously uploaded to the ChRIS store.
        """
        summary = {'success': [], 'error': []}
        val = URLValidator()
        pl_manager = PluginManager()
        for line in f:
            try:  # check file must be a text file
                strings = line.decode().strip().split()
            except UnicodeDecodeError:
                summary = {'success': [], 'error': []}
                break
            if len(strings) == 1:
                summary['error'].append({'plugin_name': strings[0],
                                         'code': 'Missing compute resource identifier.'})
            elif len(strings) > 1:
                try:
                    val(strings[0])  # check whether the first string is a url
                except ValidationError:
                    # register by name
                    plg_name = strings[0]
                    if len(strings) == 2:
                        plg_version = None
                        compute_resource = strings[1]
                    else:
                        plg_version = strings[1]
                        compute_resource = strings[2]
                    try:
                        pl_manager.register_plugin(plg_name, plg_version,
                                                   compute_resource)
                    except Exception as e:
                        summary['error'].append({'plugin_name': plg_name, 'code': str(e)})
                    else:
                        summary['success'].append({'plugin_name': plg_name})
                else:
                    # register by url
                    plg_url = strings[0]
                    compute_resource = strings[1]
                    try:
                        pl_manager.register_plugin_by_url(plg_url, compute_resource)
                    except Exception as e:
                        summary['error'].append({'plugin_name': plg_url, 'code': str(e)})
                    else:
                        summary['success'].append({'plugin_name': plg_url})
        return summary

class ComputeResourceAdminList(generics.ListCreateAPIView):
    """
    A JSON view for the collection of compute resources that can be used by ChRIS admins
    to add a new compute resource through a REST API (alternative to the HTML-based admin
    site).
    """
    http_method_names = ['get', 'post']
    serializer_class = ComputeResourceSerializer
    queryset = ComputeResource.objects.all()
    permission_classes = (permissions.IsAdminUser,)

    def list(self, request, *args, **kwargs):
        """
        Overriden to append a collection+json template to the response.
        """
        response = super(ComputeResourceAdminList, self).list(request, *args, **kwargs)
        # append write template
        template_data = {'name': '', 'compute_url': '', 'description': ''}
        return services.append_collection_template(response, template_data)

class PluginAdminSerializer(PluginSerializer):
    """
    A Plugin serializer for the PluginAdminList JSON view.
    """
    plugin_store_url = serializers.URLField(write_only=True)
    compute_name = serializers.CharField(max_length=100, write_only=True)
    version = serializers.CharField(required=False)
    dock_image = serializers.CharField(required=False)
    execshell = serializers.CharField(required=False)
    selfpath = serializers.CharField(required=False)
    selfexec = serializers.CharField(required=False)

    class Meta(PluginSerializer.Meta):
        fields = PluginSerializer.Meta.fields + ('plugin_store_url', 'compute_name')

    def validate(self, data):
        """
        Overriden to validate and register a plugin from a ChRIS store with a compute
        resource associated with this ChRIS instance.
        """
        # remove write_only fields that not part of the Plugin model
        cr_name = data.pop('compute_name')
        plugin_store_url = data.pop('plugin_store_url')
        pl_manager = PluginManager()
        try:
            self.instance = pl_manager.register_plugin_by_url(plugin_store_url, cr_name)
        except Exception as e:
            raise serializers.ValidationError({'non_field_errors': [str(e)]})
        return data

class PluginAdminList(generics.ListCreateAPIView):
    """
    A JSON view for the collection of plugins that can be used by ChRIS admins to
    register plugins through a REST API (alternative to the HTML-based admin site).
    """
    http_method_names = ['get', 'post']
    serializer_class = PluginAdminSerializer
    queryset = Plugin.objects.all()
    permission_classes = (permissions.IsAdminUser,)

    def list(self, request, *args, **kwargs):
        """
        Overriden to append document-level link relations and a collection+json template
        to the response.
        """
        response = super(PluginAdminList, self).list(request, *args, **kwargs)
        # append document-level link relations
        links = {'compute_resources': reverse('admin-computeresource-list',
                                              request=request)}
        response = services.append_collection_links(response, links)
        # append write template
        template_data = {'plugin_store_url': '', 'compute_name': ''}
        return services.append_collection_template(response, template_data)

admin.site.register(ComputeResource, ComputeResourceAdmin)
admin.site.register(PluginMeta, PluginMetaAdmin)
admin.site.register(Plugin, PluginAdmin)


"""
SQLite was free, local, and didn't require me to setup anything.  Nice
way to quickly store the data and get going.
"""

import sqlite3
import csv

# wd = Working Directory, where is the base path (data should be in subdir here)
wd = 'ENTER PATH HERE'

def create_table(cur):
    '''create the membership table'''

    cur.execute('DROP TABLE IF EXISTS membership')

    query = ('CREATE TABLE membership ( '
                'SCHOOL_YEAR TEXT, '
                'LEVEL_CODE TEXT, '
                'DIV_NUM INTEGER, '
                'DIV_NAME TEXT, '
                'SCH_NUM INTEGER, '
                'SCH_NAME TEXT, '
                'GRADE_CODE TEXT, '
                'FEDERAL_RACE_CODE INTEGER, '
                'GENDER TEXT, '
                'DISABILITY_FLAG TEXT, '
                'LEP_FLAG TEXT, '
                'DISADVANTAGED_FLAG TEXT, '
                'FALL_MEMBERSHIP_CNT INTEGER)')

    cur.execute(query)

def load_data(con, cur):
    '''inserts the csv rows into the database '''

    # fall_membership files
    for i in range(2006, 2011):
        insert(wd + 'data/' + str(i) + '_fall_membership.csv', cur)
        con.commit()

    # fm_divsch files
    for i in range(2011, 2014):
        insert(wd + 'data/' + str(i) + '_fm_divsch_1_69.csv', cur)
        con.commit()

        insert(wd + 'data/' + str(i) + '_fm_divsch_70_207.csv', cur)
        con.commit()

    # fm_state files
    for i in range(2011, 2014):
        insert(wd + 'data/' + str(i) + '_fm_state_div.csv', cur)
        con.commit()

def insert(file, cur):
    '''wrapper for the actual insert into the database'''

    with open(file, 'r') as f:
        reader = csv.reader(f)
        columns = next(reader)

        if(columns[0] == 'BEG_SCH_YR'):
            columns[0] = 'SCHOOL_YEAR'

        query = 'INSERT INTO membership ' + '({0}) values ({1})'
        query = query.format(','.join(columns), ','.join('?' * len(columns)))

        # fix the school names
        # Bailey's Crossroads Elementary == Bailey's Crossroads Elementary School
        for data in reader:
            # if(data[5][:-6] == 'School'):
            #     data[5] = data[5][:-7]

            cur.execute(query, data)

def check_data(cur):
    '''run query to print out counts for years for double checking'''

    query = 'select SCHOOL_YEAR, count(FALL_MEMBERSHIP_CNT) AS FALL_MEMBERSHIP \
        from membership GROUP BY SCHOOL_YEAR'
    cur.execute(query)

    total_rows = 0

    for row in cur.fetchall():
        print(row)
        total_rows += row[1]

    print('Total: ' + str(total_rows))

con = sqlite3.connect(wd + 'doe.db')
con.text_factory = str
cur = con.cursor()

create_table(cur)
load_data(con, cur)
check_data(cur)

con.commit()
con.close()


from rest_framework.generics import RetrieveAPIView, ListCreateAPIView, DestroyAPIView
from rest_framework.views import APIView
from rest_framework.renderers import JSONRenderer
from .serializers import CustomerAddressSerializer, CustomerSerializer, CustomerTableSerializer
from customers.models import Customer
from rest_framework.response import Response
from rest_framework.permissions import (
    IsAuthenticated,
    IsAdminUser,
)

class CustomerAddress(RetrieveAPIView):
    queryset = Customer.objects.all()
    serializer_class = CustomerAddressSerializer
    lookup_fields = [
        'address',
        'city',
        'state',
        'zip',
    ]

class CustomerAPIView(ListCreateAPIView):
    queryset = Customer.objects.all()
    serializer_class = CustomerSerializer

class CustomerTableView(ListCreateAPIView):
    queryset = Customer.objects.all()
    serializer_class = CustomerTableSerializer

class CustomerColumns(APIView):
    renderer_classes = (JSONRenderer,)

    def get(self, request, format=None):
        content = [
            {"name": "id", "title": "Customer ID", "breakpoints": "xs sm"},
            {"name": "full_name", "title": "Name"},
            {"name": "phone", "title": "Phone"},
            {"name": "email", "title": "Email", "breakpoints": "sm xs"},
            {"name": "actions", "title": "Actions", "type": "html", "sortable": False, "breakpoints": "sm xs"},
        ]
        return Response(content)

class CustomerDelete(DestroyAPIView):
    """
    Delete job
    """
    permission_classes = [
        IsAuthenticated,
        IsAdminUser,
    ]
    queryset = Customer.objects.all()


from isochrones.grid import ModelGrid
from isochrones.isochrone import Isochrone
import os, re, glob
import numpy as np
import pandas as pd
from isochrones.config import ISOCHRONES

class YAPSIModelGrid(ModelGrid):

    datadir = os.path.expanduser('~/yapsi')
    default_bands = list('UBVRIJHK')
    default_kwargs = {'Y':0.28}

    @classmethod
    def get_band(cls, b):
        if b in 'UBVRIJHK':
            return 'w', b
        else:
            raise ValueError('{0} not in YAPSI grids.'.format(b))

    @classmethod
    def _get_XYZ(cls, filename):
        m = re.search('X(\dp\d+)_Z(\dp\d+)\.dat', filename)
        if m:
            X = float(m.group(1).replace('p', '.'))
            Z = float(m.group(2).replace('p', '.'))
            Y = 1 - X - Z
            return X, Y, Z
        else:
            raise ValueError('Cannot parse XYZ from filename: {}'.format(filename))

    def get_filenames(self, phot='w', Y=0.28):
        all_files = glob.glob('{0}/yapsi_{1}_*.dat'.format(self.datadir, phot))
        files = []
        for f in all_files:
            x, y, Z = self._get_XYZ(f)
            if np.isclose(Y, y):
                files.append(f)

        return files

    @classmethod
    def get_feh(cls, filename):
        """
        example filename: yapsi_w_X0p602357_Z0p027643.dat
        """
        X,Y,Z = cls._get_XYZ(filename)

        Xsun = 0.703812
        Zsun = 0.016188

        return np.log10((Z/X) / (Zsun/Xsun))

    @classmethod
    def to_df(cls, filename):

        df = pd.read_table(filename, comment='#',
             names=['age','mass','logTeff','logL','logg','V','UmB','BmV','VmR','VmI','JmK','HmK','VmK'],
             delim_whitespace=True)

        df['age'] = np.log10(df['age']*1e9)
        df['feh'] = cls.get_feh(filename)
        df['B'] = df['BmV'] + df['V']
        df['U'] = df['UmB'] + df['B']
        df['R'] = df['V'] - df['VmR']
        df['I'] = df['V'] - df['VmI']
        df['K'] = df['V'] - df['VmK']
        df['H'] = df['K'] - df['HmK']
        df['J'] = df['K'] - df['JmK']

        df = df.drop(['UmB','BmV','VmR','VmI','JmK','HmK','VmK'], axis=1)
        return df

    def _get_df(self):
        return self.df_all('w')


from opentrons import containers, instruments

metadata = {
    'protocolName': 'Multi-only',
    'author': 'Opentrons <engineering@opentrons.com>',
    'description': 'A protocol that only uses an 8-channel pipette',
    'source': 'Opentrons Repository'
}

tiprack = containers.load('tiprack-200ul', '8')
trough = containers.load('trough-12row', '9')

plate = containers.load('96-flat', '5')

multi = instruments.Pipette(
    mount='left',
    min_volume=10,
    name="p200S",
    tip_racks=[tiprack],
    channels=8)

for tips in [tiprack.rows(0), tiprack.rows[-1]]:
    multi.pick_up_tip(tips)
    multi.aspirate(100, trough.rows[0])
    multi.aspirate(100, trough.rows[-1])

    multi.dispense(100, plate.rows[0])
    multi.dispense(100, plate.rows[-1])
    multi.drop_tip(tips)


import pymongo
import scrapy
from scrapy import Request
from itemadapter import ItemAdapter

from pymongo import MongoClient
from pprint import pprint
from flask import Flask

class FlashbotSpider(scrapy.Spider):
    name = 'flashbot'
    allowed_domains = ['rss.jobsearch.monster.com']
    client = MongoClient()
    db = client.mydb
    my_collection = db.my_collection
    # Start the crawler at this URLs
    #start_urls = ['file://rss.jobsearch.monster.com/rssquery.ashx?q={query}']
    start_urls = ['http://rss.jobsearch.monster.com/rssquery.ashx?q={query}']

    thesaurus = ["machine learning", "machine", "learning", "big data", "big", "data"]

    LOG_LEVEL = "INFO"

    def parse(self, response):

        # We stat with this url
        url = self.start_urls[0]

        # Build and send a request for each word of the thesaurus
        for query in self.thesaurus:
            target = url.format(query=query)
            print("fetching the URL: %s" % target)
            if target.startswith("file://"):
                r = Request(target, callback=self.scrapit, dont_filter=True)
            else:
                r = Request(target, callback=self.scrapit)
            r.meta['query'] = query
            yield r

    def scrapit(self, response):
        query = response.meta["query"]

        # Base item with query used to this response
        print(query, response)

        # Scrap the data
        for doc in response.xpath("//item"):
            item = {"query": query}
            item["title"] = doc.xpath("title/text()").extract()
            item["description"] = doc.xpath("description/text()").extract()
            item["link"] = doc.xpath("link/text()").extract()
            item["pubDate"] = doc.xpath("pubDate/text()").extract()
            item["guid"] = doc.xpath("guid/text()").extract()

            guid = item ["guid"] [0]
            rest= self.my_collection.find({"guid":guid})
            if rest.count() ==0:
                self.my_collection.insert_one(item)
                print("document inserted", item ["title"])
                print("item scraped:", item["title"])
            yield item
         #to download data: scrapy crawl flashbot -o toto.xml

#flask:
app = Flask(__name__)
@app.route("/")
def root():
    return "Hello world"


def load_parse_frequency_list():
    changes = []
    with open('day1-input.txt', 'r') as f:
        for line in f:
            change = int(line[1:])
            if line.startswith('+'):
                changes.append(change)
            else:
                changes.append(-change)
    return changes

def part1():
    frequency = 0
    frequency_changes = load_parse_frequency_list()
    for change in frequency_changes:
        frequency += change
    return frequency

def part2():
    frequency = 0
    frequencies_found = {}
    frequency_changes = load_parse_frequency_list()
    i = 0
    while True:
        frequency += frequency_changes[i]
        if frequency in frequencies_found:
            return frequency
        else:
            frequencies_found[frequency] = True
        i += 1
        if i == len(frequency_changes):
            # Start over at 1st frequency change
            i = 0


# 0 또는 양의 정수가 주어졌을 때, 정수를 이어 붙여 만들 수 있는 가장 큰 수를 알아내 주세요.

# 예를 들어, 주어진 정수가 [6, 10, 2]라면 [6102, 6210, 1062, 1026, 2610, 2106]를 만들 수 있고, 이중 가장 큰 수는 6210입니다.

# 0 또는 양의 정수가 담긴 배열 numbers가 매개변수로 주어질 때, 순서를 재배치하여 만들 수 있는 가장 큰 수를 문자열로 바꾸어 return 하도록 solution 함수를 작성해주세요.

# 제한 사항
# numbers의 길이는 1 이상 100,000 이하입니다.
# numbers의 원소는 0 이상 1,000 이하입니다.
# 정답이 너무 클 수 있으니 문자열로 바꾸어 return 합니다.
from operator import itemgetter
def solution(numbers):
    n = len(numbers)
    a = list(map(lambda x: " ".join(x).split(), list(map(str, numbers))))
    a.sort(reverse=True)
    print(a)

# JichangJang님의 코드를 보며 공부하였습니다.

def compare_value(first, second):
    if len(str(first)) == len(str(second)):
        if first > second:
            return first
        else:
            return second
    else:
        if int(str(first) + str(second)) >= int(str(second) + str(first)):
            return first
        return second

def my_quick_sort(numbers):
    if len(numbers) <= 1:
        return numbers
    else:
        less = []
        same = []
        bigger = []
        for i in numbers:
            if i == numbers[0]:
                same.append(i)
            elif compare_value(i, numbers[0]) == numbers[0]:
                bigger.append(i)
            else:
                less.append(i)
        return my_quick_sort(less) + same + my_quick_sort(bigger)

def mid_solution(numbers):
    for j in numbers:
        if j != 0:
            answer = my_quick_sort(numbers)
            str_answer = ""
            for i in answer:
                str_answer = str_answer + str(i)
            return str_answer
        else:
            pass
    return "0"

def best_solution(numbers):
    numbers = list(map(str, numbers))
    numbers.sort(key=lambda x:x*3, reverse =True)
    print(numbers)
    return str(int(''.join(numbers)))

numbers = [3, 30, 34, 5, 9]
result = best_solution(numbers)
print(result)


import gdal
from gdalconst import *
import numpy as np
from multiprocessing.pool import Pool, ThreadPool
from functools import partial

def getData(band, cols, rows):
    return band.ReadAsArray(0,0,cols,rows).astype(np.float32)

def calcNDVI(inFile, outFile, bandRed, bandNir):
    """Calculate the NDVI index for the input file and write a new output file

    :param inFile: raster image file to process
    :type inFile: str
    :value inFile: path and name of the raster image file

    :param outFile: raster image output file
    :type outFile: str
    :value outFile: path and name of the output raster image file

    :param bandRed: number of the red raster band
    :type bandRed: int

    :param bandNir: number of the near infrared raster band
    :type bandNir: int
    """

    driver = gdal.AllRegister()
    img = gdal.Open(inFile, GA_ReadOnly)
    cols, rows = img.RasterXSize, img.RasterYSize

    band_Red = img.GetRasterBand(bandRed)
    band_Nir = img.GetRasterBand(bandNir)

    bands_Red_Nir = [band_Red, band_Nir]

    p = Pool(4)
    mapfunc_getData = partial(getData, cols = cols, rows = rows)
    data_bands_Red_Nir = p.map(mapfunc_getData, bands_Red_Nir)

    data_NDVI = (data_bands_Red_Nir[1] - data_bands_Red_Nir[0]) / (data_bands_Red_Nir[1] + data_bands_Red_Nir[0])

    bands_Red_Nir = None
    data_bands_Red_Nir = None

    driverout = gdal.GetDriverByName("GTiff")
    dst_ds = driverout.Create(outFile,cols, rows, 1, band_Red.DataType)
    dst_ds.SetGeoTransform(img.GetGeoTransform())
    dst_ds.SetProjection(img.GetProjection())
    outBand = dst_ds.GetRasterBand(1)

    outBand.WriteArray(data_NDVI, 0, 0)
    outBand.SetNoDataValue(-9999)

    data_NDVI = None
    dst_ds = None
    outBand = None

def readRaster(bandNr, inFile, dataType):
    """ Return the data of a raster image file

    :param inFile: raster image file to process
    :type inFile: str
    :value inFile: path and name of the raster image file

    :param bandNr: number of the band of the raster image file
    :type bandNr: int

    :param dataType: defines the data type of the returned data
    :type dataType: int
    :values dataType:
        1 - np.float16 (Half precision float: sign bit, 5 bits exponent, 10 bits mantissa)
        2 - np.float32 (Single precision float: sign bit, 8 bits exponent, 23 bits mantissa),
        3 - np.float64 (Double precision float: sign bit, 11 bits exponent, 52 bits mantissa),
        4 - np.int8 (Byte (-128 to 127)),
        5 - np.int16 (Integer (-32768 to 32767)),
        6 - np.int32 (Integer (-2147483648 to 2147483647)),
        7 - np.int64 (Integer (-9223372036854775808 to 9223372036854775807)),
        else - np.float64 (Double precision float: sign bit, 11 bits exponent, 52 bits mantissa)

    :further readings: http://docs.scipy.org/doc/numpy-1.10.1/user/basics.types.html"""

    driver = gdal.AllRegister()
    img = gdal.Open(inFile, GA_ReadOnly)
    cols, rows = img.RasterXSize, img.RasterYSize
    band = img.GetRasterBand(bandNr)

    if dataType == 1:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.float16)
    elif dataType == 2:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.float32)
    elif dataType == 3:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.float64)
    elif dataType == 4:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.int8)
    elif dataType == 5:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.int16)
    elif dataType == 6:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.int32)
    elif dataType == 7:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.int64)
    else:
        dataBand = band.ReadAsArray(0,0,cols,rows).astype(np.float64)

    return dataBand


import numpy as np

import matplotlib.pyplot as plt
import keras, time, os
from keras.layers import (Dense, Dropout, Activation, Flatten, Conv2D,
                          AveragePooling2D, MaxPooling2D, BatchNormalization)
import sys
logistic = __import__('logistic')
circular = __import__('fc2_circular_reg')

if __name__ == "__main__":
    ttt = time.time()

    path_file    = 'data/'
    path_results = 'results/'

    (isPU_train, isPU_val, _,
     image_train, image_val, _,
     rwi_train, rwi_val, _,
     rew_train, rew_val, _,
     Rpt_train, Rpt_val, _,
     j0pt_train, j0pt_val, _) = logistic.load_data(path_file)

    y_train = isPU_train.astype(int)
    y_val   = isPU_val.astype(int)

    X_train = image_train
    X_val   = image_val

    seed = 7
    np.random.seed(seed)
    batch_size = 10000

    model = keras.models.Sequential()
    #numfil = 1000
    regds = 0#1e-8
    #regth = 1e-4
    #regthobj = circular.reg_grad_theta(eta_range=10, phi_range=10,
    #                                   num_filters=numfil,
    #                                   num_channels=3, regth=regth, regl2=regds)

    model.add(Conv2D(filters=50, kernel_size=2, strides=(2, 2),
                     padding='same', data_format='channels_first',
                     input_shape=(3, 10, 10),
                     kernel_regularizer=keras.regularizers.l2(regds)))
    model.add(Activation('relu'))
    model.add(Conv2D(filters=100, kernel_size=5, strides=(1, 1),
                     padding='valid', data_format='channels_first',
                     kernel_regularizer=keras.regularizers.l2(regds)))
    model.add(Activation('relu'))
    model.add(Flatten())
    model.add(Dense(1, kernel_regularizer=keras.regularizers.l2(regds)))
    model.add(Activation('sigmoid'))

    model.compile(loss="binary_crossentropy",
                  optimizer='adam', metrics=['accuracy'])

    #sanity check
    los, acc = logistic.pred(model, batch_size, X_val, y_val, rwi_val, .5, True)

    current_file_name = os.path.basename(sys.argv[0])
    # __file__ is different in python and ipython.. 4152963

    chkpnt_name = path_results + current_file_name + '_' + str(regds) + '_chk_temp'
    chkpnt_ojb = keras.callbacks.ModelCheckpoint(
                     chkpnt_name, monitor='val_loss', verbose=0,
                     save_best_only=True, save_weights_only=False,
                     mode='auto', period=1)
    early_stop = keras.callbacks.EarlyStopping(
                     monitor='val_loss', min_delta=0, patience=50, verbose=1,
                     mode='auto')
    reduce_lr = keras.callbacks.ReduceLROnPlateau(
                     monitor='val_loss', factor=0.2, patience=10, verbose=1,
                     mode='auto', epsilon=1e-9)

    history = model.fit(X_train, y_train, epochs=1000, batch_size=batch_size,
                        verbose=1, sample_weight=rwi_train,
                        validation_data=(X_val, y_val, rwi_val),
                        callbacks=[chkpnt_ojb, early_stop, reduce_lr])

    #keras.regularizers.reg_grad_theta = circular.reg_grad_theta
    model = keras.models.load_model(chkpnt_name)

    los, acc = logistic.pred(model, batch_size, X_val, y_val, rwi_val, .5, True)

    output_name = (current_file_name + '_' + str(regds) + '_' +
                   time.strftime("%Y_%m_%d_%H_%M") + '_' + str(acc))

    model.save(path_results+output_name+'_mod')
    logistic.save_obj(history.history, path_results, output_name+'_his')

    #os.rename(chkpnt_name, path_results + output_name+'_his')
    os.remove(chkpnt_name)

    filters0 = model.layers[0].get_weights()[0]
    logistic.plot_filters(filters0, path_results, output_name+'_0', 10)
    filters2 = model.layers[2].get_weights()[0]
    logistic.plot_filters(filters2, path_results, output_name+'_2', 10)
    logistic.plot_history_loss(history.history, False, path_results, output_name)
    logistic.plot_model(model, path_results, output_name)

    plt.show()

    pred_val = np.squeeze(model.predict(X_val, batch_size=batch_size, verbose=1))
    effPU = logistic.roc(rew_val, isPU_val, True,  pred_val)
    effHS = logistic.roc(rew_val, isPU_val, False, pred_val)
    np.save(path_results + output_name + '_effPU', effPU)
    np.save(path_results + output_name + '_effHS', effHS)
    print(time.time()-ttt)


import ccxt
import datetime
import FinanceDataReader as fdr
import pyupbit
import time
import requests

p_exchange = 1160.33  # Exchange rate initial value
binance = ccxt.binance({
'apiKey': '',
'secret': '',
})  # Binance API
upbit = pyupbit.Upbit('', '')  # Upbit API
myToken = ''  # Slack token
ticker = 'EOS'  # Ticker
count = 230  # Order quantity
mount = 1500000  # Order amount
p_standard = 0.02  # Premium standard
p_gap = 0.59  # Premium gap
p_cnt = 299  # Initial count

def post_message(token, channel, text):
    response = requests.post("https://slack.com/api/chat.postMessage",
    headers={"Authorization": "Bearer "+token},
    data={"channel": channel,"text": text}
        )

def get_exchange():
    try:
        today = str(datetime.date.today())
        today = int(today.replace('-', ''))
        today = str(today)
        exchange = fdr.DataReader('USD/KRW', today).iloc[-1, 0]
    except:
        today = 210817
        exchange = fdr.DataReader('USD/KRW', today).iloc[-1, 0]
    post_message(myToken, "#stock", 'Exchange: ' + str(exchange))
    print('환율: ' + str(exchange))
    return exchange

def binance_balance(tk):
    balance = binance.fetch_balance()  # Binance balance
    b_balance = balance[tk]['free']
    return b_balance

def binance_price(tk, p_exchange):
    b_ticker = binance.fetch_ticker(tk+'/USDT')  # Binance current KRW price
    price = b_ticker['close']
    price2 = price * p_exchange
    # print(tk + ' Binance 현재가: ' + str(price2))
    return price2

def binance_usd_price(tk):
    b_ticker = binance.fetch_ticker(tk+'/USDT')  # Binance current USD price
    price = b_ticker['close']
    return price

def binance_buy(tk, cnt):
    order = binance.create_market_buy_order(tk + '/USDT', cnt)  # Binance market buy by count
    print(order)
    # post_message(myToken, "#stock", order)

def binance_sell(tk, cnt):
    order = binance.create_market_sell_order(tk + '/USDT', cnt)  # Binance market sell by count
    print(order)
    # post_message(myToken, "#stock", print(order))

def upbit_price(tk):
    u_price = pyupbit.get_current_price('KRW-' + tk)  # Upbit current KRW price
    return u_price

def upbit_buy(tk, mnt):  # Upbit market buy by amount
    u_ticker = 'KRW-' + tk
    market_price_buy = upbit.buy_market_order(u_ticker, mnt)
    print(market_price_buy)

def upbit_sell(tk, cnt):  # Upbit market sell by count
    u_ticker = 'KRW-' + tk
    market_price_sell = upbit.sell_market_order(u_ticker, cnt)
    print(market_price_sell)

def get_premium(b_price, u_price, tk):  # Current premium
    premium = 100*(u_price-b_price)/u_price
    return premium

while True:
    try:
        if binance_balance('USDT') > (count*binance_usd_price(ticker)+50) and upbit.get_balance('KRW-' + ticker) > count:  # Premium trading
            if get_premium((binance_usd_price(ticker)*p_exchange), upbit_price(ticker), ticker) > (p_standard+p_gap):  # Above standard
                upbit_sell(ticker, count)
                binance_buy(ticker, count)
                post_message(myToken, "#stock", 'Premium trading: ' + str(round(get_premium((binance_usd_price(ticker)*p_exchange), upbit_price(ticker), ticker), 2)) + ' / ' + str(count) + 'ea')
                if binance_balance('USDT') < (count * binance_usd_price(ticker) + 70):  # Reset
                    p_exchange = get_exchange()
                    p_standard = round(get_premium((binance_usd_price(ticker) * p_exchange), upbit_price(ticker), ticker), 2)
                    p_standard = (p_standard*0.5) + round(get_premium((binance_usd_price(ticker) * p_exchange), upbit_price(ticker), ticker), 2)*0.5
                    post_message(myToken, "#stock", 'Standard: ' + str(p_standard) + ' Reset / Exchange: ' + str(p_exchange))

        if upbit.get_balance('KRW') > mount + 50000 and binance_balance(ticker) > mount/binance_price(ticker, p_exchange):  # Reverse premium trading
            if get_premium((binance_usd_price(ticker)*p_exchange), upbit_price(ticker), ticker) < (p_standard-p_gap):  # Below standard
                count2 = round(mount / (upbit_price(ticker) + 5), 1)
                upbit_buy(ticker, mount)
                binance_sell(ticker, count2)
                post_message(myToken, "#stock", 'R-premium trading: ' + str(round(get_premium((binance_usd_price(ticker)*p_exchange), upbit_price(ticker), ticker), 2)) + ' / ' + str(count2) + 'ea')

        p_cnt = p_cnt + 1

        if p_cnt == 300:
            p_cnt = 0
            p_standard = (p_standard * 0.98) + (get_premium((binance_usd_price(ticker)*p_exchange), upbit_price(ticker), ticker) * 0.02)
            krw_balance = round(upbit.get_balance('KRW') + binance_balance('USDT')*p_exchange)
            ticker_balance = round(upbit.get_balance('KRW-' + ticker) + binance_balance(ticker))
            now_premium = round(get_premium((binance_usd_price(ticker)*p_exchange), upbit_price(ticker), ticker), 2)
            post_message(myToken, "#stock", 'Now: ' + str(now_premium) + ' / Std: ' + str(round(p_standard, 2)) + ' / Exc: ' + str(p_exchange) + '\nCrypto: '+str(ticker_balance) + ' / KRW:  ' + str(krw_balance))
            time.sleep(7)

    except Exception as e:
        post_message(myToken, "#stock", e)
        time.sleep(30)