#!/usr/bin/env python

"""

Usage:

  ./cat2json.py cat

Aim:

Extract positions of XMM sources.

Positions are fixed to 4/5 decinmal places for RA and Dec
to save space (and this should be sub-arcsecond accuracy).

"""

import json
import math

import pycrates

def roundy(v, n):
    """Restrict to n decimal places.

    Up to the vagueris of floating-point representation.
    """

    t = 10**n
    return math.floor(v * t + 0.5) / t

def convert(infile):

    # fname = "{}[2][cols ra=sc_ra,dec=sc_dec]".format(infile)
    fname = "{}[2][cols ra,dec]".format(infile)

    cr = pycrates.read_file(fname)

    cat = cr.name

    srcs = []
    for ra,dec in zip(cr.RA.values,
                      cr.DEC.values):

        # ensure a Python not NumPy string, as want it to serialize
        # to JSON
        #
        # Restrict RA to 4dp and Dec to 5dp to try and save space
        # in the serialization (tihs should be sub-arcsecond
        # accuracy).
        #
        ra = roundy(ra, 4)
        dec = roundy(dec, 5)

        srcs.append([ra, dec])

    return {'catalog': cat, 'sources': srcs}

if __name__ == "__main__":

    import sys
    if len(sys.argv) != 2:
        sys.stderr.write("Usage: {} cat\n".format(sys.argv[0]))
        sys.exit(1)

    out = convert(sys.argv[1])
    print(json.dumps(out))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
get data from aiida pk
"""
import numpy as np
import warnings
from pymatgen.core.structure import Structure
from pymatgen.symmetry.analyzer import SpacegroupAnalyzer
from pymatgen.io.phonopy import get_phonopy_structure
from aiida.orm import load_node, QueryBuilder, Node, WorkChainNode, StructureData
from aiida.common import exceptions
from aiida.common import NotExistentAttributeError
from aiida.cmdline.utils.decorators import with_dbenv
from aiida.plugins import WorkflowFactory
from phonopy import Phonopy
from aiidaplus.utils import get_kpoints

RELAX_WF = WorkflowFactory('vasp.relax')
PHONOPY_WF = WorkflowFactory('phonopy.phonopy')

@with_dbenv()
def get_node_from_pk(pk):
    """
    get node from pk
    """
    return load_node(pk)

def check_process_class(node,
                        expected_process_class:str):
    """
    check process class of node is the same as the expected

    Args:
        node: aiida node
        expected_process_class (str): expected process class

    Raises:
        AssertionError: input node is not the same as expected
    """
    assert node.process_class.get_name() == expected_process_class, \
            "input node: {}, expected: {} (NOT MATCH)". \
            format(node.process_class, expected_process_class)

def get_structure_data_from_pymatgen(pmgstructure:Structure,
                                     symprec:float=1e-5) -> dict:
    """
    get structure data from pymatgen structure

    Raises:
        Warning: final structure do not exist

    Note:
        symprec=1e-5 (default), which is the same as VASP SYMPREC default
    """
    analyzer = SpacegroupAnalyzer(pmgstructure, symprec=symprec)
    dataset = analyzer.get_symmetry_dataset()

    dic = {}
    dic['symprec'] = symprec
    dic['volume'] = pmgstructure.lattice.volume
    dic['lattice'] = pmgstructure.lattice.matrix.tolist()
    dic['lattice_abc'] = list(pmgstructure.lattice.abc)
    dic['lattice_angles'] = list(pmgstructure.lattice.angles)
    dic['international'] = dataset['international']
    dic['pointgroup'] = dataset['pointgroup']
    dic['natoms'] = len(pmgstructure.species)
    dic['wyckoffs'] = dataset['wyckoffs']
    dic['site_symmetry_symbols'] = dataset['site_symmetry_symbols']
    dic['hall'] =dataset['hall']
    dic['hall_number'] = dataset['hall_number']

    return dic

def get_structure_data(pk, symprec=1e-5) -> dict:
    """
    get structure data from pk

    Note:
        symprec=1e-5 (default), which is the same as VASP SYMPREC default
    """
    node = get_node_from_pk(pk)
    pmgstructure = node.get_pymatgen()
    dic = get_structure_data_from_pymatgen(pmgstructure, symprec=symprec)
    dic['pk'] = pk
    dic['data_type'] = 'StructureData'
    return dic

def get_vasp_data(pk, symprec=1e-5) -> dict:
    """
    get vasp data from pk

    Note:
        - symprec=1e-5 (default), which is the same as VASP SYMPREC default
        - Warning: final structure does not exist
    """
    node = get_node_from_pk(pk)
    check_process_class(node, 'VaspWorkChain')
    results = node.outputs.misc.get_dict()

    structure = {}
    initial_structure_pk = node.inputs.structure.pk
    initial_structure = get_structure_data(initial_structure_pk,
                                           symprec=symprec)
    structure['initial'] = initial_structure
    try:
        final_structure_pk = node.outputs.structure.pk
        final_structure = get_structure_data(final_structure_pk,
                                             symprec=symprec)
        structure['final'] = final_structure
    except exceptions.NotExistent:
        warnings.warn("final structure does not exist")

    kpoints = get_kpoints(
            structure=load_node(initial_structure_pk).get_pymatgen_structure(),
            mesh=node.inputs.kpoints.get_kpoints_mesh()[0],
            )
    kpoints['intervals'] = kpoints['intervals'].tolist()

    dic = {}
    dic['data_type'] = node.process_class.get_name()
    dic['pk'] = pk
    dic['incar'] = node.inputs.parameters.get_dict()
    dic['parser_settings'] = node.inputs.settings.get_dict()['parser_settings']
    dic['potential_family'] = node.inputs.potential_family.value
    dic['potential_mapping'] = node.inputs.potential_mapping.get_dict()
    dic['kpoints'] = kpoints
    dic['structure'] = structure
    dic['maximum_force'] = results['maximum_force']
    dic['maximum_stress'] = results['maximum_stress']
    dic['energy_no_entropy'] = results['total_energies']['energy_no_entropy']

    return dic

def get_relax_data(pk, symprec=1e-5) -> dict:
    """
    get relax data from pk

    Note:
        - symprec=1e-5 (default), which is the same as VASP SYMPREC default
        - Warning: final vasp calculation is not static calc
    """
    node = get_node_from_pk(pk)
    check_process_class(node, 'RelaxWorkChain')

    # drop final static calculation
    vasp_pks = [ vasp_node.called[0].pk for vasp_node in node.called ]
    vasp_pks.reverse()
    if 'nsw' not in \
            get_node_from_pk(vasp_pks[-1]).inputs.parameters.get_dict().keys():
        del vasp_pks[-1]
    else:
        warnings.warn("final vasp calculation is not static calc")
    vasp_results = {}
    for i, vasp_pk in enumerate(vasp_pks):
        vasp_results['step_%02d' % i] = get_vasp_data(vasp_pk,
                                                           symprec=symprec)

    dic = {}
    dic['data_type'] = node.process_class.get_name()
    dic['pk'] = pk
    dic['initial_structure_pk'] = node.inputs.structure.pk
    dic['final_structure_pk'] = node.outputs.relax__structure.pk
    dic['final_energy_no_entropy'] = \
        node.outputs.misc.get_dict()['total_energies']['energy_no_entropy']
    dic['steps'] = vasp_results
    try:
        dic['max_force'] = float(np.linalg.norm(node.outputs.forces.get_array('final'), axis=1).max())
    except NotExistentAttributeError:
        print("outputs forces is not exists in relax pk:{}".format(pk))
    try:
        dic['stress'] = node.outputs.stress.get_array('final').tolist()
    except NotExistentAttributeError:
        print("outputs forces is not exists in relax pk:{}".format(pk))

    return dic

def get_phonon_data(pk, get_phonon=False):
    """
    get phonon object from aiida pk
    """
    node = load_node(pk)
    symprec = node.inputs.symmetry_tolerance.value
    dic = {}
    dic['data_type'] = node.process_class.get_name()
    dic['pk'] = pk
    dic['calculator_settings'] = node.inputs.calculator_settings.get_dict()
    dic['symmetry_tolerance'] = symprec
    dic['phonon_setting_info'] = node.outputs.phonon_setting_info.get_dict()
    dic['structure'] = {
            'input': get_structure_data(node.inputs.structure.pk, symprec),
            'primitive': get_structure_data(node.outputs.primitive.pk, symprec),
            'supercell': get_structure_data(node.outputs.supercell.pk, symprec),
            }

    if get_phonon:
        pmgstructure = node.inputs.structure.get_pymatgen()
        unitcell = get_phonopy_structure(pmgstructure)
        phonon_settings = node.outputs.phonon_setting_info.get_dict()
        phonon = Phonopy(unitcell,
                         supercell_matrix=phonon_settings['supercell_matrix'],
                         primitive_matrix=phonon_settings['primitive_matrix'])
        phonon.set_displacement_dataset(phonon_settings['displacement_dataset'])
        phonon.set_forces(node.outputs.force_sets.get_array('force_sets'))
        phonon.produce_force_constants()
        return dic, phonon
    else:
        return dic

def get_shear_data(pk):
    """
    get shear data
    """
    # get called pks
    node = load_node(pk)
    rlx_qb = QueryBuilder()
    rlx_qb.append(Node, filters={'id':{'==': pk}}, tag='wf')
    rlx_qb.append(RELAX_WF, with_incoming='wf', project=['label', 'id'])
    ph_qb = QueryBuilder()
    ph_qb.append(Node, filters={'id':{'==': pk}}, tag='wf')
    ph_qb.append(PHONOPY_WF, with_incoming='wf', project=['label', 'id'])
    relaxes = rlx_qb.all()
    relaxes.sort(key=lambda x: x[0])
    phonons = ph_qb.all()
    phonons.sort(key=lambda x: x[0])

    dic = {}
    dic['pk'] = pk
    dic['calculator_settings'] = node.inputs.calculator_settings.get_dict()
    dic['shear_conf'] = node.inputs.shear_conf.get_dict()
    dic['parent'] = get_structure_data(node.outputs.parent.pk)
    dic['relax_results'] = node.outputs.relax_results.get_dict()
    dic['shear_ratios'] = node.outputs.shear_ratios.get_dict()['shear_ratios']
    dic['gamma'] = node.outputs.gamma.value
    dic['relax_pks'] = np.array(relaxes)[:,1].astype(int).tolist()
    dic['phonon_pks'] = np.array(phonons)[:,1].astype(int).tolist()

    return dic

def get_twinboundary_relax_data(pk):
    """
    get twinboudnary relax data
    """
    # get called pks
    node = load_node(pk)
    rlx_qb = QueryBuilder()
    rlx_qb.append(Node, filters={'id':{'==': pk}}, tag='wf')
    rlx_qb.append(RELAX_WF, with_incoming='wf', project=['label', 'id'])
    relaxes = rlx_qb.all()
    dic = {}
    dic['relax_pks'] = [ relax[1] for relax in relaxes ]
    dic['relax_pks'].sort()

    return dic

# def get_twinboundary_data(pk):
#     """
#     get twinboudnary data
#     """
#     # get called pks
#     node = load_node(pk)
#     qb = QueryBuilder()
#     qb.append(Node, filters={'id':{'==': pk}})
#     qb.append(WorkChainNode, tag='workchain')
#     qb.append(StructureData, with_outgoing='workchain', project=['id', 'label'])
#     structure_pks = qb.all()
#     structure_pks = list(reversed(structure_pks))
#
#     dic = {}
#     dic['twinboudnary_summary'] = node.outputs.twinboundary_summary.get_dict()
#     dic['structure_pks'] = structure_pks
#     dic['vasp_results'] = node.outputs.vasp_results.get_dict()
#
#     return dic


# coding: utf8
from flask import request
from flask import jsonify
from flask import abort
from application import db
from application.models.memo import Memo
from application.models.relation_label_memo import RelationLabelMemo
from . import api

@api.route('/memos', methods=['POST'])
def create_memo():
    request_body_dic = request.get_json()
    selected_label_id = request_body_dic.get('selectedLabelId')

    try:
        # memo = Memo(**request_body_dic)
        memo = Memo()
        memo.update_data(**request_body_dic)
    except TypeError:
        return abort(400)
    else:
        db.session.add(memo)
        db.session.commit()
        return_dic = memo.serialize()
        label_ids = return_dic.setdefault('labelIds', [])
        if selected_label_id:
            relation = RelationLabelMemo(memo_id=memo.id, label_id=selected_label_id)
            db.session.add(relation)
            db.session.commit()
            label_ids.append(relation.label_id)

        return jsonify(data=return_dic), 201

@api.route('/memos', methods=['GET'])
def get_all_memos(memo_id=0):
    q = db.session.query(Memo, RelationLabelMemo) \
        .outerjoin(RelationLabelMemo, Memo.id == RelationLabelMemo.memo_id)

    if memo_id:
        q = q.filter(Memo.id == memo_id);

    result_dict = {}
    for memo, relation in q.all():
        if memo.id not in result_dict:
            memo_dic = result_dict.setdefault(memo.id, memo.serialize())
            memo_dic['labelIds'] = []

        if relation is not None:
            result_dict[memo.id]['labelIds'].append(relation.label_id)

    result_list = result_dict.values()
    result_list.sort(key=(lambda x: (x['updateTime'], x)))
    result_list.reverse()

    if memo_id:
        return jsonify(
            data=result_list[0]
        ), 200

    return jsonify(
        data=result_list
    ), 200

@api.route('/memos/<int:memo_id>', methods=['PUT'])
def update_memo(memo_id):
    memo = db.session.query(Memo).get(memo_id)
    if memo is None:
        return jsonify(
            userMessage="can not found memo"
        ), 404

    request_body_dic = request.get_json()
    memo.update_data(**request_body_dic)
    db.session.commit()
    return get_all_memos(memo_id=memo.id)

@api.route('/memos/<int:memo_id>', methods=['DELETE'])
def delete_memo(memo_id):
    memo = db.session.query(Memo).get(memo_id)
    if memo is None:
        return jsonify(
            userMessage="can not found label"
        ), 404
    db.session.delete(memo)
    db.session.commit()
    return "", 204

@api.route('/memos/delete', methods=['POST'])
def delete_memo_multiple():
    request_body_dic = request.get_json()
    delete_memo_ids = request_body_dic.get('ids', [])

    q = db.session.query(Memo).filter(Memo.id.in_(delete_memo_ids))
    map(db.session.delete, q.all())
    db.session.commit()
    return "", 204


"""
Script for building specific layers needed by GAN architecture.
"""
import torch.nn as nn

from torch_mimicry.modules import spectral_norm

def SNConv2d(*args, **kwargs):
    r"""
    Wrapper for applying spectral norm on conv2d layer.
    """
    if kwargs.get('default', True):
        return nn.utils.spectral_norm(nn.Conv2d(*args, **kwargs))

    else:
        return spectral_norm.SNConv2d(*args, **kwargs)

def SNLinear(*args, **kwargs):
    r"""
    Wrapper for applying spectral norm on linear layer.
    """
    if kwargs.get('default', True):
        return nn.utils.spectral_norm(nn.Linear(*args, **kwargs))

    else:
        return spectral_norm.SNLinear(*args, **kwargs)

def SNEmbedding(*args, **kwargs):
    r"""
    Wrapper for applying spectral norm on embedding layer.
    """
    if kwargs.get('default', True):
        return nn.utils.spectral_norm(nn.Embedding(*args, **kwargs))

    else:
        return spectral_norm.SNEmbedding(*args, **kwargs)

class ConditionalBatchNorm2d(nn.Module):
    r"""
    Conditional Batch Norm as implemented in
    https://github.com/pytorch/pytorch/issues/8985

    Attributes:
        num_features (int): Size of feature map for batch norm.
        num_classes (int): Determines size of embedding layer to condition BN.
    """
    def __init__(self, num_features, num_classes):
        super().__init__()
        self.num_features = num_features
        self.bn = nn.BatchNorm2d(num_features, affine=False)
        self.embed = nn.Embedding(num_classes, num_features * 2)
        self.embed.weight.data[:, :num_features].normal_(
            1, 0.02)  # Initialise scale at N(1, 0.02)
        self.embed.weight.data[:,
                               num_features:].zero_()  # Initialise bias at 0

    def forward(self, x, y):
        r"""
        Feedforwards for conditional batch norm.

        Args:
            x (Tensor): Input feature map.
            y (Tensor): Input class labels for embedding.

        Returns:
            Tensor: Output feature map.
        """
        out = self.bn(x)
        gamma, beta = self.embed(y).chunk(
            2, 1)  # divide into 2 chunks, split from dim 1.
        out = gamma.view(-1, self.num_features, 1, 1) * out + beta.view(
            -1, self.num_features, 1, 1)

        return out


#!/usr/bin/python3
"""
Library for Raspberry Pi interfacing with the 16-Bit I/O Expander MCP23017
from Microchip Technology.
"""

__all__ = ["MCP23017", "LCD20x4"]

import time
import smbus
from collections import Iterable

class MCP23017:
    """
    Class for the MCP23017 I/O port expander.
    """
    REG_BASE_ADDR = {'IODIR': 0x00, 'IPOL': 0x02, 'GPINTEN': 0x04, 'DEFVAL': 0x06, 'INTCON': 0x08, 'IOCON': 0x0A,
                        'GPPU': 0x0C, 'INTF': 0x0E, 'INTCAP': 0x10, 'GPIO':0x12, 'OLAT': 0x14}

    OUT = 0
    IN = 1
    LOW = 0
    HIGH = 1
    PUD_DOWN = 0
    PUD_UP = 1

    class RegisterValueError(ValueError):
        def __init__(self, value=''):
            self._message = value
        @property
        def message(self):
            return self._message

        @message.setter
        def message(self, value):
            self._message = value

        def __str__(self):
            return self.message

    def __init__(self, bus, address):
        self._bus = smbus.SMBus(bus)
        self.address = address
        self._register_values = dict()

        # Leave ICON.BANK = 0 but set ICON.SEQOP = 1
        self._bus.write_byte_data(self.address, MCP23017.REG_BASE_ADDR['IOCON'], 1 << 5)
        # Read all register values
        for key in MCP23017.REG_BASE_ADDR.keys():
            self._register_values[key] = self._bus.read_word_data(self.address, MCP23017.REG_BASE_ADDR[key])

    def close(self):
        self._bus.close()

    def __enter__(self):
        return self

    def __exit__(self, type_, value, traceback):
        self.close()

    def __str__(self):
        return '<Device class {0} with address {1}>'.format(self.__class__, self.address)

    def setmode(self, *pargs, **kwargs):
        """
        Method for compatibility with RPi.GPIO interface only.
        """
        pass

    def setwarnings(self, value):
        """
        Method for compatibility with RPi.GPIO interface only.
        """
        pass

    @staticmethod
    def set_bit_in_word(word, bit, value):
        """
        Static helper method to set a bit in a word.
        """
        if 0 <= bit <= 15 and value in (0, 1):
            if value:
                word |= (1 << bit)
            else:
                word &= ~(1 << bit)
            return word
        else:
            raise MCP23017.RegisterValueError

    def _write_register_word(self, key, bits, values):
        """
        Helper method to manipulate the instance register values.
        Works for both for a single bit as well as for a list of bits.
        """
        word = self._register_values[key]

        if not isinstance(bits, Iterable):
            word = MCP23017.set_bit_in_word(word, bits, values)
        else:
            number_of_bits = len(bits)
            if not isinstance(values, Iterable):
                # if value is a single value we have to create a list a corresponding value list.
                values = [values] * number_of_bits
            else:
                if len(values) != number_of_bits:
                    raise MCP23017.RegisterValueError

            for bit, value in zip(bits, values):
                word = MCP23017.set_bit_in_word(word, bit, value)

        # check if we have a the word differs to the previous one
        high_byte_diff, low_byte_diff = divmod(word ^ self._register_values[key], 256)

        if high_byte_diff and low_byte_diff:
            # both bytes changed so send the complete word
            self._bus.write_word_data(self.address, MCP23017.REG_BASE_ADDR[key], word)
        elif high_byte_diff ^ low_byte_diff:
            # only one byte differs so we have to send only this one
            value_msb, value_lsb = divmod(word, 256)
            offset, value = (0, value_lsb) if low_byte_diff else (1, value_msb)
            self._bus.write_byte_data(self.address, MCP23017.REG_BASE_ADDR[key] + offset, value)
        else:
            # no change so we do not need to set the hardware register value
            pass
        self._register_values[key] = word

    def setup(self, pins, directions, initial=None, pull_up_down=None):
        """
        Configures a pin or a list of pins either as output or as input.
        """
        try:
            self._write_register_word('IODIR', pins, directions)
            if pull_up_down is not None:
                raise NotImplementedError
            if initial is not None:
                raise NotImplementedError
        except MCP23017.RegisterValueError as e:
            e.message = 'Invalid pin(s) or direction value(s)...'
            raise

    def output(self, pins, states):
        """
        Sets the logic output level of the pin or a list of pins.
        Valid values for the output state are MCP23017.LOW / 0 / False or MCP23017.HIGH / 1 / True.
        """
        try:
            self._write_register_word('GPIO', pins, states)
        except MCP23017.RegisterValueError as e:
            e.message = 'Invalid pin(s) or state value(s)...'
            raise

    def input(self, pin):
        """
        Reads the logic level of the pin.
        """
        self._register_values['GPIO'] = self._bus.read_word_data(self.address, MCP23017.REG_BASE_ADDR['GPIO'])
        return MCP23017.HIGH if ((self._register_values['GPIO'] >> pin)) & 1 else MCP23017.LOW

    def gpio_function(self, pin):
        """
        Returns whether the pin is configured as input or as output.
        """
        self._register_values['IODIR'] = self._bus.read_word_data(self.address, MCP23017.REG_BASE_ADDR['IODIR'])
        return MCP23017.IN if ((self._register_values['IODIR'] >> pin)) & 1 else MCP23017.OUT

    def cleanup(self):
        pass

class LCD20x4:
    """
    Simple class for a character lcd connected via the MCP23017 I/O Expander.
    """

    LINE_OFFSETS = [0x00, 0x40, 0x14, 0x54]
    LINE_WIDTH = 20
    LINE_NUMBER = 4

    CMD_CLEAR_DISPLAY = 0x01
    CMD_RETURN_HOME = 0x02

    CMD_SET_ENTRY_MODE = 0x04
    ENTRY_LEFT = 0x02

    CMD_DISPLAY_CONTROL = 0x08
    CURSOR_ON = 0x02
    CURSOR_OFF = 0x00
    DISPLAY_ON = 0x04
    DISPLAY_OFF = 0x00

    # Flags for the function set instruction
    CMD_FUNCTION_SET = 0x20
    EIGHT_BIT_MODE = 0x10
    FOUR_BIT_MODE = 0x00
    ONE_LINE = 0x00
    TWO_LINES = 0x08

    CMD_SET_DDRAM_ADDRESS = 0x80

    def __init__(self, interface, rs, en, data, rw=None):
        self._pins = {}
        self._interface = interface
        self._pins['RS'] = rs
        self._pins['E'] = en
        self._bit_mode = len(data)
        if self._bit_mode not in (4, 8):
            raise ValueError('Invalid number of data pins...')
        self._pins['DATA']= data
        self._pins['RW'] = rw

        # Setup all pins as outputs
        for pin in (rs, en, *data):
            self._interface.setup(pin, MCP23017.OUT)
            self._interface.output(pin, MCP23017.LOW)
        if rw is not None:
            self._interface.setup(rw, MCP23017.OUT)
            # for now we use the pin only for write actions!
            self._interface.output(rw, MCP23017.LOW)

        function_set = LCD20x4.CMD_FUNCTION_SET |\
                                   (LCD20x4.FOUR_BIT_MODE if self._bit_mode == 4 else LCD20x4.EIGHT_BIT_MODE) |\
                                   LCD20x4.TWO_LINES
        entry_mode = LCD20x4.CMD_SET_ENTRY_MODE | LCD20x4.ENTRY_LEFT

        if self._bit_mode == 4:
            self._write(0x33, delay_ms = 4.1)
            self._write(0x32, delay_ms = 0.1)
        else:
            for _ in range(3):
                self._write(function_set, delay_ms = 4.1)
        self._write(function_set)
        self._write(entry_mode)

        self.display_off()
        self.clear_display()

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()

    def close(self):
        self._interface.cleanup()

    def _write(self, data, char_mode=False, delay_ms=0):
        """
        Writes the 8-bit value either in character or in command mode.
        """
        self._interface.output([self._pins['RS'], self._pins['E']], [char_mode, MCP23017.LOW])
        data = divmod(data, 16) if self._bit_mode == 4 else (data,)
        for number in data:
            output = []
            i = 0
            while i < self._bit_mode:
                output.append(MCP23017.HIGH if ((number >> i) & 1) else MCP23017.LOW)
                i += 1
            self._interface.output(self._pins['DATA'], output)
            self._interface.output(self._pins['E'], MCP23017.HIGH)
            self._interface.output(self._pins['E'], MCP23017.LOW)
            if delay_ms > 0: time.sleep(delay_ms / 1000.0)

    def clear_display(self):
        """
        Clears the character display.
        """
        self._write(LCD20x4.CMD_CLEAR_DISPLAY)
        self.set_cursor_position(0, 0)

    def display_off(self):
        self._write(LCD20x4.CMD_DISPLAY_CONTROL | LCD20x4.DISPLAY_OFF, delay_ms=0.037)

    def display_on(self):
        self._write(LCD20x4.CMD_DISPLAY_CONTROL | LCD20x4.DISPLAY_ON, delay_ms=0.037)

    def set_cursor_position(self, line, column):
        """
        Sets the cursor to the desired position defined by the line and column number.
        """
        if 0 <= line < LCD20x4.LINE_NUMBER and 0 <= column < LCD20x4.LINE_WIDTH:
            self._write((LCD20x4.CMD_SET_DDRAM_ADDRESS | (LCD20x4.LINE_OFFSETS[line]) + column), delay_ms=0.037)
        else:
            raise ValueError('Invalid line or column position')

    def write_line(self, line, column, text):
        """
        Writes a text to the specified line and starting at the specified column.
        """
        self.set_cursor_position(line, column)
        for char in text[:(LCD20x4.LINE_WIDTH - column)]:
            self.write_char(char)

    def clear_line(self, line, column=0):
        """
        Clears the line beginning from column.
        """
        if 0 <= line < LCD20x4.LINE_NUMBER and 0 <= column < LCD20x4.LINE_WIDTH:
            self.write_line(line, column, ' ' * (LCD20x4.LINE_WIDTH - column))

    def write_char(self, char):
        """
        Writes a character to the current cursor position.
        """
        self._write(ord(char), True)


"""
This script publishes all *summaries* on data.world.

"""

import os
import requests
import json

dir = 'summaries'
username = 'stsievert'
dataset = 'caption-contest-data'
base_url = ('https://raw.githubusercontent.com/nextml/caption-contest-data/'
            'master/contests/summaries/')

files = [{'name': filename, 'source': {'url': base_url + filename}}
         for filename in os.listdir(dir)
         if filename[0] not in {'_', '.'} and not os.path.isdir(dir + filename)]

token = os.environ.get('DATA_WORLD_TOKEN')
url = f'https://api.data.world/v0/datasets/{username}/{dataset}/files'
data = {'files': files}
headers = {'Content-Type': 'application/json',
           'Authorization': 'Bearer ' + token}
data = json.dumps(data)
r = requests.post(url, data=data, headers=headers)
print(r.json())


import pytest
from datetime import date, datetime
from analyzers import AnalyzeTokens
import os

pytestmark = pytest.mark.usefixtures("spark")

def test_token_stat(spark):
    path = os.getcwd() + "/tests/fixtures/token_stat"
    start_date = date(2017, 1, 15)
    end_date = date(2017, 1, 16)

    analyzer = AnalyzeTokens(storage_path=path, start_date=start_date, end_date=end_date, spark_session=spark,
                             database=None, current_datetime=datetime(2017, 2, 15, 15, 10))

    files = analyzer.get_files_to_analyze()

    expected_files = [path + '/2017/01/15/token_stat.json.log', path + '/2017/01/16/token_stat.json.log']

    assert len(files) == len(expected_files)
    assert len(set(files) - set(expected_files)) == 0

    results = analyzer.get_data()
    expected_results = [(u'token:2', date(2017, 1, 15), 1),
                        (u'token:2', date(2017, 1, 16), 1),
                        (u'token:3', date(2017, 1, 15), 6),
                        (u'token:1', date(2017, 1, 15), 2),
                        (u'token:1', date(2017, 1, 16), 2)]
    assert len(results) == len(expected_results)
    assert results == expected_results
    assert analyzer.get_log_analyzer_stats(datetime(2017, 2, 15, 15, 12)) == \
           "[OK] [2017-02-15 15:12:00] [2017-02-15 15:10:00] [TokenStatsUpdater] [120]"

def test_token_stat_empty_file(spark):
    path = os.getcwd() + "/tests/fixtures/token_stat"
    start_date = date(2017, 1, 17)
    end_date = date(2017, 1, 17)
    analyzer = AnalyzeTokens(storage_path=path, start_date=start_date, end_date=end_date, spark_session=spark,
                             database=None)

    files = analyzer.get_files_to_analyze()
    expected_files = [path + '/2017/01/17/token_stat.json.log']
    assert len(files) == len(expected_files)
    results = analyzer.get_data()
    assert len(results) == 0


from django.contrib.auth.decorators import login_required
from django.shortcuts import render, redirect

from core import cards
from core import menu
from core.authentication import AuthenticationManager
from health.models.activity import Activity
from health.models.consumption import Consumption
from health.models.weight import WeightHistory

def index(request):
    return render(request, 'index.html')

@login_required
def dashboard(request):
    user = AuthenticationManager.get_session_user(request=request)

    if not user.is_authenticated:
        return redirect('core:register')

    page_cards = cards.get_cards(request)
    menu_items = menu.get_menu_items('Dashboard')

    context = {
        'username': user.username,
        'menu_items': menu_items,
        'cards': page_cards
    }

    return render(request, 'dashboard.html', context)

@login_required
def timeline(request):
    user = AuthenticationManager.get_session_user(request=request)
    menu_items = menu.get_menu_items('Timeline')

    consumptions = Consumption.objects.filter(user=user, is_active='1')
    activities = Activity.objects.filter(user=user, is_active='1')
    weights = WeightHistory.objects.filter(user=user, is_active='1')

    context = {
        'username': user.username,
        'menu_items': menu_items,
        'consumptions': consumptions,
        'activities': activities,
        'weights': weights,
    }

    return render(request, 'timeline.html', context)

@login_required
def profile(request):
    user = AuthenticationManager.get_session_user(request=request)
    menu_items = menu.get_menu_items('Profile')

    data = {
        'username': user.username,
        'first_name': user.first_name,
        'last_name': user.last_name,
        'fullname': user.get_full_name(),
        'email': user.email,
        'gender': user.profile.gender if user.profile.gender is not None else "",
        'age': user.profile.get_age() if user.profile.birth_date is not None else "",
        'birth_date': user.profile.birth_date if user.profile.birth_date is not None else "",
        'bmi': user.profile.bmi if user.profile.bmi is not None else "",
        'weight': user.profile.weight if user.profile.weight is not None else "",
        'height': user.profile.height if user.profile.height is not None else ""
    }

    context = {
        'profile': data,
        'menu_items': menu_items
    }

    return render(request, 'profile.html', context)


# -*- coding: utf-8 -*-
"""
WSPIA_eu spider created on the top of ATSSpider

scrapy crawl wspia_eu -a iteration=1 -a extract=1 -a mining_job_id=9999 -a url="http://www.biurokarier.wspia.eu/lista-ogloszen?q="

Seed URL:
    http://www.biurokarier.wspia.eu/lista-ogloszen?q=
"""

from re import compile
from scrapy.http import Request
from scrapy.selector import Selector
from urlparse import urljoin

from brightcorp.base.atsspiders import ATSSpider
from brightcorp.items import BrightcorpItemLoader
from brightcorp.processors import ConvertDateString, NormalizedJoin, Prefix

pattern = {
    'ref_number': compile(r'id\/(\d+)'),
}

class WSPIA_eu(ATSSpider):

    name = 'wspia_eu'

    def parse(self, response):
        sel = Selector(response)

        for href in sel.xpath(
            '//div[@class="pageItemContent"]/div/div[@class="careerItem"]//div[@class="careerItemPosition"]/a/@href'
        ).extract():
            yield Request(
                callback=self.parse_job_callback(),
                url=urljoin(response.url, href)
            )
        # pagination
        next_page = sel.xpath(
            '//div[@class="pager"]/div/ul/li[@class="pagerNext"]/span/a/@href'
        ).extract()
        if next_page:
            yield Request(
                callback=self.parse,
                url=urljoin(response.url, next_page[0])
            )

    def parse_job(self, response):
        sel = Selector(response)

        loader = BrightcorpItemLoader(selector=sel)

        loader.add_xpath(
            'title',
            '//div[contains(@class, "pageItem")]/h1[@class="pageItemName"]/text()'
        )
        loader.add_xpath(
            'company',
            '//div[@class="pageItemContent"]//div[@class="careerItem"]/span[contains(text(), "%s")]/following-sibling::span[1]/text()' % unicode('Nazwa firmy:', 'utf-8')
        )
        loader.add_xpath(
            'industry',
            '//div[@class="pageItemContent"]//div[@class="careerItem"]/span[contains(text(), "%s")]/following-sibling::span[1]/text()' % unicode('Branża:', 'utf-8')
        )
        loader.add_xpath(
            'location',
            [
                '//div[@class="pageItemContent"]//div[@class="careerItem"]/span[contains(text(), "%s")]/following-sibling::span[1]/text()' % unicode('Miasto:', 'utf-8'),
                '//div[@class="pageItemContent"]//div[@class="careerItem"]/span[contains(text(), "%s")]/following-sibling::span[1]/text()' % unicode('Region:', 'utf-8'),
            ],
            NormalizedJoin(', ')
        )
        loader.add_xpath(
            'date',
            '//div[@class="pageItemContent"]//div[@class="careerItem"]/span[contains(text(), "%s")]/following-sibling::span[1]/text()' % unicode('Data publikacji:', 'utf-8'),
            ConvertDateString('%d-%m-%Y')
        )
        loader.add_xpath(
            'description',
            '//div[@class="pageItemContent"]/div/div[@class="careerContent"]'
        )
        loader.add_value(
            'referencenumber',
            response.url,
            Prefix('%s-' % self.name),
            re=pattern['ref_number']
        )
        loader.add_value('url', response.url)
        loader.add_value('apply_url', response.url)

        yield loader.load_item()


from argparse import Namespace
from pathlib import Path
from typing import Union
from gooey import Gooey, GooeyParser

import pandas as pd
from sklearn.preprocessing import StandardScaler

from .generator_config import LinearGeneratorConfig
from model.Linear.dataframe_iterator import DataFrameIterator
from model.Linear.config_samples import LinearBostonHousePricesConfig

class LinearGeneratorConfigList():
    def __init__(self,
                 linear_generator_config_list=[
                     LinearBostonHousePricesConfig(),
                     LinearGeneratorConfig(),
                     ],
                 ):
       self.linear_generator_config_list = linear_generator_config_list

    def config(self, cmd, args):
        for config in self.linear_generator_config_list:
            if config.NAME == cmd:
                config.update(args)
                return config

    def _sub_parser(self, subs=GooeyParser().add_subparsers()):
        for config in self.linear_generator_config_list:
            parser = subs.add_parser(config.NAME)
            config._parser(parser)

    def _parser(self, parser=GooeyParser()):
        subs = parser.add_subparsers()
        self._sub_parser(subs)
        return parser

generator_parser = LinearGeneratorConfigList()._parser

class LinearGenerator():
    def __init__(self, linear_generator_config=LinearGeneratorConfig()):
        self.config = linear_generator_config

    def train_valid_generator(self):
        try:
            dataframe = pd.read_csv(self.config.DATAFRAME_PATH)

            standard_scaler = StandardScaler()
            standard_scaler.fit(dataframe[self.config.X_COL])
            dataframe[self.config.X_COL] = \
                standard_scaler.transform(dataframe[self.config.X_COL])
        except Exception as e:
            print(e)
            assert False, "wrong dataframe path {}.".format(
                    self.config.DATAFRAME_PATH)
        train_generator = DataFrameIterator(
                dataframe=dataframe,
                x_col=self.config.X_COL,
                y_col=self.config.Y_COL,
                class_mode=self.config.CLASS_MODE,
                batch_size=self.config.BATCH_SIZE,
                shuffle=True,
                )
        if self.config.VALID_DATAFRAME_PATH:
            try:
                valid_dataframe = pd.read_csv(self.config.VALID_DATAFRAME_PATH)
                valid_dataframe[self.config.X_COL] = \
                    standard_scaler.transform(valid_dataframe[self.config.X_COL])
            except Exception as e:
                print(e)
                assert False, "wrong dataframe path {}.".format(
                    self.config.VALID_DATAFRAME_PATH)
            valid_generator = DataFrameIterator(
                dataframe=valid_dataframe,
                x_col=self.config.X_COL,
                y_col=self.config.Y_COL,
                class_mode=self.config.CLASS_MODE,
                batch_size=self.config.BATCH_SIZE,
                shuffle=False,
                )
        else:
            valid_generator = None
        return (train_generator, valid_generator)


#!/usr/bin/env python3

import boto3
from os import environ
from tabulate import tabulate
from datetime import datetime

ec2_res = boto3.resource('ec2', region_name=environ['AWS_REGION_NAME'])
ec2_cli = boto3.client('ec2', region_name=environ['AWS_REGION_NAME'])

def get_owner_id():
    return boto3.client('sts').get_caller_identity().get('Account')

def get_images_info(owner_id):
    for ami in ec2_res.images.filter(Owners=owner_id):

        cdate = datetime.strptime(ami.creation_date, '%Y-%m-%dT%H:%M:%S.%fZ')
        yield {
            'creation_date': cdate,
            'id': ami.id,
            'name': ami.name,
        }

def group_by_name(amis):
    groups = {}

    for ami in amis:
        last_sep_index = [
            pos for pos, char in enumerate(ami['name']) if char == '_'
        ][-1]

        name = ami['name'][0:last_sep_index]
        if not groups.get(name):
            groups[name] = []

        groups[name].append(ami)

    return groups

def sort_by_age(group):
    amis = []

    def to_dict():
        dict_ = {}

        for g in group:
            cdate = g['creation_date']
            dict_[cdate] = g

        return dict_

    for ami in sorted(to_dict().items(), reverse=True):
        _, info = ami
        amis.append(info)

    return amis

def filter_for_deletion(amis):
    min_available = int(environ['MINIMUM_AVAILABLE_AMIS'])

    if len(amis) <= min_available:
        return amis, None

    to_keep = amis[0:min_available]
    to_delete = amis[min_available:]

    return to_keep, to_delete

def output(group, keep_list, del_list):
    def fl(amis):
        L = []

        for ami in amis:
            d = ami['creation_date']
            L.append([
                '{0}-{1}-{2} {3}:{4}:{5}'.format(
                    d.month, d.day, d.year, d.hour, d.minute, d.second
                ),
                ami['name'],
                ami['id']
            ])

        return tabulate(L, headers=['Creation Date', 'AMI Name', 'AMI Id'])

    #
    charseq = '*' * len(group)
    print('\n{0}\n{1}\n{2}\n'.format(charseq, group, charseq))

    #
    print('\n\nAMIS TO KEEP:\n\n{0}'
          '\n\nAMIS TO DELETE:\n\n{1}\n'.format(
              (fl(keep_list) if keep_list else 'No AMIs to keep.'),
              (fl(del_list) if del_list else 'No AMIs to delete.')
          ))

def lambda_handler(event, context):
    owner  = get_owner_id()
    amis   = get_images_info([owner])
    groups = group_by_name(amis)

    for title, group in groups.items():
        sorted_list = sort_by_age(group)
        keep_list, del_list = filter_for_deletion(sorted_list)

        output(title, keep_list, del_list)

        if del_list:
            for ami in del_list:
                ec2_cli.deregister_image(ImageId=ami['id'])


import tensorflow as tf
import pickle
import numpy as np
import matplotlib.pyplot as plt

"""
You have to implement an image classification neural network for the same dataset as before. Feel free to reduce
the number of examples for each class in your dataset, if it takes too long to train on your hardware.

For this task, you should make at least two neural networks: the fully connected one that works on the same extracted
features as before and another one convolutional with class prediction at the end.

You can do it either in Keras or Pytorch. Better to do both.

As an output, you should provide your code, trained model files (2 pcs. at least), your dataset, and the same precision
metrics [calculated on test images] as you did before.

Your code should provide 3 execution modes/functions: train (for training new model), test (for testing the trained
and saved model on the test dataset), and infer (for inference on a particular folder with images or a single image).
"""

def save_the_model(model, fname):
    import json
    print('Saving the model')
    json_string = model.to_json()
    with open(fname + '.txt', 'w', encoding='utf-8') as f:
        json.dump(json_string, f, ensure_ascii=False)
    print('Saving the weights')
    model.save_weights(fname + '.hdf5')
    print('Done')

def get_model_from_files(fnm):
    import json
    from tensorflow.keras.models import model_from_json
    with open(fnm + '.txt') as infile:
        json_string = json.load(infile)
    model = model_from_json(json_string)
    model.load_weights(fnm + '.hdf5', by_name=False)
    model.compile(loss='binary_crossentropy', optimizer='Adagrad', metrics=['accuracy'])
    model.summary()
    return model

def fully_connected_model():

    # Uploaded features from the last homework. The number of features reduced from 546 to 40 with L1 regularization.

    with open('train_dataset.pickle', 'rb') as f:
        train_dataset_df = pickle.load(f)

    target = train_dataset_df.pop('y')
    Y = np.asarray(target).astype(np.float32)
    X = np.asarray(train_dataset_df).astype(np.float32)
    dataset = tf.data.Dataset.from_tensor_slices((X, Y)).batch(16)
    features, labels = next(iter(dataset))
    train_dataset = dataset.shuffle(len(train_dataset_df)).batch(1)

    # The code for a moder training

    # model2 = tf.keras.Sequential([
    #     tf.keras.layers.Dense(30, activation=tf.nn.relu, input_shape=(40,)),  # input shape required
    #     tf.keras.layers.Dense(20, activation=tf.nn.relu),
    #     tf.keras.layers.Dense(17)
    # ])
    #
    # loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
    #
    # def loss(model, x, y, training):
    #     # training=training is needed only if there are layers with different
    #     # behavior during training versus inference (e.g. Dropout).
    #     y_ = model(x, training=training)
    #
    #     return loss_object(y_true=y, y_pred=y_)
    #
    # def grad(model, inputs, targets):
    #     with tf.GradientTape() as tape:
    #         loss_value = loss(model, inputs, targets, training=True)
    #     return loss_value, tape.gradient(loss_value, model.trainable_variables)
    #
    # optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)
    #
    # loss_value, grads = grad(model2, features, labels)
    # optimizer.apply_gradients(zip(grads, model2.trainable_variables))
    #
    # # Keep results for plotting
    # train_loss_results = []
    # train_accuracy_results = []
    #
    # num_epochs = 201
    #
    # for epoch in range(num_epochs):
    #     epoch_loss_avg = tf.keras.metrics.Mean()
    #     epoch_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()
    #
    #     # Training loop - using batches of 32
    #     for x, y in train_dataset:
    #         # Optimize the model
    #         loss_value, grads = grad(model2, x, y)
    #         optimizer.apply_gradients(zip(grads, model2.trainable_variables))
    #
    #         # Track progress
    #         epoch_loss_avg.update_state(loss_value)  # Add current batch loss
    #         # Compare predicted label to actual label
    #         # training=True is needed only if there are layers with different
    #         # behavior during training versus inference (e.g. Dropout).
    #         epoch_accuracy.update_state(y, model2(x, training=True))
    #
    #     # End epoch
    #     train_loss_results.append(epoch_loss_avg.result())
    #     train_accuracy_results.append(epoch_accuracy.result())
    #
    #     if epoch % 50 == 0:
    #         print("Epoch {:03d}: Loss: {:.3f}, Accuracy: {:.3%}".format(epoch,
    #                                                                     epoch_loss_avg.result(),
    #                                                                     epoch_accuracy.result()))

    # Training log:

    """ Epoch 000: Loss: 2.813, Accuracy: 12.292%
        Epoch 050: Loss: 2.133, Accuracy: 51.979%
        Epoch 100: Loss: 1.946, Accuracy: 61.667%
        Epoch 150: Loss: 1.785, Accuracy: 68.750%
        Epoch 200: Loss: 1.673, Accuracy: 73.125%"""

    with open('train_accuracy_results.pickle', 'rb') as f2:
        train_accuracy_results = pickle.load(f2)

    with open('train_loss_results.pickle', 'rb') as f3:
        train_loss_results = pickle.load(f3)

    fig, axes = plt.subplots(2, sharex=True, figsize=(12, 8))
    fig.suptitle('Training Metrics')

    axes[0].set_ylabel("Loss", fontsize=14)
    axes[0].plot(train_loss_results)

    axes[1].set_ylabel("Accuracy", fontsize=14)
    axes[1].set_xlabel("Epoch", fontsize=14)
    axes[1].plot(train_accuracy_results)
    plt.show()

    # save_the_model(model2, 'model_fully_connected')
    #-------------------------------------------------------------------------------------

    with open('test_dataset.pickle', 'rb') as f2:
        test_dataset_df = pickle.load(f2)

    target_test = test_dataset_df.pop('y')
    Y_test = np.asarray(target_test).astype(np.float32)
    X_test = np.asarray(test_dataset_df).astype(np.float32)
    dataset_t = tf.data.Dataset.from_tensor_slices((X_test, Y_test))
    test_dataset = dataset_t.shuffle(len(test_dataset_df)).batch(1)

    model3 = get_model_from_files('model_fully_connected')

    test_accuracy = tf.keras.metrics.Accuracy()

    for (x, y) in test_dataset:
        # training=False is needed only if there are layers with different
        # behavior during training versus inference (e.g. Dropout).
        logits = model3(x, training=False)
        prediction = tf.argmax(logits, axis=1, output_type=tf.int32)
        test_accuracy(prediction, y)

    print("Test set accuracy: {:.3%}".format(test_accuracy.result())) # Test set accuracy: 15.625%

if __name__ == '__main__':
    fully_connected_model()


import os
os.environ["KIVY_NO_ARGS"] = "1"
#os.environ["KIVY_NO_CONSOLELOG"] = "1"

PATH_BASE = os.path.join(os.getcwd(), os.path.dirname(__file__))

#print os.getcwd()
#print __file__
#print("PATH_BASE: ")
#print(PATH_BASE)
#print(os.path.join(PATH_BASE, 'data/fonts/Ubuntu-R.ttf'))

import kivy
kivy.require('1.9.1') # replace with your current kivy version !

from kivy.app import App
from kivy.clock import Clock
from kivy.properties import ObjectProperty
from kivy.uix.floatlayout import FloatLayout
from kivy.logger import Logger
from kivy.uix.settings import Settings
from kivy.uix.vkeyboard import VKeyboard
from kivy.core.text import LabelBase
from kivy.config import Config
from kivy.garden import iconfonts

import thread

from widgets import *

from client import Client

Config.set('graphics', 'height', '480')
Config.set('graphics', 'width', '800')
Config.set('graphics', 'borderless', '0')
Config.set('kivy', 'keyboard_mode', 'systemanddock')
Config.write()

print(kivy.kivy_base_dir)

VKeyboard.key_background_color = [0.2, 0.2, 0.2, 1]
VKeyboard.margin_hint = [0, 0, 0, 0]
VKeyboard.layout_path = os.path.join(PATH_BASE, 'keyboards')

LabelBase.register(name="Ubuntu",
                   fn_regular    = os.path.join(PATH_BASE, 'data/fonts/Ubuntu-R.ttf'),
                   fn_bold       = os.path.join(PATH_BASE, 'data/fonts/Ubuntu-B.ttf'),
                   fn_italic     = os.path.join(PATH_BASE, 'data/fonts/Ubuntu-RI.ttf'),
                   fn_bolditalic = os.path.join(PATH_BASE, 'data/fonts/Ubuntu-BI.ttf'))

iconfonts.register('default_font', os.path.join(PATH_BASE, 'data/fonts/font-awesome.ttf'), os.path.join(PATH_BASE, 'data/fonts/font-awesome.fontd'))

settingsjson = '''
[
    {
        "type": "string",
        "title": "Octoprint Address",
        "desc": "The IP to use for connecting to Octoprint",
        "section": "Octoscreen",
        "key": "host"
    },
    {
        "type": "numeric",
        "title": "Octoprint Port",
        "desc": "The port to use for connecting to Octoprint",
        "section": "Octoscreen",
        "key": "port"
    },
    {
        "type": "string",
        "title": "Octoprint API Key",
        "desc": "The API Key to use for connecting to Octoprint",
        "section": "Octoscreen",
        "key": "apikey"
    }
]
'''

settingsdefaults = {
    'host': '127.0.0.1',
    'port': 5000,
    'apikey': ''
}

class OctoprintLcd(FloatLayout):

    def switchDefault(self):
        self.ids.tabbedpanel.switch_to(self.ids.tabbedpanel.default_tab)

class OctoprintLcdApp(App):

    client = ObjectProperty(None)

    def __init__(self):
        super(OctoprintLcdApp, self).__init__()

    def build(self):

        self.settings_cls = Settings

        self.client = Client()
        self.client.init(self.config.get('Octoscreen', 'host'), self.config.get('Octoscreen', 'port'), self.config.get('Octoscreen', 'apikey'))
        return OctoprintLcd()

    def build_config(self, config):
        """
        Set the default values for the configs sections.
        """
        config.setdefaults('Octoscreen', settingsdefaults)

    def build_settings(self, settings):
        """
        Add our custom section to the default configuration object.
        """
        settings.add_json_panel('Octoscreen', self.config, data=settingsjson)

    def on_config_change(self, config, section, key, value):
        """
        Respond to changes in the configuration.
        """
        Logger.info("main.py: App.on_config_change: {0}, {1}, {2}, {3}".format(
            config, section, key, value))

    def close_settings(self, settings):
        """
        The settings panel has been closed.
        """
        Logger.info("main.py: App.close_settings: {0}".format(settings))
        self.client.init(self.config.get('Octoscreen', 'host'), self.config.get('Octoscreen', 'port'), self.config.get('Octoscreen', 'apikey'))
        super(OctoprintLcdApp, self).close_settings(settings)

OctoprintLcdApp.run(OctoprintLcdApp())


# uncompyle6 version 3.7.4
# Python bytecode 3.7 (3394)
# Decompiled from: Python 3.7.9 (tags/v3.7.9:13c94747c7, Aug 17 2020, 18:58:18) [MSC v.1900 64 bit (AMD64)]
# Embedded file name: D:\dev\TS4\_deploy\Client\Releasex64\Python\Generated\protocolbuffers\Loot_pb2.py
# Compiled at: 2020-12-13 14:23:13
# Size of source mod 2**32: 3823 bytes
from google.protobuf import descriptor
from google.protobuf import message
from google.protobuf import reflection
from google.protobuf import descriptor_pb2
import protocolbuffers.S4Common_pb2 as S4Common_pb2
import protocolbuffers.ResourceKey_pb2 as ResourceKey_pb2
import protocolbuffers.Situations_pb2 as Situations_pb2
import protocolbuffers.UI_pb2 as UI_pb2
DESCRIPTOR = descriptor.FileDescriptor(name='Loot.proto',
  package='EA.Sims4.Network',
  serialized_pb='\n\nLoot.proto\x12\x10EA.Sims4.Network\x1a\x0eS4Common.proto\x1a\x11ResourceKey.proto\x1a\x10Situations.proto\x1a\x08UI.proto"Ú\x01\n\x0eSituationEnded\x12-\n\ticon_info\x18\x01 \x02(\x0b2\x1a.EA.Sims4.Network.IconInfo\x12\x13\n\x0bfinal_score\x18\x03 \x01(\r\x12;\n\x0bfinal_level\x18\x04 \x01(\x0b2&.EA.Sims4.Network.SituationLevelUpdate\x12\x13\n\x07sim_ids\x18\x05 \x03(\x06B\x02\x10\x01\x122\n\x0baudio_sting\x18\x06 \x01(\x0b2\x1d.EA.Sims4.Network.ResourceKey')
_SITUATIONENDED = descriptor.Descriptor(name='SituationEnded',
  full_name='EA.Sims4.Network.SituationEnded',
  filename=None,
  file=DESCRIPTOR,
  containing_type=None,
  fields=[
 descriptor.FieldDescriptor(name='icon_info',
   full_name='EA.Sims4.Network.SituationEnded.icon_info',
   index=0,
   number=1,
   type=11,
   cpp_type=10,
   label=2,
   has_default_value=False,
   default_value=None,
   message_type=None,
   enum_type=None,
   containing_type=None,
   is_extension=False,
   extension_scope=None,
   options=None),
 descriptor.FieldDescriptor(name='final_score',
   full_name='EA.Sims4.Network.SituationEnded.final_score',
   index=1,
   number=3,
   type=13,
   cpp_type=3,
   label=1,
   has_default_value=False,
   default_value=0,
   message_type=None,
   enum_type=None,
   containing_type=None,
   is_extension=False,
   extension_scope=None,
   options=None),
 descriptor.FieldDescriptor(name='final_level',
   full_name='EA.Sims4.Network.SituationEnded.final_level',
   index=2,
   number=4,
   type=11,
   cpp_type=10,
   label=1,
   has_default_value=False,
   default_value=None,
   message_type=None,
   enum_type=None,
   containing_type=None,
   is_extension=False,
   extension_scope=None,
   options=None),
 descriptor.FieldDescriptor(name='sim_ids',
   full_name='EA.Sims4.Network.SituationEnded.sim_ids',
   index=3,
   number=5,
   type=6,
   cpp_type=4,
   label=3,
   has_default_value=False,
   default_value=[],
   message_type=None,
   enum_type=None,
   containing_type=None,
   is_extension=False,
   extension_scope=None,
   options=(descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\x10\x01'))),
 descriptor.FieldDescriptor(name='audio_sting',
   full_name='EA.Sims4.Network.SituationEnded.audio_sting',
   index=4,
   number=6,
   type=11,
   cpp_type=10,
   label=1,
   has_default_value=False,
   default_value=None,
   message_type=None,
   enum_type=None,
   containing_type=None,
   is_extension=False,
   extension_scope=None,
   options=None)],
  extensions=[],
  nested_types=[],
  enum_types=[],
  options=None,
  is_extendable=False,
  extension_ranges=[],
  serialized_start=96,
  serialized_end=314)
_SITUATIONENDED.fields_by_name['icon_info'].message_type = UI_pb2._ICONINFO
_SITUATIONENDED.fields_by_name['final_level'].message_type = Situations_pb2._SITUATIONLEVELUPDATE
_SITUATIONENDED.fields_by_name['audio_sting'].message_type = ResourceKey_pb2._RESOURCEKEY
DESCRIPTOR.message_types_by_name['SituationEnded'] = _SITUATIONENDED

class SituationEnded(message.Message, metaclass=reflection.GeneratedProtocolMessageType):
    DESCRIPTOR = _SITUATIONENDED


import sys

import logging
import subprocess
from pathlib import Path
from subprocess import CompletedProcess
from typing import Any, IO, Iterable, List, Optional, Union

from tjpy_subprocess_util.exception import SubProcessExecutionException, SubProcessStartException

_logger = logging.getLogger(__name__)

class Result:
    def __init__(self,
                 exit_code: int,
                 stdout: str,
                 stderr: str):
        self.exit_code = exit_code
        self.stdout = stdout
        self.stderr = stderr

    @property
    def trimmed_stdout(self):
        # the output of a sub-process normally ends with a new line for formatting purposes
        # however, it is not part of the value that is being returned from the sub-process
        # this utility property automatically removes this formatting new-line
        if self.stdout.endswith("\n"):
            return self.stdout[:-1]
        else:
            return self.stdout

class SubProcessExecution:

    @staticmethod
    def execute(args: List[str],
                check_error_code: bool = True,
                follow_output: bool = False,
                working_directory: Path = None,
                logging_level: str = "DEBUG",
                custom_input: Optional[str] = None) -> Result:
        SubProcessExecution._log_execute_call(args, working_directory, logging_level)

        stdout: Union[None, int, IO[Any]] = sys.stdout if follow_output else subprocess.PIPE
        stderr: Union[None, int, IO[Any]] = sys.stderr if follow_output else subprocess.PIPE
        stdin: Union[None, int, IO[Any]] = sys.stdin if custom_input is None else None
        try:
            result: CompletedProcess = subprocess.run(
                args,
                cwd=working_directory,
                stdout=stdout,
                stderr=stderr,
                stdin=stdin,
                input=custom_input,
                encoding="utf-8"
            )

            if check_error_code:
                if result.returncode != 0:
                    if not follow_output:
                        if len(result.stdout) != 0:
                            _logger.info("Stdout of failed command:\n" + result.stdout)
                        if len(result.stderr) != 0:
                            _logger.info("Stderr of failed command:\n" + result.stderr)
                    else:
                        _logger.info("Output of failed command has been redirected to stdout and stderr streams.")
                        pass
                result.check_returncode()
            else:
                if follow_output:
                    pass
        except subprocess.CalledProcessError as called_sub_process_error:
            raise SubProcessExecutionException(list(args),
                                               called_sub_process_error.returncode,
                                               SubProcessExecution._output_to_string(called_sub_process_error.stdout),
                                               SubProcessExecution._output_to_string(called_sub_process_error.stderr),
                                               ) from called_sub_process_error
        except subprocess.SubprocessError as sub_process_error:
            raise SubProcessStartException(list(args)) from sub_process_error
        return Result(
            exit_code=result.returncode,
            stdout=SubProcessExecution._output_to_string(result.stdout),
            stderr=SubProcessExecution._output_to_string(result.stderr)
        )

    @staticmethod
    def _output_to_string(output):
        return "" if output is None else str(output)

    @staticmethod
    def _log_execute_call(args: Iterable[str], working_directory: Optional[Path], log_level: str):
        command_text = SubProcessExecution._get_command_text_for_logging(args, working_directory)
        log_level_int: int = logging._nameToLevel[log_level]
        _logger.log(log_level_int, f"Executing command: {command_text}")

    @staticmethod
    def _get_command_text_for_logging(args: Iterable[str], working_directory: Optional[Path]) -> str:
        working_directory_note = ""
        if working_directory is not None:
            working_directory_note = f" in {working_directory}"
        filler = '" "'
        formatted_args = f'"{filler.join(list(args)[1:])}"'
        return f"{next(iter(args))} {formatted_args}{working_directory_note}"


from __future__ import (
    absolute_import,
    unicode_literals,
)

import logging
import threading

import six

class PySOALogContextFilter(logging.Filter):
    def __init__(self):
        super(PySOALogContextFilter, self).__init__('')

    def filter(self, record):
        context = self.get_logging_request_context()
        if context:
            record.correlation_id = context.get('correlation_id') or '--'
            record.request_id = context.get('request_id') or '--'
        else:
            record.correlation_id = '--'
            record.request_id = '--'
        record.service_name = self._service_name or 'unknown'
        return True

    _logging_context = threading.local()

    _service_name = None

    @classmethod
    def set_logging_request_context(cls, **context):
        if not getattr(cls._logging_context, 'context_stack', None):
            cls._logging_context.context_stack = []
        cls._logging_context.context_stack.append(context)

    @classmethod
    def clear_logging_request_context(cls):
        if getattr(cls._logging_context, 'context_stack', None):
            cls._logging_context.context_stack.pop()

    @classmethod
    def get_logging_request_context(cls):
        if getattr(cls._logging_context, 'context_stack', None):
            return cls._logging_context.context_stack[-1]
        return None

    @classmethod
    def set_service_name(cls, service_name):
        cls._service_name = service_name

class RecursivelyCensoredDictWrapper(object):
    # Some performance info about this set
    #
    # In [1]: %timeit 'password' in SENSITIVE_FIELDS
    # The slowest run took 16.69 times longer than the fastest. This could mean that an intermediate result is being
    # cached.
    # 10000000 loops, best of 3: 129 ns per loop
    #
    # In [2]: %timeit 'nope' in SENSITIVE_FIELDS
    # 10000000 loops, best of 3: 131 ns per loop
    #
    # In [3]: %timeit random_string()
    # 100000 loops, best of 3: 10.4 us per loop
    #
    # In [4]: %timeit random_string() in SENSITIVE_FIELDS
    # 100000 loops, best of 3: 10.7 us per loop
    #
    # Suffice it to say, a lookup against this set should always take less than 250 ns.
    SENSITIVE_FIELDS = frozenset({
        # Authentication and related
        'password', 'pass', 'passwd', 'passphrase', 'pass_phrase', 'pass-phrase', 'passPhrase',
        'passwords', 'passwds', 'passphrases', 'pass_phrases', 'pass-phrases', 'passPhrases',
        'private', 'private_key', 'private-key', 'privateKey',
        'privates', 'private_keys', 'private-keys', 'privateKeys',
        'secret', 'secret_key', 'secret-key', 'secretKey',
        'secrets', 'secret_keys', 'secret-keys', 'secretKeys',
        'security_code', 'security-code', 'securityCode',
        'security_codes', 'security-codes', 'securityCodes',
        'auth', 'token', 'auth_token', 'auth-token', 'authToken',
        'authorization', 'authorization_token', 'authorization-token', 'authorizationToken',
        'authentication', 'authentication_token', 'authentication-token', 'authenticationToken',

        # Credit cards, banking, and related
        'credit_card', 'credit-card', 'creditCard', 'credit_card_number', 'credit-card-number', 'creditCardNumber',
        'card_number', 'card-number', 'cardNumber', 'cc_number', 'cc-number', 'ccNumber', 'ccn',
        'credit_cards', 'credit-cards', 'creditCards', 'credit_card_numbers', 'credit-card-numbers',
        'creditCardNumbers',
        'card_numbers', 'card-numbers', 'cardNumbers', 'cc_numbers', 'cc-numbers', 'ccNumbers', 'ccns',
        'cid', 'csc', 'csn', 'cvc', 'cvc2', 'cvd', 'cve', 'cvn2', 'cvv', 'cvv2', 'icvv',
        'cids', 'cscs', 'csns', 'cvcs', 'cvc2s', 'cvds', 'cves', 'cvn2s', 'cvvs', 'cvv2s', 'icvvs',
        'card_id', 'card-id', 'cardId', 'card_identification', 'card-identification', 'cardIdentification',
        'card_ids', 'card-ids', 'cardIds', 'card_identifications', 'card-identifications', 'cardIdentifications',
        'card_identification_code', 'card-identification-code', 'cardIdentificationCode',
        'card_identification_codes', 'card-identification-codes', 'cardIdentificationCodes',
        'card_identification_number', 'card-identification-number', 'cardIdentificationNumber',
        'card_identification_numbers', 'card-identification-numbers', 'cardIdentificationNumbers',
        'card_identification_value', 'card-identification-value', 'cardIdentificationValue',
        'card_identification_values', 'card-identification-values', 'cardIdentificationValues',
        'card_security', 'card-security', 'cardSecurity',
        'card_securities', 'card-securities', 'cardSecurities',
        'card_security_code', 'card-security-code', 'cardSecurityCode',
        'card_security_codes', 'card-security-codes', 'cardSecurityCodes',
        'card_security_number', 'card-security-number', 'cardSecurityNumber',
        'card_security_numbers', 'card-security-numbers', 'cardSecurityNumbers',
        'card_security_value', 'card-security-value', 'cardSecurityValue',
        'card_security_values', 'card-security-values', 'cardSecurityValues',
        'card_validation', 'card-validation', 'cardValidation',
        'card_validations', 'card-validations', 'cardValidations',
        'card_validation_code', 'card-validation-code', 'cardValidationCode',
        'card_validation_codes', 'card-validation-codes', 'cardValidationCodes',
        'card_validation_number', 'card-validation-number', 'cardValidationNumber',
        'card_validation_numbers', 'card-validation-numbers', 'cardValidationNumbers',
        'card_validation_value', 'card-validation-value', 'cardValidationValue',
        'card_validation_values', 'card-validation-values', 'cardValidationValues',
        'card_verification', 'card-verification', 'cardVerification',
        'card_verifications', 'card-verifications', 'cardVerifications',
        'card_verification_code', 'card-verification-code', 'cardVerificationCode',
        'card_verification_codes', 'card-verification-codes', 'cardVerificationCodes',
        'card_verification_number', 'card-verification-number', 'cardVerificationNumber',
        'card_verification_numbers', 'card-verification-numbers', 'cardVerificationNumbers',
        'card_verification_value', 'card-verification-value', 'cardVerificationValue',
        'card_verification_values', 'card-verification-values', 'cardVerificationValues',
        'account_number', 'account-number', 'accountNumber',
        'account_numbers', 'account-numbers', 'accountNumbers',
        'bank_account', 'bank-account', 'bankAccount',
        'bank_accounts', 'bank-accounts', 'bankAccounts',
        'bank_account_number', 'bank-account-number', 'bankAccountNumber',
        'bank_account_numbers', 'bank-account-numbers', 'bankAccountNumbers',
        'pin', 'pin_code', 'pin-code', 'pinCode', 'pin_number', 'pin-number', 'pinNumber',
        'pins', 'pin_codes', 'pin-codes', 'pinCodes', 'pin_numbers', 'pin-numbers', 'pinNumbers',
        'personal_id_number', 'personal-id-number', 'personalIdNumber',
        'personal_id_numbers', 'personal-id-numbers', 'personalIdNumbers',
        'personal_identification_number', 'personal-identification-number', 'personalIdentificationNumber',
        'personal_identification_numbers', 'personal-identification-numbers', 'personalIdentificationNumbers',
    })

    CENSOR_TYPES = six.string_types + six.integer_types

    CENSORED_STRING = '**********'

    def __init__(self, wrapped_dict):
        """
        Wraps a dict to censor its contents. The first time `repr` is called, it copies the dict, recursively
        censors sensitive fields, caches the result, and returns the censored dict repr-ed. All future calls use
        the cache.

        :param wrapped_dict: The `dict` that should be censored
        :type wrapped_dict: dict
        """
        if not isinstance(wrapped_dict, dict):
            raise ValueError('wrapped_dict must be a dict')

        self._wrapped_dict = wrapped_dict
        self._dict_cache = None
        self._repr_cache = None

    def _get_repr_cache(self):
        if not self._dict_cache:
            self._dict_cache = self._copy_and_censor_dict(self._wrapped_dict)

        return repr(self._dict_cache)

    def __repr__(self):
        if not self._repr_cache:
            self._repr_cache = self._get_repr_cache()
        return self._repr_cache

    def __str__(self):
        return self.__repr__()

    def __bytes__(self):
        # If this method is called, we must be in Python 3, which means __str__ must be returning a string
        return self.__str__().encode('utf-8')

    def __unicode__(self):
        # If this method is called, we must be in Python 2, which means __str__ must be returning bytes
        return self.__str__().decode('utf-8')

    @classmethod
    def _copy_and_censor_unknown_value(cls, v, should_censor_strings):
        if isinstance(v, dict):
            return cls._copy_and_censor_dict(v)

        if isinstance(v, (list, tuple, set, frozenset)):
            return cls._copy_and_censor_iterable(v, should_censor_strings)

        if should_censor_strings and v and isinstance(v, cls.CENSOR_TYPES) and not isinstance(v, bool):
            return cls.CENSORED_STRING

        return v

    @classmethod
    def _copy_and_censor_dict(cls, d):
        # This should only be _marginally_ slower than copy.deepcopy
        return {k: cls._copy_and_censor_unknown_value(v, k in cls.SENSITIVE_FIELDS) for k, v in six.iteritems(d)}

    @classmethod
    def _copy_and_censor_iterable(cls, i, should_censor_strings):
        return type(i)(cls._copy_and_censor_unknown_value(v, should_censor_strings) for v in i)


from pathlib import Path
from typing import List, Union

import magenta.music as mm
from magenta.music.protobuf.music_pb2 import NoteSequence

from . import utils

def sample(
    model_name: str,
    num_steps_per_sample: int,
    output_dir: Union[str, Path],
    **sample_kwargs,
) -> List[NoteSequence]:
    model = utils.get_model(model_name)
    sample_sequences = model.sample(n=2, length=num_steps_per_sample, **sample_kwargs)

    midi_writer = utils.MIDIWriter()
    midi_writer.write_midi(sample_sequences, output_dir)
    midi_writer.write_plot(sample_sequences, output_dir)
    return sample_sequences

def interpolate(
    model_name: str,
    start_sequence: NoteSequence,
    end_sequence: NoteSequence,
    num_steps_per_sample: int,
    num_output: int,
    total_bars: int,
    output_dir: Union[str, Path],
) -> NoteSequence:
    model = utils.get_model(model_name)
    interpolate_sequences = model.interpolate(
        start_sequence=start_sequence,
        end_sequence=end_sequence,
        num_steps=num_output,
        length=num_steps_per_sample,
    )

    midi_writer = utils.MIDIWriter()
    midi_writer.write_midi(interpolate_sequences, output_dir, prefix="interpolate")
    midi_writer.write_plot(interpolate_sequences, output_dir, prefix="interpolate")

    interpolate_sequence = mm.sequences_lib.concatenate_sequences(
        interpolate_sequences, sequence_durations=[4] * num_output
    )
    midi_writer.write_midi(interpolate_sequence, output_dir, prefix="merge")
    midi_writer.write_plot(
        interpolate_sequence,
        output_dir,
        prefix="merge",
        plot_max_length_bar=total_bars,
        bar_fill_alphas=[0.5, 0.5, 0.05, 0.05],
    )

    return interpolate_sequence

def groove(
    model_name: str,
    interpolate_sequence: NoteSequence,
    num_steps_per_sample: int,
    num_output: int,
    total_bars: int,
    output_dir: Union[str, Path],
) -> NoteSequence:
    model = utils.get_model(model_name)
    split_interpolate_sequences = mm.sequences_lib.split_note_sequence(
        interpolate_sequence, 4
    )
    encoding, mu, sigma = model.encode(note_sequences=split_interpolate_sequences)
    groove_sequences = model.decode(z=encoding, length=num_steps_per_sample)
    groove_sequence = mm.sequences_lib.concatenate_sequences(
        groove_sequences, [4] * num_output
    )

    midi_writer = utils.MIDIWriter()
    midi_writer.write_midi(groove_sequence, output_dir, prefix="groove")
    midi_writer.write_plot(
        groove_sequence,
        output_dir,
        prefix="groove",
        plot_max_length_bar=total_bars,
        bar_fill_alphas=[0.5, 0.5, 0.05, 0.05],
    )
    return groove_sequence


"""Symplectic, time-reversible, integrators for Hamiltonian trajectories."""
from typing import Callable, Dict, List, NamedTuple, Tuple, Union

import jax

from blackjax.inference.metrics import EuclideanKineticEnergy

__all__ = ["velocity_verlet"]

PyTree = Union[Dict, List, Tuple]

class IntegratorState(NamedTuple):
    """State of the trajectory integration.

    We keep the gradient of the potential energy to speedup computations.
    """

    position: PyTree
    momentum: PyTree
    potential_energy: float
    potential_energy_grad: PyTree

def new_integrator_state(potential_fn, position, momentum):
    potential_energy, potential_energy_grad = jax.value_and_grad(potential_fn)(position)
    return IntegratorState(position, momentum, potential_energy, potential_energy_grad)

def velocity_verlet(
    potential_fn: Callable, kinetic_energy_fn: EuclideanKineticEnergy
) -> Callable:
    """The velocity Verlet (or Verlet-Störmer) integrator.

    The velocity Verlet is a two-stage palindromic integrator [1]_ of the form
    (a1, b1, a2, b1, a1) with a1 = 0. It is numerically stable for values of
    the step size that range between 0 and 2 (when the mass matrix is the
    identity).

    While the position (a1 = 0.5) and velocity Verlet are the most commonly used
    in samplers, it is known in the numerical computation literature that the value
    $a1 \approx 0.1932$ leads to a lower integration error [2,3]_. The authors of [1]_
    show that the value $a1 \approx 0.21132$ leads to an even higher step acceptance
    rate, up to 3 times higher than with the standard position verlet (p.22, Fig.4).

    By choosing the velocity verlet we avoid two computations of the gradient
    of the kinetic energy. We are trading accuracy in exchange, and it is not
    clear whether this is the right tradeoff.

    References
    ----------
    .. [1]: Bou-Rabee, Nawaf, and Jesús Marıa Sanz-Serna. "Geometric
            integrators and the Hamiltonian Monte Carlo method." Acta Numerica 27
            (2018): 113-206.
    .. [2]: McLachlan, Robert I. "On the numerical integration of ordinary
            differential equations by symmetric composition methods." SIAM Journal on
            Scientific Computing 16.1 (1995): 151-168.
    .. [3]: Schlick, Tamar. Molecular modeling and simulation: an
            interdisciplinary guide: Vol. 21. Springer
            Science & Business Media, 2010.

    """
    a1 = 0
    b1 = 0.5
    a2 = 1 - 2 * a1

    potential_grad_fn = jax.value_and_grad(potential_fn)
    kinetic_energy_grad_fn = jax.grad(kinetic_energy_fn)

    def one_step(state: IntegratorState, step_size: float) -> IntegratorState:
        position, momentum, _, potential_energy_grad = state

        momentum = jax.tree_util.tree_multimap(
            lambda momentum, potential_grad: momentum - b1 * step_size * potential_grad,
            momentum,
            potential_energy_grad,
        )

        kinetic_grad = kinetic_energy_grad_fn(momentum)
        position = jax.tree_util.tree_multimap(
            lambda position, kinetic_grad: position + a2 * step_size * kinetic_grad,
            position,
            kinetic_grad,
        )

        potential_energy, potential_energy_grad = potential_grad_fn(position)
        momentum = jax.tree_util.tree_multimap(
            lambda momentum, potential_grad: momentum - b1 * step_size * potential_grad,
            momentum,
            potential_energy_grad,
        )

        return IntegratorState(
            position, momentum, potential_energy, potential_energy_grad
        )

    return one_step


import csv
import pandas as pd
import numpy as np
from numpy.polynomial.polynomial import polyfit
import matplotlib.pyplot as plt
from SK_Agreement.Randomness_Tests.approximate_entropy_test import approximate_entropy_test
from SK_Agreement.Randomness_Tests.cumulative_sums_test import cumulative_sums_test
from SK_Agreement.Randomness_Tests.monobit_test import monobit_test
from SK_Agreement.Randomness_Tests.serial_test import serial_test
import cmath
import itertools
from SK_Agreement.info_reconciliation import divide_into_parts
from SK_Agreement.Randomness_Tests.monobit_test import count_ones_zeroes

def read_coefficients_from_csv(filename, columns_file):
    """
    Read coefficients from csv file.

    :param filename: str (e.g. alice_wifi_capture.csv)
    :param columns_file: str (path to column names for file)
    :return vectors: list (2-D list of channel coefficients. file lines  x channel count (52))

    """
    correct_column_names = open(columns_file, mode="r").readlines()
    correct_column_names = [line.strip() for line in correct_column_names]

    csv_reader = csv.DictReader(open(filename, mode="r"))
    current_column_names = csv_reader.fieldnames

    with open(columns_file) as f:
        for i, l in enumerate(f):
            pass
    file_length = i + 1

    # if this is the first time we are using this file we need headers
    if current_column_names != correct_column_names:
        df = pd.read_csv(filename, usecols=range(file_length), names=correct_column_names, header=None)
        df.to_csv(filename, encoding='utf-8', index=False)

    channel_coefficients = []

    with open(filename, mode="r") as csv_file:
        csv_reader = csv.DictReader(csv_file)
        for row in csv_reader:
            line = []
            for number in range(52):
                line.append(row[f"CH_{number}"])
            channel_coefficients.append(line)

    return channel_coefficients

def compute_absolutes(raw_channel_coefficients):
    """
    Replaces "i" with "j" in channel readings then computes the absolutes

    :param: raw_channel_coefficients: 2-D list of complex values
    :return processed_channel_coefficients: 2-D list of floats

    """

    processed_channel_coefficients = []

    for row in raw_channel_coefficients:
        line = []
        for channel in row:
            try:
                value = float('%.4f' % abs(complex(channel)))
            except:
                print("Error with complex values")
                print(channel)
            line.append(value)

        processed_channel_coefficients.append(line)

    return processed_channel_coefficients

def compute_phase(raw_channel_coefficients):
    """
    Replaces "i" with "j" in channel readings then computes the absolutes

    :param: raw_channel_coefficients: 2-D list of complex values
    :return processed_channel_coefficients: 2-D list of floats

    """

    processed_channel_coefficients = []

    for row in raw_channel_coefficients:
        line = []
        for channel in row:
            line.append(float('%.4f' % cmath.phase(complex(channel))))
        processed_channel_coefficients.append(line)

    return processed_channel_coefficients

def extract_real_part(raw_channel_coefficients):
    """
    Replaces "i" with "j" in channel readings then extract the real part of the readings

    :param: raw_channel_coefficients: 2-D list of complex values
    :return processed_channel_coefficients: 2-D list of floats

    """

    processed_channel_coefficients = []

    for row in raw_channel_coefficients:
        line = []
        for channel in row:
            line.append(float('%.4f' % complex(channel.replace('i', 'j')).real))
        processed_channel_coefficients.append(line)

    return processed_channel_coefficients

def extract_imag_part(raw_channel_coefficients):
    """
    Replaces "i" with "j" in channel readings then extract the imag part of the readings

    :param: raw_channel_coefficients: 2-D list of complex values
    :return processed_channel_coefficients: 2-D list of floats

    """

    processed_channel_coefficients = []

    for row in raw_channel_coefficients:
        line = []
        for channel in row:
            line.append(float('%.4f' % complex(channel.replace('i', 'j')).imag))
        processed_channel_coefficients.append(line)

    return processed_channel_coefficients

def best_fit_line_threshold(data):
    """
    Finds the best fit 2nd order polynomial for data.
    Quantizes data using the polynomial as a threshold.

    :param: data: list of floats (e.g. absolutes of 52 channels)
    :return quantized_measurments: list of floats

    """

    x = np.arange(len(data))
    y = data

    b, m1, m2 = polyfit(x, y, 2)

    best_fit_line = b + m1 * x + m2 * x * x

    #plt.plot(x, y, '.')
    #plt.plot(x, b + m1 * x + m2 * x * x, '-')
    #plt.show()

    quantized_measurements = y

    for n in range(len(y)):
        if y[n] > best_fit_line[n]:
            quantized_measurements[n] = 1
        elif y[n] < best_fit_line[n]:
            quantized_measurements[n] = 0

    return quantized_measurements

def check_randomness(key):
    """
    Runs randomness tests from NIST on key.
    All tests have to pass to declare the key random.
    Selected tests:
        - monobit: checks that there are about as many 1s as 0s
        - approximate entropy:  compare the frequency of overlapping blocks of two consecutive/adjacent lengths
          (m and m+1) against the expected result for a random sequence
        - cumulative sums: determine whether then cumulative sum of the partial sequences occurring in the tested
          sequence is too large or too small relative to the expected behavior of that cumulative sum
          for random sequences.
        - serial: checks the frequency of all possible overlapping m-bit patterns

    :param: key: list of ints (generated in quantization)
    :return test_result: bool

    """
    monobit_result, monobit_p, _ = monobit_test(key)

    # if monobit test doesn't pass there is no need to run the others
    if monobit_result:
        approx_entropy_result, approx_entropy_p, _ = approximate_entropy_test(key)
        cumulative_sums_result, _, cumulative_sums_plist = cumulative_sums_test(key)
        serial_result, _, serial_plist = serial_test(key, patternlen=3)
        test_result = approx_entropy_result and cumulative_sums_result and serial_result
    else:
        test_result = monobit_result

    return test_result

def average_channel_thresholding_abs(coefficients):
    """
    Computes average for each channel from measurements uses that for threshold.
    Then uses a sliding window of length 10 to pick out the channels from the each measurement.
    If the largest absolute value in that sample is positive than it quantizes this measurement as a 1.
    Otherwise it quantizes it as a 0.
    :param coefficients: 2-D list
    :return: key: list
    """

    np_processed_coefficients = np.asarray(coefficients)
    channel_mean = np_processed_coefficients.mean(axis=0)
    channel_std = np_processed_coefficients.std(axis=0)

    sequence = []
    max_deviation_channels = []

    for i in range(len(coefficients)):
        all_channels = np_processed_coefficients[i] - channel_mean - 0 * channel_std

        #for selector in range(3):
        selector = i % 3
        selected_channels = all_channels[selector * 10 + 11 : (selector + 1) * 10 + 11]
        #selected_channels = all_channels[24+selector]
        abs_max = selected_channels.flat[abs(selected_channels).argmax()]

        if np.sign(abs_max) == 1:
            sequence.append(1)
        else:
            sequence.append(0)

        all_channels = list(all_channels)
        max_deviation_channels.append(all_channels.index(abs_max))

    return sequence, max_deviation_channels

def average_channel_thresholding_phase(coefficients, max_deviation_channels):
    """
    Computes average for each channel from measurements uses that for threshold.
    Then uses a sliding window of length 10 to pick out the channels from the each measurement.
    If the largest absolute value in that sample is positive than it quantizes this measurement as a 1.
    Otherwise it quantizes it as a 0.
    :param coefficients: 2-D list
    :return: key: list
    """

    np_processed_coefficients = np.asarray(coefficients)
    channel_mean = np_processed_coefficients.mean(axis=0)
    channel_std = np_processed_coefficients.std(axis=0)

    sequence = []

    for i in range(len(coefficients)):
        all_channels = np_processed_coefficients[i] - channel_mean - 0 * channel_std

        selected_channels = all_channels[max_deviation_channels[i]]

        abs_max = selected_channels.flat[abs(selected_channels).argmax()]

        if np.sign(abs_max) == 1:
            sequence.append(1)
        else:
            sequence.append(0)

    return sequence

def quantize_measurements(data_file, averaging_intervals_count, averaging_length, column_file):

    """
    Generates qunatized bits from csv. Only read the file once. Averaging threshold absolute quantization.
    Returns list of  bit sequences
    :return: list of list of 1s and 0s as single string
    """

    raw_coefficients = read_coefficients_from_csv(data_file, column_file)

    averaging_start = 150
    averaging_stop = averaging_start + averaging_length

    all_processed_coefficients_abs = compute_absolutes(raw_coefficients)
    all_processed_coefficients_phase = compute_phase(raw_coefficients)

    quantized_all = []

    #channels_to_remove_from_each_averaging_interval = []

    for i in range(averaging_intervals_count):

        #channels_to_remove = high_low_threshold(all_processed_coefficients_abs[averaging_start:averaging_stop])
        #channels_to_remove_from_each_averaging_interval.append(channels_to_remove)

        quantized_abs, max_deviation_channels = average_channel_thresholding_abs(
                                                all_processed_coefficients_abs[averaging_start:averaging_stop])

        quantized_phase = average_channel_thresholding_phase(
                                                all_processed_coefficients_phase[averaging_start:averaging_stop],
                                                max_deviation_channels)

        a = np.array(quantized_abs)
        b = np.array(quantized_phase)
        quantized = a & b

        quantized_all.append(list(quantized))
        averaging_stop = averaging_stop + averaging_length
        averaging_start = averaging_start + averaging_length

    #print(channels_to_remove_from_each_averaging_interval)

    complete_quantized_sequence = list(itertools.chain.from_iterable(quantized_all))

    #bin_quantized_sequence = bin_quantization_2_bit(complete_quantized_sequence, codeword_length)

    random_check = check_randomness(complete_quantized_sequence)

    complete_quantized_sequence_str = "".join(map(str, complete_quantized_sequence))

    return random_check, complete_quantized_sequence_str

def bin_quantization_4_bit(complete_quantized_sequence, codeword_length):

    n_bit_parts = divide_into_parts(complete_quantized_sequence, codeword_length)

    bin_quantized_bits = []
    for sequence in n_bit_parts:
        value = int(''.join(map(str, sequence)), 2)
        if -1 < value <= 1:
            bit_string = "0000"
        elif 1 < value <= 4:
            bit_string = "0001"
        elif 4 < value <= 7:
            bit_string = "0010"
        elif 7 < value <= 11:
            bit_string = "0011"
        elif 11 < value <= 16:
            bit_string = "0100"
        elif 16 < value <= 22:
            bit_string = "0101"
        elif 22 < value <= 27:
            bit_string = "0110"
        elif 27 < value <= 31:
            bit_string = "0111"
        elif 31 < value <= 36:
            bit_string = "1000"
        elif 36 < value <= 41:
            bit_string = "1001"
        elif 41 < value <= 47:
            bit_string = "1010"
        elif 47 < value <= 52:
            bit_string = "1011"
        elif 52 < value <= 56:
            bit_string = "1100"
        elif 56 < value <= 59:
            bit_string = "1101"
        elif 59 < value <= 62:
            bit_string = "1110"
        elif 62 < value <= 64:
            bit_string = "1111"
        bin_quantized_bits.append(bit_string)

    complete_quantized_sequence = list(itertools.chain.from_iterable(bin_quantized_bits))

    return complete_quantized_sequence

    # [ 0.  1.  4.  7. 11. 16. 22. 27. 31. 36. 41. 47. 52. 56. 59. 62. 63.]

def bin_quantization_3_bit(complete_quantized_sequence, codeword_length):

    n_bit_parts = divide_into_parts(complete_quantized_sequence, codeword_length)

    bin_quantized_bits = []
    for sequence in n_bit_parts:
        value = int(''.join(map(str, sequence)), 2)
        if -1 < value <= 15:
            bit_string = "000"
        elif 15 < value <= 30:
            bit_string = "001"
        elif 30 < value <= 47:
            bit_string = "010"
        elif 47 < value <= 63:
            bit_string = "011"
        elif 63 < value <= 79:
            bit_string = "100"
        elif 79 < value <= 97:
            bit_string = "101"
        elif 97 < value <= 112:
            bit_string = "110"
        elif 112 < value <= 129:
            bit_string = "111"
        bin_quantized_bits.append(bit_string)

    complete_quantized_sequence = list(itertools.chain.from_iterable(bin_quantized_bits))

    return complete_quantized_sequence

    # [0.0, 15.0, 30.0, 47.0, 63.0, 79.0, 97.0, 112.0, 127.0]

def bin_quantization_2_bit(complete_quantized_sequence, codeword_length):

    n_bit_parts = divide_into_parts(complete_quantized_sequence, codeword_length)

    bin_quantized_bits = []
    for sequence in n_bit_parts:
        value = int(''.join(map(str, sequence)), 2)
        if -1 < value <= 11:
            bit_string = "00"
        elif 11 < value <= 31:
            bit_string = "01"
        elif 31 < value <= 52:
            bit_string = "10"
        elif 52 < value <= 63:
            bit_string = "11"
        bin_quantized_bits.append(bit_string)

    complete_quantized_sequence = list(itertools.chain.from_iterable(bin_quantized_bits))

    return complete_quantized_sequence

    # [ 0. 11. 31. 52. 63.]

def average_channel_thresholding_statistics(coefficients, multiple_regions, count):
    """
    Computes average for each channel from measurements uses that for threshold.
    Then uses a sliding window of length 10 to pick out the channels from the each measurement.
    If the largest absolute value in that sample is positive than it quantizes this measurement as a 1.
    Otherwise it quantizes it as a 0.
    :param coefficients: 2-D list
    :return: key: list
    """

    np_processed_coefficients = np.asarray(coefficients)
    channel_mean = np_processed_coefficients.mean(axis=0)
    channel_std = np_processed_coefficients.std(axis=0)

    key = []

    for i in range(len(coefficients)):
        if multiple_regions:
            selector = i % count
            if count == 3:
                offset = 11
            if count == 5:
                offset = 1
        else:
            selector = count
            offset = 11

        all_channel = np_processed_coefficients[i] - channel_mean - 0 * channel_std
        selected_channel = all_channel[selector * 10 + offset:(selector + 1) * 10 + offset]
        abs_max = selected_channel.flat[abs(selected_channel).argmax()]

        if np.sign(abs_max) == 1:
            key.append(1)
        else:
            key.append(0)

    return key

def quantize_measurements_statistics(raw_coefficients, quantized_bits_number, averaging_length,
                                     column_file, multiple_regions, number):

    """
    Generates qunatized bits from csv. Only read the file once. Averaging threshold absolute quantization.
    Varying regions. Returns list of averaging_length/2 bit sequences
    :return: list of list of 1s and 0s ints
    """

    averaging_intervals_count = quantized_bits_number // averaging_length

    processed_coefficients_list = []
    averaging_start = 0
    averaging_stop = averaging_start + averaging_length

    all_processed_coefficients = compute_absolutes(raw_coefficients)

    quantized_all = []

    for i in range(averaging_intervals_count):
        quantized = average_channel_thresholding_statistics(coefficients=processed_coefficients_list[averaging_start:averaging_stop],
                                                            multiple_regions=multiple_regions,
                                                            count=number)
        quantized_all.append(quantized)
        averaging_stop = averaging_stop + averaging_length
        averaging_start = averaging_start + averaging_length

    complete_quantized_sequence = list(itertools.chain.from_iterable(quantized_all))

    random_check = check_randomness(complete_quantized_sequence)

    return random_check, complete_quantized_sequence

def quantize_measurements_bin_stats(raw_coefficients, averaging_intervals_count, averaging_length, codeword_length):

    """
    Generates qunatized bits from csv. Only read the file once. Averaging threshold absolute quantization.
    Returns list of  bit sequences
    :return: list of list of 1s and 0s as single string
    """

    averaging_start = 100
    averaging_stop = averaging_start + averaging_length

    all_processed_coefficients_abs = compute_absolutes(raw_coefficients)
    all_processed_coefficients_phase = compute_phase(raw_coefficients)

    quantized_all = []

    for i in range(averaging_intervals_count):
        quantized_abs = average_channel_thresholding(all_processed_coefficients_abs[averaging_start:averaging_stop])
        quantized_phase = average_channel_thresholding(all_processed_coefficients_phase[averaging_start:averaging_stop])

        a = np.array(quantized_abs)
        b = np.array(quantized_phase)
        quantized = a & b

        quantized_all.append(list(quantized))
        averaging_stop = averaging_stop + averaging_length
        averaging_start = averaging_start + averaging_length

    complete_quantized_sequence = list(itertools.chain.from_iterable(quantized_all))

    #bin_quantized_sequence = bin_quantization_3_bit(complete_quantized_sequence, codeword_length)

    #quantized_bit_number = len(bin_quantized_sequence)

    quantized_bit_number = len(complete_quantized_sequence)

    bin_quantized_sequence_parts = divide_into_parts(complete_quantized_sequence, codeword_length)

    random_check = check_randomness(complete_quantized_sequence)

    return random_check, bin_quantized_sequence_parts, quantized_bit_number

def high_low_threshold(coefficients):

    ration = 0.5

    np_processed_coefficients = np.asarray(coefficients)

    channel_means = np_processed_coefficients.mean(axis=0)
    channel_stds = np_processed_coefficients.std(axis=0)

    high_threshold = list(channel_means + ration * channel_stds)
    low_threshold = list(channel_means - ration * channel_stds)

    channels_to_remove_from_each_line = []

    for i in range(len(coefficients)):
        channels_to_remove = []
        for j in range(len(coefficients[i])):
            if low_threshold[j] < coefficients[i][j] < high_threshold[j]:
                channels_to_remove.append(j)

        channels_to_remove_from_each_line.append(channels_to_remove)

    print(channels_to_remove_from_each_line)

    return channels_to_remove_from_each_line

def quantization_without_mediocre_channels(coefficients, channels_to_remove_from_each_averaging_region):

    channels_to_remove_from_each_averaging_region = [[0, 4, 9, 11, 14, 15, 18, 23, 24, 26, 29, 30, 33, 34, 35, 36, 38, 39, 40, 42, 43, 44, 46, 47, 49, 50], [6, 7, 13, 19, 20, 21, 23, 24, 25, 27, 28, 31, 32, 37, 39, 41, 43, 46, 47, 51], [1, 2, 3, 7, 8, 10, 11, 12, 13, 14, 15, 19, 23, 25, 27, 28, 30, 33, 34, 36, 37, 38, 42, 43, 44, 45, 46, 50, 51], [2, 4, 7, 9, 12, 23, 29, 31, 34, 35, 36, 38, 39, 40, 42, 43, 47, 48, 50], [1, 2, 3, 4, 5, 7, 8, 11, 14, 15, 16, 17, 18, 19, 21, 22, 27, 31, 33, 34, 35, 36, 37, 39, 40, 41, 44, 48, 49, 50, 51]]

    ration = 0.5

    np_processed_coefficients = np.asarray(coefficients)

    channel_means = np_processed_coefficients.mean(axis=0)
    channel_stds = np_processed_coefficients.std(axis=0)

    high_threshold = list(channel_means + ration * channel_stds)
    low_threshold = list(channel_means - ration * channel_stds)

    # TO BE CONTINUED

    key = []

    for i in range(len(coefficients)):
        all_channels = np_processed_coefficients[i] - channel_mean - 0 * channel_std
        #for selector in range(3):
        selector = i % 3
        #selected_channels = all_channels[selector * 10 + 11:(selector + 1) * 10 + 11]
        selected_channels = all_channels[24+selector]
        abs_max = selected_channels.flat[abs(selected_channels).argmax()]

        if np.sign(abs_max) == 1:
            key.append(1)
        else:
            key.append(0)

    return key


#!/usr/bin/env python

#####
# Imports
#####

import argparse
import os
# import numpy as np
import pandas as pd

from keras.models import load_model
import lightgbm as lgbm

#####
# Constants
#####
ID_NAME = "SK_ID_CURR"
TARGET_NAME = "TARGET"
#####
# Defaults
#####

#####
# Load the data
#####
def load(filename):

    df = None
    print("--- Loading filename: {}".format(filename))
    df = pd.read_csv(filename)
    print("--- Loaded file, shape: {}".format(df.shape))

    df_Y = df.pop(ID_NAME)
    df_X = df

    return df_X, df_Y

def predict(model, X_test):
    # Predict model
    predictions = model.predict(X_test)
    return predictions

#####
# Execute the training process
#####
def execute(inputfile, modelfile, submissionfile):

    print("--- Parameters")
    print("inputfile:      ", inputfile)
    print("modelfile:      ", modelfile)
    print("submissionfile: ", submissionfile)

    # Load the data
    print("--- Loading input, filename: ", inputfile)
    df_X, df_Y = load(inputfile)

    print("--- Loading model, filename: ", modelfile)
    modeldir, modelfilename = os.path.split(modelfile)
    _, modelfileext = os.path.splitext(modelfilename)
    if modelfileext == ".h5":
        model = load_model(modelfile)
    else:
        model = lgbm.Booster(model_file=modelfile)

    print("--- Making predictions")
    predictions = predict(model, df_X.values)

    submissiondf = pd.DataFrame()
    submissiondf[ID_NAME] = df_Y
    submissiondf[TARGET_NAME] = predictions
    print(submissiondf.head())

    print("Saving submission, submission: ", submissionfile)
    submissiondf.to_csv(submissionfile, header=True, index=False)

#####
# Parse the command line
#####
def cli():
    """
    Command line interface
    """

    # Parse the command line and return the args/parameters
    parser = argparse.ArgumentParser(description="training script")
    parser.add_argument("-i", "--inputfile", help="is the CSV test file containing raw test data (required)")
    parser.add_argument("-m", "--modelfile", help="is the fully qualified filename for the model HDF5 file (required)")
    parser.add_argument("-s", "--submissionfile", help="is the CSV filename to store the submission output (required)")
    args = parser.parse_args()
    return args

#####
# Mainline program
#####
def main():
    """
    Mainline
    """

    # Get the command line parameters
    args = cli()
    if not args.inputfile:
        raise Exception("Missing argument: --inputfile")
    if not args.modelfile:
        raise Exception("Missing argument: --modelfile")
    if not args.submissionfile:
        raise Exception("Missing argument: --submissionfile")

    from os.path import realpath
    args.inputfile = realpath(args.inputfile)
    args.modelfile = realpath(args.modelfile)
    args.submissionfile = realpath(args.submissionfile)

    # Execute the command
    execute(args.inputfile, args.modelfile, args.submissionfile)

main()