from sympy import symbols, Matrix, cos, sin, diff, transpose

"""
This is a basic example of how to compute jacobians and
turn it into simpler c++ code using symbolic programming (sympy).

The equations can be much more complicated and sympy will easily
calculate the derivatives for you.
"""

phi, theta, psi, mx, my, mz = symbols("phi theta psi mx my mz")
R2 = Matrix([[cos(psi), -sin(psi), 0], [sin(psi), cos(psi), 0], [0, 0, 1]])
R1 = Matrix([[cos(theta), 0, sin(theta)], [0, 1, 0], [-sin(theta), 0, cos(theta)]])
R0 = Matrix([[1, 0, 0], [0, cos(phi), -sin(phi)], [0, sin(phi), cos(phi)]])
m = Matrix([mx, my, mz])

print(diff(transpose(R2 * R1 * R0) * m, phi))
print(diff(transpose(R2 * R1 * R0) * m, theta))
print(diff(transpose(R2 * R1 * R0) * m, psi))

j1 = str(diff(transpose(R2 * R1 * R0) * m, phi))
j2 = str(diff(transpose(R2 * R1 * R0) * m, theta))
j3 = str(diff(transpose(R2 * R1 * R0) * m, psi))

assert (
    j1
    == "Matrix([[0], [mx*(sin(phi)*sin(psi) + sin(theta)*cos(phi)*cos(psi)) + my*(-sin(phi)*cos(psi) + sin(psi)*sin(theta)*cos(phi)) + mz*cos(phi)*cos(theta)], [mx*(-sin(phi)*sin(theta)*cos(psi) + sin(psi)*cos(phi)) + my*(-sin(phi)*sin(psi)*sin(theta) - cos(phi)*cos(psi)) - mz*sin(phi)*cos(theta)]])"
)
assert (
    j2
    == "Matrix([[-mx*sin(theta)*cos(psi) - my*sin(psi)*sin(theta) - mz*cos(theta)], [mx*sin(phi)*cos(psi)*cos(theta) + my*sin(phi)*sin(psi)*cos(theta) - mz*sin(phi)*sin(theta)], [mx*cos(phi)*cos(psi)*cos(theta) + my*sin(psi)*cos(phi)*cos(theta) - mz*sin(theta)*cos(phi)]])"
)
assert (
    j3
    == "Matrix([[-mx*sin(psi)*cos(theta) + my*cos(psi)*cos(theta)], [mx*(-sin(phi)*sin(psi)*sin(theta) - cos(phi)*cos(psi)) + my*(sin(phi)*sin(theta)*cos(psi) - sin(psi)*cos(phi))], [mx*(sin(phi)*cos(psi) - sin(psi)*sin(theta)*cos(phi)) + my*(sin(phi)*sin(psi) + sin(theta)*cos(phi)*cos(psi))]])"
)

j1 = j1.replace("sin(phi)", "s0")
j1 = j1.replace("cos(phi)", "c0")
j1 = j1.replace("sin(theta)", "s1")
j1 = j1.replace("cos(theta)", "c1")
j1 = j1.replace("sin(psi)", "s2")
j1 = j1.replace("cos(psi)", "c2")

print(j1)

j2 = j2.replace("sin(phi)", "s0")
j2 = j2.replace("cos(phi)", "c0")
j2 = j2.replace("sin(theta)", "s1")
j2 = j2.replace("cos(theta)", "c1")
j2 = j2.replace("sin(psi)", "s2")
j2 = j2.replace("cos(psi)", "c2")

print(j2)

j3 = j3.replace("sin(phi)", "s0")
j3 = j3.replace("cos(phi)", "c0")
j3 = j3.replace("sin(theta)", "s1")
j3 = j3.replace("cos(theta)", "c1")
j3 = j3.replace("sin(psi)", "s2")
j3 = j3.replace("cos(psi)", "c2")

print(j3)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# This file is part of https://github.com/zsquareplusc/python-msp430-tools
# (C) 2001-2010 Chris Liechti <cliechti@gmx.net>
#
# SPDX-License-Identifier:    BSD-3-Clause
"""\
Simple C preprocessor (almost). It is not fully compliant to a real
ANSI C preprocessor, but it understands a helpful subset.
"""

# TODO:
# - stringify #
# - it isn't fully compatible to a real cpp

import sys
import os
import re
import logging
import binascii
import codecs
from msp430.asm import infix2postfix
from msp430.asm import rpn

def hexlify(text):
    return binascii.hexlify(text.encode('utf-8')).decode('utf-8')

def line_joiner(line_iterator):
    """\
    Given a readline function, return lines, but handle line continuations
    ('\\\n'). When lines are joined, the same number of blank lines is output
    so that the line counter for the consumer stays correct.
    """
    while True:
        joined_line = '\\\n'
        joined_lines = 0
        while joined_line[-2:] == '\\\n':
            joined_line = joined_line[:-2]
            line = next(line_iterator)
            if not line:
                break
            joined_line += line.rstrip() + '\n'  # XXX drops spaces too
            joined_lines += 1
        while joined_lines > 1:
            yield '\n'
            joined_lines -= 1
        if not joined_line:
            break
        yield joined_line

class PreprocessorError(Exception):
    """Preprocessor specific errors"""

class Scanner(infix2postfix.Scanner):
    scannerRE = re.compile(r'''
       (?P<SPACE>           \s+             ) |
       (?P<LEFT>            \(              ) |
       (?P<RIGHT>           \)              ) |
       (?P<HEXNUMBER>       0x[0-9a-f]+     ) |
       (?P<BINNUMBER>       0b[0-9a-f]+     ) |
       (?P<NUMBER>          \d+             ) |
       (?P<UNARYOPERATOR>   defined|!|(\B[-+~]\b)   ) |
       (?P<OPERATOR>        \|\||&&|<<|>>|==|!=|<=|>=|[-+*/\|&\^<>] ) |
       (?P<VARIABLE>        \.?[$_a-z]\w*   )
    ''', re.VERBOSE | re.IGNORECASE | re.UNICODE)

cpp_precedence_list = [
    # lowest precedence
    ['||', 'or'],
    ['&&', 'and'],
    ['|', '^', '&'],
    ['==', '!='],
    ['<', '<=', '>', '>='],
    ['<<', '>>'],
    ['+', '-'],
    ['*', '/', '%'],
    ['!', 'not'],
    ['~', 'neg', '0 +'],
    ['defined'],
    ['(', ')'],
    # highest precedence
]

precedence = infix2postfix.convert_precedence_list(cpp_precedence_list)

class Undefined(object):
    def __int__(self):
        return 0

    def __str__(self):
        return ''

    def __repr__(self):
        return '<UNDEFINED>'
undefined = Undefined()

class Evaluator(rpn.RPN):
    """\
    An RPN calculator with infix to postfix converter, so that expressions for
    #if can be evaluated.
    """
    re_defined_translation = re.compile(r'LOOKUP (\w+?) defined')

    def __init__(self):
        rpn.RPN.__init__(self)
        self.defines = {}
        # add some aliases for logic operators
        self.builtins['!'] = self.builtins['not']
        self.builtins['&&'] = self.builtins['and']
        self.builtins['||'] = self.builtins['or']

    def _translate_defined(self, match):
        return u'DEFINED {}'.format(match.group(1))

    @rpn.word('LOOKUP')
    def word_LOOKUP(self, stack):
        key = self.next_word()
        #~ print "LOOKUP", key
        if key in self.defines:
            try:
                backup = self[:]
                value = self.eval(self.defines[key])
                self[:] = backup  # XXX better way to do this
                self.push(value)
            except Exception:
                #~ print "RPN eval failed using directly: %r %r %s" % (key, self.defines[key], e) # XXX debug
                self.push(self.defines[key])
        else:
            self.push(undefined)
        #~ print "LOOKUP %r %r" % (key, self[-1])

    @rpn.word('DEFINED')
    def word_DEFINED(self, stack):
        key = self.next_word()
        self.push(key in self.defines)

    def eval(self, expression):
        self.clear()
        try:
            rpn_expr = infix2postfix.infix2postfix(
                    expression,
                    scanner=Scanner,
                    precedence=precedence,
                    variable_prefix='LOOKUP ')
        except ValueError:
            raise PreprocessorError('error in expression: {!r}'.format(expression))
        #~ print "RPN: %r" % (rpn_expr,) # XXX debug
        # hack: replace "LOOKUP <word> DEFINED" with "DEFINED <word>"
        rpn_expr = self.re_defined_translation.sub(self._translate_defined, rpn_expr)
        self.interpret_sequence(rpn_expr.split(' '))
        if len(self) != 1:
            raise PreprocessorError('error in expression: {!r} stack: {}'.format(expression, self))
        return self.pop()

class AnnoatatedLineWriter(object):
    """\
    Write lines to the output. If line numbers jump also write out an
    information line with line number and filename: '# nn "filename"'
    """
    def __init__(self, output, filename):
        self.output = output
        self.filename = filename
        self.marker = None

    def write(self, lineno, text):
        """
        Write line. It is expected to be called with lines only (ending in
        '\n') otherwise will the markers be misplaced.
        """
        # emit line number and file hints for the next stage
        if self.marker != lineno:
            self.output.write('# {} "{}"\n'.format(lineno, self.filename))
        self.marker = lineno + 1
        try:
            self.output.write(text)
        except IOError:
            raise EOFError()

class Preprocessor(object):
    """\
    A text processing tool like a C Preprocessor. However it is not 100%
    compatible to a C Preprocessor.
    """

    re_scanner = re.compile(r'''
            (?P<MACRO>      ^[\t ]*\#[\t ]*define[\t ]+(?P<MACRO_NAME>\w+)\((?P<MACRO_ARGS>.*?)\)(?P<MACRO_DEF>[\t ]+(.+))? ) |
            (?P<DEFINE>     ^[\t ]*\#[\t ]*define[\t ]+(?P<DEF_NAME>\w+)([\t ]+)?(?P<DEF_VALUE>.+)?     ) |
            (?P<INCLUDE>    ^[\t ]*\#[\t ]*include[\t ]+[<"](?P<INC_NAME>[\w\\/\.]+)[">]  ) |
            (?P<PRAGMA>     ^[\t ]*\#[\t ]*pragma($|[\t ]+.*)                   ) |
            (?P<MESSAGE>    ^[\t ]*\#[\t ]*message($|[\t ]+.*)                  ) |
            (?P<WARNING>    ^[\t ]*\#[\t ]*warning($|[\t ]+.*)                  ) |
            (?P<ERROR>      ^[\t ]*\#[\t ]*error($|[\t ]+.*)                    ) |
            (?P<IF>         ^[\t ]*\#[\t ]*if[\t ]+(?P<IF_EXPR>.*)              ) |
            (?P<ELIF>       ^[\t ]*\#[\t ]*elif[\t ]+(?P<ELIF_EXPR>.*)          ) |
            (?P<IFDEF>      ^[\t ]*\#[\t ]*ifdef[\t ]+(?P<IFDEF_NAME>.*)        ) |
            (?P<IFNDEF>     ^[\t ]*\#[\t ]*ifndef[\t ]+(?P<IFNDEF_NAME>.*)      ) |
            (?P<ELSE>       ^[\t ]*\#[\t ]*else                                 ) |
            (?P<ENDIF>      ^[\t ]*\#[\t ]*endif                                ) |
            (?P<UNDEF>      ^[\t ]*\#[\t ]*undef[\t ]+(?P<UNDEF_NAME>.*)        ) |
            (?P<NONPREPROC> ^[^\#].*         )
            ''', re.VERBOSE | re.UNICODE)

    re_silinecomment = re.compile(r'(//).*')
    re_inlinecomment = re.compile(r'/\*.*?\*/')
    re_macro_usage = re.compile(r'(?P<MACRO>\w+)[\t ]*\((?P<ARGS>.*?)\)', re.UNICODE)
    re_splitter = re.compile(r'''
            (?P<STRING>     "([^"\\]*?(\\.[^"\\]*?)*?)"     ) |
            (?P<WORD>       \w+         ) |
            (?P<NONWORD>    [^"\w]+       )
            ''', re.VERBOSE | re.UNICODE)

    def __init__(self):
        self.macros = {}
        self.log = logging.getLogger('cpp')
        self.namespace = Evaluator()
        self.include_path = ['.']

    def _apply_macro(self, match_obj):
        name = match_obj.group('MACRO')
        if name in self.macros:
            values = [x.strip() for x in match_obj.group('ARGS').split(',')]
            args, expansion = self.macros[name]
            if len(args) != len(values):
                raise PreprocessorError('Macro invocation with wrong number of parameters. Expected {} got {}: {!r}'.format(
                        len(args), len(values), values))
            self.expansion_done = True
            return expansion % dict(zip(args, values))
        else:
            return match_obj.group(0)   # nothing of ours, return original

    def expand(self, line):
        """Expand object and function like macros in given line."""
        recusion_limit = 10
        self.expansion_done = True
        while self.expansion_done and recusion_limit:
            # replace function like macros
            self.expansion_done = False
            line = self.re_macro_usage.sub(self._apply_macro, line)
            # replace object like macros
            pos = 0
            res = []
            while pos < len(line):
                m = self.re_splitter.match(line, pos)
                if m is None:
                    raise PreprocessorError(u'No match in macro replacement code: {!r}...'.format(
                        line[pos:pos + 10],))
                pos = m.end()
                token_type = m.lastgroup
                if token_type in ('STRING', 'NONWORD', 'OTHER'):
                    word = m.group(token_type)
                    res.append(word)
                elif token_type == 'WORD':
                    word = m.group(token_type)
                    if word in self.namespace.defines:
                        res.append(self.namespace.defines[word])
                        self.expansion_done = True
                    else:
                        res.append(word)
                else:
                    raise PreprocessorError(u'No match in macro replacement code: {!r}...'.format(
                        line[pos:pos + 10],))
            line = ''.join(res)
            recusion_limit -= 1
        if recusion_limit == 0:
            self.log.error('recursive define, stopped expansion: {!r} '.format(line))
        #~ print "expand -> %r" % (res)          #DEBUG
        return line.replace('##', '')

    def preprocess(self, infile, outfile, filename, include_callback=None):
        """Scan lines and process preprocessor directives"""
        self.log.info("processing {}".format(filename))
        error_found = False
        empty_lines = 0
        process = True
        my_if_was_not_hidden = False
        if_name = '<???>'
        hiddenstack = []
        in_comment = False
        writer = AnnoatatedLineWriter(outfile, filename)
        line = ''
        lineno = 0
        try:
            for line in line_joiner(iter(infile)):
                lineno += 1
                #~ print "|", line.rstrip()
                line = self.re_inlinecomment.sub('', line)  # .strip()
                if in_comment:
                    p = line.find('*/')
                    if p >= 0:
                        in_comment = False
                        line = line[p + 2:]
                        if not line.strip():
                            continue
                    else:
                        continue
                #~ line = line.rstrip()
                line = self.re_silinecomment.sub('', line)
                #~ print ">", line
                if not in_comment:
                    p = line.find('/*')
                    if p >= 0:
                        in_comment = True
                        line = line[:p]
                        if not line.strip():
                            continue
                #~ if not line.strip(): continue

                m = self.re_scanner.match(line)
                if m is None:
                    raise PreprocessorError("error: invalid preprocessing directive: {!r}".format(line))
                elif m.lastgroup == 'IF':
                    expression = m.group('IF_EXPR')
                    value = self.namespace.eval(expression)
                    self.log.debug("#if {!r} -> {!r}".format(expression, value))
                    hiddenstack.append((process, my_if_was_not_hidden, if_name, False))
                    if_name = expression
                    if process:
                        process = value
                        my_if_was_not_hidden = True
                    else:
                        my_if_was_not_hidden = False
                    continue
                elif m.lastgroup == 'ELIF':
                    # do what #else would do
                    if my_if_was_not_hidden:
                        process = not process
                    # then do what #if would do
                    expression = m.group('ELIF_EXPR')
                    value = self.namespace.eval(expression)
                    self.log.debug("#elif {!r} -> {!r}".format(expression, value))
                    # replace state on the stack
                    hiddenstack.append((process, my_if_was_not_hidden, if_name, True))
                    if_name = expression
                    if process:
                        process = value
                        my_if_was_not_hidden = True
                    else:
                        my_if_was_not_hidden = False
                    continue
                elif m.lastgroup == 'IFDEF':
                    symbol = m.group('IFDEF_NAME').strip()
                    value = symbol in self.namespace.defines
                    self.log.debug("#ifdef {!r} -> {!r}".format(symbol, value))
                    hiddenstack.append((process, my_if_was_not_hidden, if_name, False))
                    if_name = symbol
                    if process:
                        process = value
                        my_if_was_not_hidden = True
                    else:
                        my_if_was_not_hidden = False
                    continue
                elif m.lastgroup == 'IFNDEF':
                    symbol = m.group('IFNDEF_NAME').strip()
                    value = symbol not in self.namespace.defines
                    self.log.debug("#ifndef {!r} -> {!r}".format(symbol, value))
                    hiddenstack.append((process, my_if_was_not_hidden, if_name, False))
                    if_name = symbol
                    if process:
                        process = value
                        my_if_was_not_hidden = True
                    else:
                        my_if_was_not_hidden = False
                    continue
                elif m.lastgroup == 'ELSE':
                    self.log.debug("#else {!r}".format(if_name))
                    if my_if_was_not_hidden:
                        process = not process
                    continue
                elif m.lastgroup == 'ENDIF':
                    self.log.debug("#endif {!r}".format(if_name))
                    while True:
                        (process, my_if_was_not_hidden, if_name, implicit_endif) = hiddenstack.pop()
                        if not implicit_endif:
                            break
                    continue
                elif not process:
                    continue
                elif m.lastgroup == 'INCLUDE':
                    include_name = m.group('INC_NAME')
                    self.log.debug('including "{}"'.format(include_name))
                    for location in self.include_path:
                        path = os.path.normpath(os.path.join(location, include_name))
                        if os.path.exists(path):
                            if include_callback is not None:
                                include_callback(path)
                            self.preprocess(codecs.open(path, 'r', 'utf-8'), outfile, path, include_callback)
                            writer.marker = None  # force marker output
                            break
                    else:
                        raise PreprocessorError('include file {!r} not found'.format(include_name))
                    continue
                elif m.lastgroup == 'MACRO':
                    name = m.group('MACRO_NAME')
                    args = [x.strip() for x in m.group('MACRO_ARGS').split(',')]
                    if m.group('MACRO_DEF'):
                        definition = m.group('MACRO_DEF').strip()
                    else:
                        definition = ''
                    if name in self.macros:
                        self.log.warn("{!r} redefinition ignored".format(name))
                    else:
                        # prepare the macro value to be used as format string
                        # (python's % operator)
                        definition = definition.replace('%', '%%')
                        for arg in args:
                            definition = re.sub(
                                    r'(^|[^\w_])#({})([^\w_]|$)'.format(arg),
                                    r'\1"%({})s"\3'.format(hexlify(arg)),
                                    definition)
                            definition = re.sub(
                                    r'(^|[^\w_])({})([^\w_]|$)'.format(arg),
                                    r'\1%({})s\3'.format(hexlify(arg)),
                                    definition)
                        self.macros[name] = ([hexlify(x) for x in args], definition)
                        self.log.debug("defined macro {!r} => {!r}".format(name, self.macros[name]))
                    continue
                elif m.lastgroup == 'DEFINE':
                    symbol = m.group('DEF_NAME')
                    if m.group('DEF_VALUE'):
                        definition = m.group('DEF_VALUE').strip()
                    else:
                        definition = ''
                    if symbol in self.namespace.defines:
                        self.log.warn("{!r} redefinition ignored".format(symbol))
                    else:
                        self.namespace.defines[symbol] = definition
                        self.log.debug("defined {!r} => {!r}".format(symbol, self.namespace.defines[symbol]))
                    continue
                elif m.lastgroup == 'UNDEF':
                    symbol = m.group('UNDEF_NAME').strip()
                    self.log.debug("undefined {}".format(symbol))
                    del self.namespace.defines[symbol]
                    continue
                elif m.lastgroup == 'MESSAGE':
                    sys.stderr.write(u'{}:{}: message: {}\n'.format(
                            filename,
                            lineno,
                            line.strip(),))
                    continue
                elif m.lastgroup == 'WARNING':
                    sys.stderr.write(u'{}:{}: warning: {}\n'.format(
                            filename,
                            lineno,
                            line.strip(),))
                    continue
                elif m.lastgroup == 'ERROR':
                    sys.stderr.write(u'{}:{}: error: {}\n'.format(
                            filename,
                            lineno,
                            line.strip(),))
                    error_found = True
                    continue
                elif m.lastgroup == 'PRAGMA':
                    pass  # => line will be output below
                elif m.lastgroup == 'NONPREPROC':
                    line = self.expand(line)
                else:
                    raise PreprocessorError('Invalid input: {!r}'.format(line))

                if not line.strip():
                    empty_lines += 1
                    if empty_lines > 1:
                        continue
                else:
                    empty_lines = 0
                writer.write(lineno, line)
            # at the end of the loop, check if there were unbalanced #if's
            if hiddenstack:
                raise PreprocessorError('missing #endif')

        except PreprocessorError as e:
            # annotate exception with location in source file
            e.line = lineno
            e.filename = filename
            e.text = line
            self.log.info('error while processing "{}"'.format(line.strip()))
            raise
        except:
            self.log.info('error while processing "{}"'.format(line.strip()))
            raise
        else:
            self.log.info('done "{}"'.format(filename))
        return error_found

class Discard(object):
    """File like target object that consumes and discards all data"""
    def write(self, s):
        """dummy write"""

def main():
    import sys
    import argparse
    logging.basicConfig()

    parser = argparse.ArgumentParser()

    parser.add_argument(
        'SOURCE',
        type=argparse.FileType('r'),
        nargs='?',
        default='-')

    group = parser.add_argument_group('Input')

    group.add_argument(
        '-p', '--preload',
        help='process this file first. its output is discarded but definitions are kept.',
        metavar='FILE')

    group.add_argument(
        '-D', '--define',
        action='append',
        dest='defines',
        metavar='SYM[=VALUE]',
        default=[],
        help='define symbol')

    group.add_argument(
        '-I', '--include-path',
        action='append',
        metavar='PATH',
        default=[],
        help='Add directory to the search path list for includes')

    group = parser.add_argument_group('Output')

    group.add_argument(
        '-o', '--outfile',
        type=argparse.FileType('w'),
        help='name of the destination file',
        default='-',
        metavar='FILE')

    parser.add_argument(
        '--dependency-scan',
        action='store_true',
        default=False,
        help='just print names of includes, not actual content')

    parser.add_argument(
        '-v', '--verbose',
        action='store_true',
        default=False,
        help='print status messages to stderr')

    parser.add_argument(
        '--develop',
        action='store_true',
        default=False,
        help='print debug messages to stderr')

    args = parser.parse_args()

    if args.develop:
        logging.getLogger('cpp').setLevel(logging.DEBUG)
    elif args.verbose:
        logging.getLogger('cpp').setLevel(logging.INFO)
    else:
        logging.getLogger('cpp').setLevel(logging.WARN)

    cpp = Preprocessor()
    # extend include search path
    # built in places for msp430.asm
    d = os.path.join(os.path.dirname(sys.modules['msp430.asm'].__file__), 'include')
    cpp.include_path.append(d)
    cpp.include_path.append(os.path.join(d, 'upstream'))
    # user provided directories (-I)
    cpp.include_path.extend(args.include_path)
    # insert predefined symbols (XXX function like macros not yet supported)
    for definition in args.defines:
        if '=' in definition:
            symbol, value = definition.split('=', 1)
        else:
            symbol, value = definition, '1'
        cpp.namespace.defines[symbol] = value

    # process files now
    if args.preload:
        cpp.preprocess(open(args.preload), Discard(), args.preload)

    if args.dependency_scan:
        # add a callback that writes out filenames of includes
        # remember outfile as we're changing it below
        def print_include(path, outfile=args.outfile):
            outfile.write('{}\n'.format(path))
        args.outfile = Discard()  # discard following output
    else:
        print_include = None

    try:
        error_found = cpp.preprocess(args.SOURCE, args.outfile, args.SOURCE.name, print_include)
        if error_found:
            sys.exit(2)
    except PreprocessorError as e:
        sys.stderr.write(u'{e.filename}:{e.line}: {e}\n'.format(e=e))
        if args.develop:
            if hasattr(e, 'text'):
                sys.stderr.write(u'{e.filename}:{e.line}: input line: {e.text!r}\n'.format(e=e))
        sys.exit(1)

if __name__ == '__main__':
    main()


import json
import logging
import os
import subprocess
import sys
import threading
import uuid
from queue import Queue
import time
import errno

import gi
gi.require_version('GLib', '2.0')
from gi.repository import GLib

try:
    from pydbus import SessionBus, SystemBus
    from pydbus.generic import signal
except ImportError as err:
    msg = "Can't import pydbus library: {}".format(err)
    logging.error(msg)
    print(msg)
    sys.exit(-1)

try:
    from pacman import pac
except ImportError as err:
    logging.error(err.msg)
    msg = "Can't find {} bindings. Unable to install/uninstall apps".format(
        err.name)
    logging.error(msg)
    sys.exit(-1)

INTERFACE = 'com.antergos.welcome'

class DBusService(object):
    """
    <node>
        <interface name='com.antergos.welcome'>
            <method name='get_package_exists'>
                <arg type='s' name='package_name' direction='in'/>
                <arg type='s' name='response' direction='out'/>
            </method>
            <method name='check_updates'>
                <arg type='as' name='response' direction='out'/>
            </method>
            <method name='refresh_alpm'>
                <arg type='s' name='uid' direction='out'/>
            </method>
            <method name='install_package'>
                <arg type='s' name='package_name' direction='in'/>
                <arg type='s' name='response' direction='out'/>
            </method>
            <method name='remove_package'>
                <arg type='s' name='package_name' direction='in'/>
                <arg type='s' name='response' direction='out'/>
            </method>
            <method name='install_packages'>
                <arg type='as' name='package_names' direction='in'/>
                <arg type='as' name='response' direction='out'/>
            </method>
            <method name='system_upgrade'>
                <arg type='as' name='response' direction='out'/>
            </method>
            <method name='exit'/>
            <method name='is_alpm_on'>
                <arg type='b' name='response' direction='out'/>
            </method>
            <method name='is_package_installed'>
                <arg type='s' name='package_name' direction='in'/>
                <arg type='b' name='response' direction='out'/>
            </method>
            <property name="command_finished" type="(ssas)" access="readwrite">
                <annotation name="org.freedesktop.DBus.Property.EmitsChangedSignal" value="true"/>
            </property>
        </interface>
    </node>
    """

    def __init__(self, mainloop, object_path="/com/antergos/welcome"):
        self.alpm = None
        self.updates_available = self.store_loaded = False
        self.mainloop = mainloop
        self._command_finished = ()

        # We will store package metadata before its needed to improve
        # performance on the frontend.
        self.all_packages = {}

        self.initialize_alpm()

        # File lock db.lck (pacman)
        # Times are in seconds
        self.is_locked = False
        self.lock_file = "/var/lib/pacman/db.lck"
        self.lock_timeout = 30
        self.lock_delay = 1

        # lock to serialize alpm petitions (install/uninstall)
        self.lock = threading.Lock()

        self.command_queue = Queue()
        t = threading.Thread(target=self._command_queue_worker)
        t.daemon = True
        t.start()

    def initialize_alpm(self):
        try:
            self.alpm = pac.Pac()
            logging.debug("Alpm library initialized")
        except Exception as err:
            logging.error("Cannot initialize alpm library")
            sys.exit(-1)

    @staticmethod
    def get_uuid():
        return str(uuid.uuid1())

    # DBus methods -------------------------------------------------------------

    def get_package_exists(self, package_name):
        """ Checks for package in ALPM database. Return True if found, otherwise False. """
        pkg = self.alpm.get_package_info(package_name)
        return pkg is not {}

    def check_updates(self, dbus_context):
        """ Check for available updates. """
        if self.is_authorized(dbus_context):
            return self._check_updates()
        else:
            return []

    def is_alpm_on(self, dbus_context):
        if self.is_authorized(dbus_context):
            return bool(self.alpm)
        else:
            return False

    def is_package_installed(self, package_name):
        """ Return if the given package is installed. """
        return bool(self.alpm.is_package_installed(str(package_name)))

    def refresh_alpm(self, dbus_context):
        """ Refreshes alpm databases """
        if self.is_authorized(dbus_context):
            uid = self.get_uuid()
            self.command_queue.put((uid, 'refresh', []))
            return uid
        else:
            return ""

    def install_package(self, package_name, dbus_context):
        """ Install the given package. """
        if self.is_authorized(dbus_context):
            uid = self.get_uuid()
            self.command_queue.put((uid, 'install', [package_name]))
            return uid
        else:
            return ""

    def remove_package(self, package_name, dbus_context):
        """ Uninstall the given package. """
        if self.is_authorized(dbus_context):
            uid = self.get_uuid()
            self.command_queue.put((uid, 'remove', [package_name]))
            return uid
        else:
            return ""

    def install_packages(self, package_names, dbus_context):
        """ Install updates """
        if self.is_authorized(dbus_context):
            uid = self.get_uuid()
            self.command_queue.put(
                (uid, 'install_packages', list(package_names)))
            return uid
        else:
            return ""

    def system_upgrade(self, dbus_context):
        """ Install updates """
        if self.is_authorized(dbus_context):
            uid = self.get_uuid()
#            self.command_queue.put((uid, 'refresh', []))
            self.command_queue.put((uid, 'system_upgrade', []))
            return uid
        else:
            return ""

    def exit(self, dbus_context):
        if self.is_authorized(dbus_context) and self.mainloop:
            self.mainloop.quit()

    # DBus signals -------------------------------------------------------------
    @property
    def command_finished(self):
        return self._command_finished

    @command_finished.setter
    def command_finished(self, value):
        self._command_finished = value
        logging.debug("command_finished")
        # print(value)
        self.PropertiesChanged("com.antergos.welcome", {
                               "command_finished": self.command_finished}, [])

    PropertiesChanged = signal()

    # Internal alpm methods ----------------------------------------------------

    def _check_updates(self):
        """ Check for available updates. """
        updates = self.alpm.check_updates()
        logging.info(updates)
        if not updates:
            updates = []
        return updates

    def _refresh_alpm(self):
        with self.lock:
            logging.info("Refreshing databases...")
            try:
                self.alpm.refresh()
            except Exception as general_error:
                logging.error(general_error)

    def _install_package(self, package):
        with self.lock:
            logging.info("Installing %s", package)
            try:
                self.alpm.install([str(package)])
            except Exception as general_error:
                logging.error(general_error)

    def _remove_package(self, package):
        with self.lock:
            logging.info("Removing %s", package)
            try:
                self.alpm.remove([str(package)])
            except Exception as general_error:
                logging.error(general_error)

    def _install_packages(self, packages):
        with self.lock:
            logging.info("Installing %s", packages)
            try:
                self.alpm.install(packages)
            except Exception as general_error:
                logging.error(general_error)

    def _system_upgrade(self):
        with self.lock:
            logging.info("Full system upgrade...")
            try:
                self.alpm.system_upgrade()
            except Exception as general_error:
                logging.error(general_error)

    def _command_queue_worker(self):
        while True:
            if self.lock_ok():
                uid, command, packages = self.command_queue.get()
                if command == 'install':
                    self._install_package(packages[0])
                    # Send signal to frontends
                    self.command_finished = (uid, command, packages)
                elif command == 'install_packages':
                    self._install_packages(packages)
                    self.command_finished = (uid, command, packages)
                elif command == 'remove':
                    self._remove_package(packages[0])
                    self.command_finished = (uid, command, packages)
                elif command == 'refresh':
                    self._refresh_alpm()
                    self.command_finished = (uid, command, packages)
                elif command == 'check_updates':
                    output = self._check_updates()
                    self.command_finished = (uid, command, packages)
                elif command == 'system_upgrade':
                    self._system_upgrade()
                    self.command_finished = (uid, command, packages)
                elif command == 'frontend_loaded':
                    self._do_frontend_loaded()
                    self.command_finished = (uid, command, packages)
                else:
                    logging.error(_("Unknown command %s"), command)

    # Polkit -------------------------------------------------------------------

    def is_authorized(self, dbus_context, interactive=True):
        action_id = "com.antergos.welcome.install"
        details = {
            'polkit.icon': 'antergos-welcome',
            'polkit.message': 'antergos-welcome'}

        return dbus_context.check_authorization(action_id, details, interactive=True)

    # db.lck -------------------------------------------------------------------

    def lock_ok(self):
        if self._is_lock_available():
            return True
        # Lock exists, we will try to acquire it (waiting)
        if self._acquire_lock():
            # Yes! We got it! Delete it.
            self._release_lock()
            return True
        logging.error("db.lck exists and is owned by another process!")
        return False

    def _is_lock_available(self):
        """ Returns True if the file is currently available to be locked. """
        return not os.path.exists(self.lock_file)

    def _acquire_lock(self, delay=1, timeout=10):
        """
        Acquire the lock, if possible. Otherwise, check again every `delay`
        seconds until it either gets the lock or exceeds `timeout` number of
        seconds.
        """
        start_time = time.time()
        while True:
            try:
                # Attempt to create the lockfile.
                # These flags cause os.open to raise an OSError if the file already exists.
                fd = os.open(self.lock_file, os.O_CREAT |
                             os.O_EXCL | os.O_RDWR)
                break
            except OSError as err:
                if err.errno != errno.EEXIST:
                    logging.error("Cannot acquire lock!")
                    return False
                if (time.time() - start_time) >= timeout:
                    logging.error("Lock timeout!")
                    return False
                # wait delay seconds and try again
                time.sleep(delay)
        self.is_locked = True
        return True

    def _release_lock(self):
        """ Get rid of the lock by deleting the lockfile. """
        self.is_locked = False
        os.unlink(self.lockfile)


from tkinter import *
from PIL import ImageTk, Image
import subprocess

# widget =GUI elements: button, textboxes,labels, images
# windows= serves as a container to hold or contain these widgets

window = Tk()  # instantiate an instance of a window
window.geometry("1000x650")
window.title("Prodigy's Pursuit")

logo = PhotoImage(file='Grand Logo.png')
window.iconphoto(True, logo)
window.config(background="#E7C6FF")

img = ImageTk.PhotoImage(Image.open("About.png"))
label = Label(window, image=img)
label.pack()
label.place(x=0, y=0)

label = Label(window,
              text="About",
              font=('Grand Royal', 22),
              fg='#3A0CA3',
              bg='#E5B3FE')
label.pack()
label.place(x=60, y=80)

label = Label(window,
              text="Prodigy's Pursuit",
              font=('Grand Royal', 32),
              fg='#3A0CA3',
              bg='#E5B3FE')
label.pack()
label.place(x=60, y=120)

label = Label(window,
              text="This career counselling platform for engineers is\n"
                   "designed to help users explore different engineering\n"
                   "career paths and make informed decisions about their\n"
                   "professional futures. It provides personalized career\n "
                   "guidance based on the user's qualifications, skills, and\n"
                   "interests, along with insights into the current job\n"
                   "market,educational resources, and networking \n"
                   "opportunities.Overall, it is a comprehensive \n"
                   "and user-friendly platform for engineers \n"
                   "to receive guidance and support \n"
                   "throughout their professional journeys.",
              justify="left",
              font=('Cambria', 12),
              fg='#3A0CA3',
              bg='#E5B3FE')
label.pack()
label.place(x="60", y=200)

submit_button = Button(window,
                       text="BACK",
                       font=("Jacques Francois", 15),
                       fg="#E7C6FF",
                       bg="#F72585",
                       activeforeground="#E7C6FF",
                       activebackground="#7209B7",
                       state=ACTIVE)
submit_button.pack(side=RIGHT)
submit_button.place(x=60, y=453)

def on_button_click():
    # Close the current file
    window.destroy()
    # Open the new file
    subprocess.Popen(["python", "HomePage.py"])

button = Button(window,
                text=" ← ",
                command=on_button_click,
                font=("Montserrat", 15),
                fg="#E7C6FF",
                bg="#F72585",
                activeforeground="#E7C6FF",
                activebackground="#7209B7",
                state=ACTIVE)
button.pack()
button.place(x=950, y=10)

window.mainloop()


from setuptools import setup

package_name = 'teleop_joy'

setup(
    name=package_name,
    version='0.0.0',
    packages=[package_name],
    data_files=[
        ('share/ament_index/resource_index/packages',
            ['resource/' + package_name]),
        ('share/' + package_name, ['package.xml']),
    ],
    install_requires=['setuptools'],
    zip_safe=True,
    maintainer='turtle01',
    maintainer_email='turtle01@todo.todo',
    description='TODO: Package description',
    license='TODO: License declaration',
    tests_require=['pytest'],
    entry_points={
        'console_scripts': [
        'remote_joy     = teleop_joy.remote_joy:main',
        ],
    },
)


import subprocess
import logging
import sys
from os import walk
from collections import defaultdict
import matplotlib as mpl
import matplotlib.pyplot as plt
from statistics import mean

FORMAT = '%(asctime)-15s - %(levelname)s - %(module)10s:%(lineno)-5d - %(message)s'
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format=FORMAT)
LOG = logging.getLogger(__name__)

def read_file(file_name):
    f = open(file_name, "r")
    result = {}
    idx = -1
    for l in f:
        if "master" in l:
            idx = int(l.split()[1])

        else:
            if idx != -1:
                result[idx] = float(l.split()[2])
                idx = -1

    return mean(result.values())

def plot(w_num):
    data = defaultdict(lambda : [[], []])
    fs = []
    dir_path = "result/100k/"
    for (dirpath, dirnames, filenames) in walk(dir_path):
        fs.extend(filenames)

    for f in fs:
        cats = f.split("&")
        hash_type = cats[0]
        m = int(cats[1].split("m")[0])
        w = int(cats[2].split("w")[0])

        if w == w_num:
            data[hash_type][0].append(m)
            val = read_file(dir_path + f)
            data[hash_type][1].append(val)

    plt.xlabel('Number of masters')
    plt.ylabel('Calculation time')

    plot_line = ['ro-', 'go-']
    for idx, (k, v) in enumerate(data.items()):
        v = list(zip(*sorted(zip(v[0], v[1]))))
        plt.plot(v[0], v[1], plot_line[idx], label=k)

    leg = plt.legend(loc='best', ncol=2, mode="expand", shadow=True, fancybox=True)
    leg.get_frame().set_alpha(0.3)

    plt.savefig(f'plot/result_w{w_num}.png')
    plt.figure().clear()
    plt.close()
    plt.cla()
    plt.clf()

def plot2(m_num):
    data = defaultdict(lambda : [[], []])
    fs = []
    dir_path = "result/100k/"
    for (dirpath, dirnames, filenames) in walk(dir_path):
        fs.extend(filenames)
    for f in fs:
        cats = f.split("&")
        hash_type = cats[0]
        m = int(cats[1].split("m")[0])
        w = int(cats[2].split("w")[0])
        if m == m_num:
            data[hash_type][0].append(w)
            val = read_file(dir_path + f)
            data[hash_type][1].append(val)

    plt.xlabel('Number of workers')
    plt.ylabel('Speed up')

    plot_line = ['ro-', 'go-']
    for idx, (k, v) in enumerate(data.items()):
        v = list(zip(*sorted(zip(v[0], v[1]))))
        v[1] = [(v[1][0] / val) for val in v[1]]
        plt.plot(v[0], v[1], plot_line[idx], label=k)

    leg = plt.legend(loc='best', ncol=2, mode="expand", shadow=True, fancybox=True)
    leg.get_frame().set_alpha(0.3)

    plt.savefig(f'plot/result_m{m_num}.png')
    plt.figure().clear()
    plt.close()
    plt.cla()
    plt.clf()

if __name__ == '__main__':
    LOG.info("start tracing\n")
    w_num = [1]
    m_num = [1, 2, 4, 8]
    for w in w_num:
        plot(w)
    for m in m_num:
        plot2(m)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" Model for fitting an absorption profile to spectral data. """

from __future__ import (division, print_function, absolute_import,
                        unicode_literals)

__all__ = ["ProfileFittingModel"]

import logging
import numpy as np
import scipy.optimize as op
from astropy import units as u
from astropy.table import Row
from astropy.constants import c as speed_of_light
from collections import OrderedDict
from scipy.special import wofz
from scipy import integrate

from .base import BaseSpectralModel

logger = logging.getLogger(__name__)

def _gaussian(x, *parameters):
    """
    Evaluate a Gaussian profile at x, given the profile parameters.

        y = amplitude * exp(-(x - position)**2 / (2.0 * sigma**2))

    :param x:
        The x-values to evaluate the Gaussian profile at.

    :param parameters:
        The position, sigma, and amplitude of the Gaussian profile.
    """
    position, sigma, amplitude = parameters
    return amplitude * np.exp(-(x - position)**2 / (2.0 * sigma**2))

def _lorentzian(x, *parameters):
    """
    Evaluate a Lorentzian profile at x, given the profile parameters:

        y = (amplitude/PI) * (width/((x - positions)**2 + width**2))

    :param x:
        The x-values to evaluate the Lorentzian profile at.

    :param parameters:
        The position, width, and amplitude of the Lorentzian profile.
    """
    position, width, amplitude = parameters
    return (amplitude/np.pi) * (width/((x - position)**2 + width**2))

def _voigt(x, *parameters):
    """
    Evaluate a Voigt profile at x, given the profile parameters.

    :param x:
        The x-values to evaluate the Voigt profile at.

    :param parameters:
        The position, fwhm, amplitude, and shape of the Voigt profile.
    """
    try:
        n = len(x)
    except TypeError:
        n = 1

    position, fwhm, amplitude, shape = parameters

    profile = 1 / wofz(np.zeros((n)) + 1j * np.sqrt(np.log(2.0)) * shape).real
    profile *= amplitude * wofz(2*np.sqrt(np.log(2.0)) * (x - position)/fwhm \
        + 1j * np.sqrt(np.log(2.0))*shape).real
    return profile

class ProfileFittingModel(BaseSpectralModel):

    _profiles = {
        "gaussian": (_gaussian, ("mean", "sigma", "amplitude")),
        "lorentzian": (_lorentzian, ("mean", "width", "amplitude")),
        "voigt": (_voigt, ("mean", "fwhm", "amplitude", "shape"))
    }

    def __init__(self, transitions, **kwargs):
        """
        Initialize a class for modelling an atomic transition using a profile.
        """

        super(ProfileFittingModel, self).__init__(transitions, **kwargs)

        # Initialize metadata with default fitting values.
        self.metadata.update({
            "profile": "gaussian",
            "central_weighting": True,
            "window": 5,
            "continuum_order": 0,
            "detection_sigma": 0.5,
            "detection_pixels": 3,
            "max_iterations": 5,
            "wavelength_tolerance": 0.25,
            "velocity_tolerance": None,
            "mask": [],
            "antimask_flag": False,
            "elements": [self._verify_elements()],
            "species": [self._verify_species()]
        })
        # Update metadata with settings,....?
        for key, value in kwargs.items():
            if key in self.metadata:
                self.metadata[key] = value
            else:
                logging.warn("Ignoring keyword {} for {}".format(
                    key, self.__class__.__name__))

        # Set the model parameter names based on the current metadata.
        self._update_parameter_names()
        self._verify_transitions()
        self._verify_metadata()

        # Create a _repr_element for this.
        self._repr_element = self.transitions["element"][0]

        return None

    def _verify_transitions(self):
        """
        Verify that the atomic or molecular transitions associated with this
        class are valid.
        """

        # Check format first.
        transitions = self.transitions
        super(ProfileFittingModel, self)._verify_transitions()
        if len(transitions) > 1 and not isinstance(transitions, Row):
            raise ValueError("only a single transition can be associated with "
                             "a ProfileFittingModel")

        # Check that the transition does not have multiple element names.
        try:
            elem2 = transitions["elem2"][0]
        except IndexError:
            elem2 = transitions["elem2"]
        if elem2 != "":
            raise ValueError("only an atomic transition can be associated with "
                             "a ProfileFittingModel")
        return True

    def _verify_elements(self):
        """
        Return the element that will be measured by this model.
        """
        return self.transitions["element"][0].split()[0]

    def _verify_species(self):
        """
        Return the species that will be measured by this model.
        Ignore isotopes.
        """
        return np.floor(self.transitions["species"][0]*10)/10

    def _verify_metadata(self):
        """
        Verify the metadata associated with this class.
        """
        # TODO
        return True

    def _update_parameter_names(self):
        """
        Update the model parameter names based on the current metadata.
        """

        # Profile parameter names.
        func, parameter_names = self._profiles[self.metadata["profile"]]
        parameter_names = list(parameter_names)

        # Continuum coefficients.
        parameter_names += ["c{0}".format(i) \
            for i in range(self.metadata["continuum_order"] + 1)]

        # Update the bounds.
        bounds = {}
        if self.metadata["profile"] == "gaussian":
            bounds.update({
                "sigma": (-0.5, 0.5),
                "amplitude": (0, 1),
            })

        elif self.metadata["profile"] == "voigt":
            bounds["fwhm"] = (-0.5, 0.5)

        if self.metadata["wavelength_tolerance"] is not None \
        or self.metadata["velocity_tolerance"] is not None:

            # Convert velocity tolerance into wavelength.
            wavelength = self.transitions["wavelength"]
            try:
                wavelength = wavelength[0]
            except IndexError:
                None
            vt = abs(self.metadata.get("velocity_tolerance", None) or np.inf)
            wt = abs(self.metadata.get("wavelength_tolerance", None) or np.inf)

            bound = np.nanmin([
                wt, wavelength * vt/speed_of_light.to("km/s").value])

            bounds["mean"] = (wavelength - bound, wavelength + bound)

        else:
            # TODO: Allow the wavelength to be fixed.
            raise NotImplementedError("wavelength cannot be fixed yet; "
                                      "set a small tolerance on wavelength")

        self._parameter_names = parameter_names
        self._parameter_bounds = bounds

        return True

    def _initial_guess(self, spectrum, **kwargs):
        """
        Generate an initial guess for the model parameters.

        :param spectrum:
            The observed spectrum.
        """

        wavelength = self.transitions["wavelength"]
        try:
            wavelength = wavelength[0]
        except IndexError:
            None

        p0 = [
            wavelength,
            kwargs.pop("p0_sigma", 0.1),
        ]

        if spectrum is None:
            p0.append(0.5)
        else:
            p0.append(1.0 - \
                spectrum.flux[spectrum.dispersion.searchsorted(wavelength)])

        if self.metadata["profile"] == "voigt":
            p0.append(kwargs.pop("p0_shape", 0))

        # Continuum?
        if self.metadata["continuum_order"] > -1:
            p0.extend(([0] * self.metadata["continuum_order"]) + [1])

        return np.array(p0)

    def fit(self, spectrum, **kwargs):
        """
        Fit an absorption profile to the transition in the spectrum.
        """

        spectrum = self._verify_spectrum(spectrum)

        failure = False

        # Update internal metadata with any input parameters.
        # Ignore additional parameters because other BaseSpectralModels will
        # have different input arguments.
        for key in set(self.metadata).intersection(kwargs):
            self.metadata[key] = kwargs[key]

        # What model parameters are in the fitting process?
        # In synth these would be abundances, etc. Here they are profile/cont
        # parameters.
        self._update_parameter_names()

        # Get a bad initial guess.
        p0 = self._initial_guess(spectrum, **kwargs)

        # Build a mask based on the window fitting range, and prepare the data.
        mask = self.mask(spectrum)
        x, y = spectrum.dispersion[mask], spectrum.flux[mask]
        yerr, absolute_sigma = ((1.0/spectrum.ivar[mask])**0.5, True)
        if not np.all(np.isfinite(yerr)):
            yerr, absolute_sigma = (np.ones_like(x), False)

        # Central weighting?
        if self.metadata["central_weighting"]:
            yerr /= (1 + np.exp(-(x - p0[0])**2 / (4.0 * p0[1]**2)))

        # How many iterations to do?
        nearby_lines = []
        iterative_mask = np.isfinite(y * yerr)
        for iteration in range(self.metadata["max_iterations"]):

            if not any(iterative_mask):
                self.metadata["is_acceptable"] = False
                try:
                    del self.metadata["fitted_result"]
                except KeyError:
                    None

                return failure

            try:
                p_opt, p_cov = op.curve_fit(self.fitting_function,
                    xdata=x[iterative_mask],
                    ydata=y[iterative_mask],
                    sigma=yerr[iterative_mask],
                    p0=p0, absolute_sigma=absolute_sigma, maxfev=10000)

            except:
                logger.exception(
                    "Exception raised in fitting atomic transition {0} "\
                    "on iteration {1}".format(self, iteration))

                if iteration == 0:
                    self.metadata["is_acceptable"] = False
                    try:
                        del self.metadata["fitted_result"]
                    except KeyError:
                        None

                    return failure

            # Look for outliers peaks.
            # TODO: use continuum or model?
            O = self.metadata["continuum_order"]
            continuum = np.ones_like(x) \
                if 0 > O else np.polyval(p_opt[-(O + 1):], x)

            model = self(x, *p_opt)

            sigmas = (y - model)/np.std(y[iterative_mask])
            sigmas[~iterative_mask] = 0 # Ignore points that are already masked
            outliers = np.where(sigmas < -self.metadata["detection_sigma"])[0]

            # Look for groups of neighbouring outlier points.
            separators = np.repeat(1 + np.where(np.diff(outliers) > 1)[0], 2)
            separators = np.hstack([0, separators, None]).reshape(-1, 2)

            for start, end in separators:
                indices = outliers[start:end]

                # Require a minimum group size by number of pixels.
                if indices.size < self.metadata["detection_pixels"]: continue

                # Try and fit an absorption function to the centroid of the
                # region.

                lower_group_wl = x[indices[0]]
                upper_group_wl = x[indices[-1]]

                def model_nearby_line(x_, *p):
                    # Strict requirements on these parameters, since we don't
                    # need to know them precisely.
                    if not (lower_group_wl <= p[0] <= upper_group_wl) \
                    or abs(p[1]) > p_opt[1] \
                    or not (1 >= p[2] > 0):
                        return np.nan * np.ones_like(x_)

                    return model[iterative_mask] * self(x_, *p)

                # Initial parameters for this line.
                p0_outlier = [
                    np.mean(x[indices]),
                    0.5 * p_opt[1],
                    np.max(continuum[indices] - y[indices])
                ]
                if self.metadata["profile"] == "voigt":
                    p0_outlier.append(p0[3])

                try:
                    p_out, cov = op.curve_fit(model_nearby_line,
                            xdata=x[iterative_mask],
                            ydata=y[iterative_mask],
                            sigma=yerr[iterative_mask],
                            p0=p0_outlier, absolute_sigma=absolute_sigma,
                            check_finite=True)

                except:
                    # Just take a narrow range and exclude that?
                    # TODO: The old SMH just did nothing in this scenario, but
                    #       we may want to revise that behaviour.
                    None

                else:

                    # Exclude +/- 3 sigma of the fitted line
                    l, u = (p_out[0] - 3 * p_out[1], p_out[0] + 3 * p_out[1])

                    # Store this line and the region masked out by it.
                    nearby_lines.append([p_out, (u, l)])

                    # Now update the iterative mask to exclude this line.
                    iterative_mask *= ~((u > x) * (x > l))

            # Update p0 with the best guess from this iteration.
            p0 = p_opt.copy()

        # Finished looking for neighbouring lines
        # `max_iterations` rounds of removing nearby lines. Now do final fit:
        p_opt, p_cov = op.curve_fit(self.fitting_function,
            xdata=x[iterative_mask],
            ydata=y[iterative_mask],
            sigma=yerr[iterative_mask],
            p0=p0, absolute_sigma=absolute_sigma)

        assert p_cov is not None

        # Make many draws from the covariance matrix.
        draws = kwargs.pop("covariance_draws", 100)
        percentiles = kwargs.pop("percentiles", (2.5, 97.5))

        if np.all(np.isfinite(p_cov)):
            p_alt = np.random.multivariate_normal(p_opt, p_cov, size=draws)
        else:
            p_alt = np.nan * np.ones((draws, p_opt.size))

        # Integrate the profile.
        profile, _ = self._profiles[self.metadata["profile"]]
        if profile == _gaussian:
            ew = abs(p_opt[1] * p_opt[2] * np.sqrt(2 * np.pi))
            ew_alt = np.abs(p_alt[:, 1] * p_alt[:, 2] * np.sqrt(2 * np.pi))
            ew_uncertainty = np.percentile(ew_alt, percentiles) - ew

        else:
            N, integrate_sigma = (len(_), kwargs.pop("integrate_sigma", 10))
            l, u = (
                p_opt[0] - integrate_sigma * p_opt[1],
                p_opt[0] + integrate_sigma * p_opt[1]
            )

            ew = np.abs(integrate.quad(profile, l, u, args=tuple(p_opt[:N]))[0])
            ew_alt = np.abs([integrate.quad(profile, l, u, args=tuple(_[:N]))[0] \
                             for _ in p_alt])
            ew_uncertainty = np.percentile(ew_alt, percentiles) - ew

        # Calculate chi-square for the points that we modelled.
        ivar = spectrum.ivar[mask]
        if not np.any(np.isfinite(ivar)): ivar = 1
        residuals = y - self(x, *p_opt)
        residuals[~iterative_mask] = np.nan
        chi_sq = residuals**2 * ivar

        dof = np.isfinite(chi_sq).sum() - len(p_opt) - 1
        chi_sq = np.nansum(chi_sq)

        model_y = self(x, *p_opt)
        model_yerr = np.percentile(
            [self(x, *_) for _ in p_alt], percentiles, axis=0) - model_y
        model_yerr = np.max(np.abs(model_yerr), axis=0)

        """
        # DEBUG PLOT
        fig, ax = plt.subplots()
        ax.plot(x, y, c='k', drawstyle='steps-mid')

        O = self.metadata["continuum_order"]
        bg = np.ones_like(x) if 0 > O else np.polyval(p_opt[-(O + 1):], x)
        for p, (u, l) in nearby_lines:
            bg *= self(x, *p)

            m = (u >= x) * (x >= l)
            ax.scatter(x[m], y[m], facecolor="r")

        ax.plot(x, bg, c='r')
        ax.plot(x, model_y, c='b')

        model_err = np.percentile(
            [self(x, *_) for _ in p_alt], percentiles, axis=0)

        ax.fill_between(x, model_err[0] + model_y, model_err[1] + model_y,
            edgecolor="None", facecolor="b", alpha=0.5)
        """

        # Convert x, model_y, etc back to real-spectrum indices.
        x, model_y, model_yerr, residuals = self._fill_masked_arrays(
            spectrum, x, model_y, model_yerr, residuals)

        # We ignore the uncertainty in wavelength position because it only
        # affects the uncertainty in REW at the ~10^-5 level.
        rew = np.log10(ew/p_opt[0])
        rew_uncertainty = np.log10((ew + ew_uncertainty)/p_opt[0]) - rew

        fitting_metadata = {
            "equivalent_width": (ew, ew_uncertainty[0], ew_uncertainty[1]),
            "reduced_equivalent_width": np.hstack([rew, rew_uncertainty]),
            "data_indices": np.where(mask)[0][iterative_mask],
            "model_x": x,
            "model_y": model_y,
            "model_yerr": model_yerr,
            "residual": residuals,
            "nearby_lines": nearby_lines,
            "chi_sq": chi_sq,
            "dof": dof
        }

        # Update the equivalent width in the transition.
        # REMOVED: see Issue #38
        #self.transitions["equivalent_width"] = ew

        # Convert p_opt to ordered dictionary
        named_p_opt = OrderedDict(zip(self.parameter_names, p_opt))
        self.metadata["fitted_result"] = (named_p_opt, p_cov, fitting_metadata)

        # Only mark as acceptable if the model meets the quality constraints.
        self.is_acceptable = True
        #self.meets_quality_constraints_in_parent_session

        return self.metadata["fitted_result"]

    def __call__(self, dispersion, *parameters):
        """
        Generate data at the dispersion points, given the parameters.

        :param dispersion:
            An array of dispersion points to calculate the data for.

        :param parameters:
            Keyword arguments of the model parameters and their values.
        """

        function, profile_parameters = self._profiles[self.metadata["profile"]]

        N = len(profile_parameters)
        y = 1.0 - function(dispersion, *parameters[:N])

        # Assume rest of the parameters are continuum coefficients.
        if parameters[N:]:
            y *= np.polyval(parameters[N:], dispersion)

        return y

    @property
    def equivalent_width(self):
        ew, ew_pos, ew_neg = self.metadata["fitted_result"][2]["equivalent_width"]
        return (ew * u.Angstrom,
                ew_pos * u.Angstrom,
                ew_neg * u.Angstrom)


from django.http import JsonResponse
from django.shortcuts import render
from ..config.getData import getAPageData, get_nid, getCurrentPage, getPageDividerString

def shopping_index(request):        # /shopping/?currentPage=0
    currentPage = getCurrentPage(request.GET.get("currentPage"))
    pageDivideString = getPageDividerString(currentPage=currentPage)
    nid = get_nid()
    request.session["nidnidnid"] = nid
    return render(request, "shopping_index.html", {
        "currentPage": currentPage, "pageDivideString": pageDivideString, "nid": nid,
    })

def getDataApi(request):            # /shopping/getDataApi/
    try:
        assert request.method == "POST", "此接口仅限于接受post请求"
        assert request.POST.get("nid") != None, "缺少必要参数"
        assert request.session.get("nidnidnid", "") == request.POST.get("nid"), "参数校验不正确"
        currentPage = getCurrentPage(request.POST.get("currentPage"))
        result = getAPageData(currentPage)
        request.session.clear()
        return JsonResponse(result, json_dumps_params={'ensure_ascii': False}, )
    except Exception as error:
        return JsonResponse({
            "code": 404, "message": f"{error},获取数据失败",
        }, json_dumps_params={'ensure_ascii': False}, )


import igraph
import matplotlib
import numpy as np
from igraph import Graph
from numpy.core.fromnumeric import mean, size
import matplotlib.pyplot as plt
from numpy.random.bit_generator import SeedlessSeedSequence

from game import HDG

class Lattice2d:
    """
    Class use to create and manipulate 2d lattices of individuals of 2 types (here in our case dove and hawk).

    Parameters
    ----------
    width : int, dim of the 2dlattice
    height : int, dim of the 2dlattice
    radius : int, radius use to generate edges ine the lattice (cf lattice)
    """

    def __init__(self, width: int, height: int, radius: int) -> None:
        self.w = width
        self.h = height
        self.graph = Graph.Lattice(dim=[height,width],circular=True,nei= radius)

    def fill_graph(self, population: list) -> None:
        """
        Fill the 2d graph by the sample population sent as parameter.

        Parameters
        ----------
        population : list
            Population is a list of D/H that will be placed in the graph vertices dictionnary under the key value.
        """
        if( len(population) != self.w*self.h):
            raise NameError("The given population does not match the size of the graph")
        self.graph.vs["value"] = population

    def get_neighborhood_conformation(self, id: int) -> tuple[int,int]:
        """
        Return a tuple of int containing the number of Hawk and  the number of Dove in the neighborhood

        Parameters
        ----------
        id : list, id of the vertice at for which we want to know the neighborhood.
        """
        if( (id > self.w*self.h-1) or id<0):
            raise NameError("Id is invalid")

        neis = self.graph.neighbors(id)
        vertices = self.graph.vs
        nb_dove = 0
        nb_hawk = 0

        for elem in neis:
            nb_dove += vertices[elem].attributes()["value"]
        nb_hawk = len(neis)-nb_dove

        return (nb_hawk, nb_dove)

    def change_edges(self, p: float) -> None:
        """
        Update the graph by selecting each edge and changing it by an edge between two random vertices with a proba p.
        We will not choose edges that leaves isolated points.

        Parameters
        ----------
        p : float, proba of changing an edge
        """
        edges = []
        for elem in self.graph.es:
            edges.append(elem.tuple)

        id_limit = len(self.graph.vs)
        neighbors_numbers = [len(self.get_neighbors(i)) for i in range(id_limit)]
        replaced_edges = []
        new_edges = []
        for id, e in enumerate(edges):
            if(np.random.random() <= p):
                replaced_edges.append(e)
                end = False
                id_v1 = 0
                id_v2 = 0
                edge = (0,0)
                while( not end):
                    id_v1 = np.random.randint(0,id_limit)
                    id_v2 = np.random.randint(0,id_limit)
                    edge = (id_v1, id_v2)
                    if( (id_v1 != id_v2) and (not edge in edges) and (not edge in new_edges)):
                        end = True
                        new_edges.append(edge)

        for new_edge,edge in zip(new_edges,replaced_edges):
            # We make sure that no point is left alone without edge
            if ( (neighbors_numbers[edge[0]] > 1) and (neighbors_numbers[edge[1]] > 1) ):
                self.graph.delete_edges([edge])
                self.graph.add_edges([new_edge])
                # We update the number of neighbors
                neighbors_numbers[edge[0]] -= 1
                neighbors_numbers[edge[1]] -= 1
                neighbors_numbers[new_edge[0]] += 1
                neighbors_numbers[new_edge[1]] += 1

    def get_neighbors(self, index: int) -> list:
        return self.graph.neighbors(index)

    def get_vertices(self):
        return self.graph.vs

    def print_vertices(self) -> None:
        for v in self.graph.vs:
            print(v)

class InfiniteNPlayerHDGNetworkDynamic:
    """
    Class that will generate a network as a 2dLattice and include the methods used to update the graph following a game
    payoff matrix.

    Parameters
    ----------
    c_h : float, Cost for the hawks
    R : float, Reward
    width : int, dim of the 2dlattice
    height : int, dim of the 2dlattice
    radius : int, radius use to generate edges ine the lattice (cf lattice)
    population : list, list of values to put in the graph (each vertice will contain a value)
    mode : int, 0 for lattice only and 1 for small world
    """

    CONST_HAWK = 0
    CONST_DOVE = 1
    CONST_W = 5
    CONST_MODE_LATTICE = 0
    CONST_MODE_SW = 1
    CONST_PROBA_EDGE_SW = 0.05 # Probrability of an edge changing in the small world sim

    def __init__(self, c_h: float, R: float, width: int, height: int, radius: int, population: list, mode: int) -> None:
        # generate the network that contains all the information/method concerning the graph
        self.network = Lattice2d(width, height, radius)
        self.network.fill_graph(population)
        self.mode = mode

        # Create a game with N = numbers of neigbhorhood of each vertices  + the one being tested which is fixed for the Lattice
        # but the N will change for each loop and vertex when using smal world (cf update)
        self.game = HDG(len(self.network.get_neighbors(0))+1, c_h, R)

    def update(self) -> None:
        """
        Update the graph vertices values by following the process:
            - If small world mode update the graph by changing the edges following a fixed probability
            - Calculate a G_i (gain) for the vertice i by playing against all the neighbor link to this vertex
            - Select a vertex j in the neighborhood
            - If the strategy of j !=i then we calculate G_j as if i was of type j
            - Calculate the probablity of replacing i by j by following a fermi function
            - Update the graph vertices with the new list of values
        """
        # If small world we change the edges
        if( self.mode == self.CONST_MODE_SW):
            self.network.change_edges(self.CONST_PROBA_EDGE_SW)

        gain_i_list = []
        gain_j_list = []
        new_pop = []
        vertices = self.network.get_vertices()
        for i in range(len(vertices)):
            # For Hawk the strategy is zero and for Dove it is 1
            strategy = vertices[i].attributes()["value"]

            nb_dove = self.network.get_neighborhood_conformation(i)[1] + strategy

            # If we are in the small world configuration we need to update N of the game before calculating the payoff
            if(self.mode == self.CONST_MODE_SW):
                self.game.set_N(len(self.network.get_neighbors(i))+1)

            gains = self.game.expected_payoffs(nb_dove)
            gain_i_list.append(gains[strategy])

            neis = self.network.get_neighbors(i)
            index_selected = np.random.randint(0,len(neis))
            # If the selected neighbor is of the same type we put -1 if not we put Gj
            if(vertices[neis[index_selected]].attributes()["value"] == strategy):
                gain_j_list.append(-1)
                new_pop.append(strategy)
            else:
                gain_j_list.append(gains[not strategy])
                # We need to calculate the probability of changing the strategy of the vertice i p = 1/(1+exp(-w*(G_i-G_j)))
                proba = 1/(1+ np.exp(-self.CONST_W*( gains[not strategy] - gains[strategy])))
                if( np.random.random() <= proba):
                    new_pop.append(not strategy)

                else:
                    new_pop.append(strategy)

        self.network.fill_graph(new_pop)

    def calculate_dove_ratio(self) -> float:
        value_list = self.network.graph.vs()["value"]
        return sum(value_list)/len(value_list)

    @staticmethod
    def generate_population(size: int, ratio_dove: float) -> list:
        if( ratio_dove > 1 or ratio_dove < 0):
            raise NameError("Invalid ratio of doves")

        binomial_samples = np.random.binomial(1,ratio_dove,size)

        return binomial_samples

if __name__ == "__main__":

    if True:
        w = 25
        h = 25
        x = 0.5
        radius_list = [i for i in range(1,5)]
        R = 1.0
        c_h_list = [i/100 for i in range(0,101)]
        results = []
        mode = InfiniteNPlayerHDGNetworkDynamic.CONST_MODE_SW

        nb_saved = 10
        steps = 50

        results = []
        for radius in radius_list:
            result = []
            # We iterate over all values of c_h and simulate for each the comportement of a population randomly sample with a
            # fixed ratio of dove/hawk and save the ratio of dove at the end of each loop of a c_h
            for elem in c_h_list:
                pop = InfiniteNPlayerHDGNetworkDynamic.generate_population(w*h, x)
                obj = InfiniteNPlayerHDGNetworkDynamic(c_h=elem, R=R, width=w, height=h, population=pop, radius=radius, mode=mode)

                list_values = []
                for i in range(steps):
                    if(i >= steps-nb_saved):
                        list_values.append(obj.calculate_dove_ratio())
                    obj.update()

                result.append(np.around(mean(list_values),decimals=4))
                print("End of step : " + str(elem) )

            results.append(result)

        N = [5, 13, 25, 41]

        plt.figure(figsize=(10,10))
        if(mode == InfiniteNPlayerHDGNetworkDynamic.CONST_MODE_LATTICE):
            plt.title("Equilibria of the HDG in 2d lattice network")
        elif( mode == InfiniteNPlayerHDGNetworkDynamic.CONST_MODE_SW):
            plt.title("Equilibria of the HDG in small world network")
        plt.axis([0,1,0,1])
        font = {'family':'serif','size':15}
        plt.xlabel("C_h",fontdict=font)
        plt.ylabel("x*",fontdict=font)
        for id,elem in enumerate(results):
            plt.plot(c_h_list, elem, label="N = "+str(N[id]))
            plt.scatter(c_h_list, elem)
        plt.legend(loc="lower right",fontsize="x-large")
        plt.show()


import matplotlib.pyplot as plt
import numpy as np
import argparse
from collections import Counter

from keras.preprocessing.image import ImageDataGenerator
from keras.utils import to_categorical
from keras.models import Sequential, load_model
from keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, LeakyReLU, Dropout, BatchNormalization, Activation, GlobalAveragePooling2D
from keras.callbacks import TensorBoard, ModelCheckpoint, EarlyStopping
from keras.optimizers import Adam, SGD
from keras.regularizers import l1,l2
from numpy.random import randint
from sklearn.utils import shuffle, resample
from skimage import io
from sklearn.metrics import confusion_matrix, classification_report, roc_auc_score, ConfusionMatrixDisplay
from sklearn.metrics import precision_score, accuracy_score, recall_score, f1_score
from sklearn.model_selection import StratifiedKFold
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import RandomizedSearchCV, KFold
from sklearn.metrics import make_scorer

from imblearn.over_sampling import SMOTE
from imblearn.keras import BalancedBatchGenerator
import keras.losses

import sys
sys.path.insert(1, "/home/alehner1/summer20/flatworms/utilities")
from focal_loss import focal_loss
keras.losses.focal_loss_fixed = focal_loss()

import metrics
from evaluation import analyze, first_image_predict, vote_predict, probability_predict, sum_confusion_matrix

def build_net(input_shape = (128, 160, 1), num_types = 4, dropout = 0.4, n_layer1 = 1, n_layer2 = 2, n_layer3 = 2, n_layer4 = 2):
    neural_net = Sequential()

    neural_net.add(Conv2D(32, (5,5), padding="same", input_shape=input_shape))
    neural_net.add(BatchNormalization())
    neural_net.add(Dropout(dropout))
    for i in range(n_layer1):
        neural_net.add(Conv2D(32, (5,5), padding="same", input_shape=input_shape))
        neural_net.add(BatchNormalization())

    neural_net.add(MaxPooling2D(pool_size=(2, 2)))

    for i in range(n_layer2):
        neural_net.add(Conv2D(64, (3,3), padding="same"))
        neural_net.add(BatchNormalization())

    neural_net.add(MaxPooling2D(pool_size=(2, 2)))

    for i in range(n_layer3):
        neural_net.add(Conv2D(128, (3,3), padding="same"))
        neural_net.add(BatchNormalization())

    neural_net.add(MaxPooling2D(pool_size=(2, 2)))

    for i in range(n_layer4):
        neural_net.add(Conv2D(256, (3,3), padding="same"))
        neural_net.add(BatchNormalization())

    neural_net.add(MaxPooling2D(pool_size=(2, 2)))
    neural_net.add(Dropout(dropout))

    neural_net.add(Flatten())
    neural_net.add(Dense(100, activation="relu"))
    neural_net.add(BatchNormalization())
    neural_net.add(Dropout(dropout))

    neural_net.add(Dense(100, activation="relu"))
    neural_net.add(BatchNormalization())
    neural_net.add(Dropout(dropout))

    neural_net.add(Dense(50, activation="relu"))
    neural_net.add(BatchNormalization())
    neural_net.add(Dropout(dropout))

    neural_net.add(Dense(num_types, activation="softmax"))

    neural_net.compile(optimizer= "Nadam", loss = focal_loss(),
                       metrics=['accuracy', metrics.recall, metrics.precision])
    return neural_net

def test(model, X_test, y_test):

    #getting the customized test sets (X only) for each testing method:
    first_image_pred = first_image_predict(model, X_test)
    probability_pred = probability_predict(model, X_test)
    vote_pred = vote_predict(model, X_test)

    print("\nusing first image:")
    analyze(first_image_pred, y_test)

    print("\nwith probabilities:")
    analyze(probability_pred, y_test)

    print("\nafter voting:")
    analyze(probability_pred, y_test)

def filter_classes(X, y):
    shape = X.shape
    X_flat = np.reshape(X, (shape[0], shape[1] * shape[2] * shape[3]))
    y_flat = np.reshape(y, (shape[0], 1))

    combined_data = np.concatenate((X_flat, y_flat), axis = 1)

    combined_data_0 = combined_data[combined_data[:,-1] == 0] #normal
    combined_data_2 = combined_data[combined_data[:,-1] == 2] #contracted
    combined_data_2[:,-1] -= 1
    combined_data_3 = combined_data[combined_data[:,-1] == 3] #c-shape
    combined_data_3[:,-1] -= 1
    combined_data_4 = combined_data[combined_data[:,-1] == 4] #screw shaped
    combined_data_4[:,-1] -= 1

    X_flat = None
    y_flat = None
    combined_data = None

    combined_data = np.concatenate((combined_data_0, combined_data_2, combined_data_3, combined_data_4))

    combined_data_0 = None
    combined_data_2 = None
    combined_data_3 = None
    combined_data_4 = None

    X_filtered = combined_data[:, :-1]
    X_filtered = np.reshape(X_filtered, (X_filtered.shape[0], shape[1], shape[2], shape[3]))
    y_filtered = combined_data[:, -1]

    return X_filtered, y_filtered

def extract_frames(X_set, y_set, start_frame, jump_frame):

    types_num = y_set.shape[1]
    frames_num = X_set.shape[1]
    frames_num = int((frames_num - start_frame)/jump_frame)
    X_new_set = np.zeros((X_set.shape[0]*frames_num, X_set.shape[2], X_set.shape[3], X_set.shape[4]))
    y_new_set = np.zeros((X_set.shape[0]*frames_num, types_num))

    num = 0
    for i, sequence in enumerate(X_set):
        for j in range(frames_num):
            # starting frame start_frame
            X_new_set[num] = sequence[start_frame + (j * jump_frame)]
            y_new_set[num] = y_set[i]
            num += 1

    return X_new_set, y_new_set

def balance_train(X_train, y_train, num_types, class_samples):

    X_raw_train = X_train
    y_raw_train = y_train
    X_raw_train = np.reshape(X_raw_train, (X_raw_train.shape[0], X_raw_train.shape[1] * X_raw_train.shape[2] * X_raw_train.shape[3]))
    y_raw_train = np.reshape(y_raw_train, (y_raw_train.shape[0], num_types))
    combined_train = np.concatenate((X_raw_train, y_raw_train), axis=1)

    #separating the first class
    combined_train_0 = combined_train[combined_train[:,-num_types] == 1]
    combined_train_rest = combined_train[combined_train[:,-num_types] != 1]

    #downsampling the first class
    combined_train_0 = resample(combined_train_0,
                                replace=False,
                                n_samples=class_samples[0],
                                random_state=123)
    combined_train_balanced = np.concatenate([combined_train_0, combined_train_rest])
    combined_train_balanced = shuffle(combined_train_balanced)

    y_raw_train = combined_train_balanced[:, -num_types:]
    X_raw_train = combined_train_balanced[:, :-num_types]

    y_new_train = y_raw_train
    X_new_train = np.reshape(X_raw_train, (X_raw_train.shape[0], X_train.shape[1], X_train.shape[2], X_train.shape[3]))

    y_train = None
    X_train = None
    y_raw_train = None
    X_raw_train = None
    combined_train = None
    combined_train_0 = None
    combined_train_rest = None
    combined_train_balanced = None

    X_new_train, y_new_train = shuffle(X_new_train, y_new_train)

    return X_new_train, y_new_train

def tune_params(param_grid, X_train, y_train, n_epochs = 15, n_folds = 3, n_iter = 5):

    # model class to use in the scikit random search CV
    model = KerasClassifier(build_fn=build_net, epochs=n_epochs, batch_size=20, verbose=1)

    grid = RandomizedSearchCV(estimator=model, cv=KFold(n_folds), param_distributions=param_grid,
                          verbose=20,  n_iter=n_iter, n_jobs=1)

    grid_result = grid.fit(X_train, y_train)
    best_params = grid_result.best_params_
    print(best_params)
    return best_params

def main():
    #hyperparameters:
    num_types = 4
    n_epochs = 100
    start_frame = 10
    jump_frame = 2
    class_samples = {0: 6000, 1: 6000, 2: 5000, 3: 2000}
    es_patience = 20
    dropout = 0.5
    n_splits = 5

    X = np.load('/sonidata/flatworms/dchaike1_scratch/dchaike1/new_shape_stack_X.npy')
    y = np.load('/sonidata/flatworms/dchaike1_scratch/dchaike1/new_shape_stack_y.npy')

    print("hello")

    #filtering some of the classes from the dataset
    X, y = filter_classes(X, y)

    X = np.expand_dims(X, -1)
    print('Original dataset shape %s' % Counter(y))
    print(X.shape)
    print(y.shape)

    X, y = shuffle(X, y)
    matrices = np.zeros((n_splits, num_types, num_types))
    total_test_pred = None
    total_test_ans = None
    #implementing cross-validation:
    skf = StratifiedKFold(n_splits = n_splits)
    i_fold = 0
    for rest_ind, test_ind in skf.split(X, y):
        X_test, y_test = X[test_ind], y[test_ind]
        X_rest, y_rest = X[rest_ind], y[rest_ind]
        #X_train, y_train = X_rest, y_rest

        val_skf = StratifiedKFold(n_splits = n_splits - 1)
        for train_ind, val_ind in skf.split(X_rest, y_rest):
            X_val, y_val = X_rest[val_ind], y_rest[val_ind]
            X_train, y_train = X_rest[train_ind], y_rest[train_ind]
            break
        print('split trainset shape %s' % Counter(y_train))
        print(X_train.shape)
        print(y_train.shape)
        #converting the labels to one-hot-vector
        y_train = to_categorical(y_train)
        y_test = to_categorical(y_test)
        y_val = to_categorical(y_val)

        X_train, y_train = extract_frames(X_train, y_train, start_frame, jump_frame)

        print('extracted trainset shape %s' % Counter(y_train.argmax(axis=-1)))
        print(X_train.shape)
        print(y_train.shape)
        X_val, y_val = extract_frames(X_val, y_val, num_types, jump_frame)

        #starting from start_frame in the test set
        X_new_test = np.zeros((X_test.shape[0], X_test.shape[1]-start_frame, X_test.shape[2], X_test.shape[3], X_test.shape[4]))
        y_new_test = np.ones((X_test.shape[0], num_types))
        for i, sequence in enumerate(X_test):
            X_new_test[i] = sequence[start_frame:]
            y_new_test[i] = y_test[i]
        X_test, y_test = X_new_test, y_new_test

        #solving class imbalance in the training set
        print('balanced trainset shape %s' % Counter(y_train.argmax(axis=-1)))
        print(X_train.shape)

        base_model = keras.applications.xception.Xception(weights="imagenet", include_top=False)
        avg = keras.layers.GlobalAveragePooling2D()(base_model.output)
        output = keras.layers.Dense(num_types, activation="softmax")(avg)
        model = keras.Model(inputs=base_model.input, outputs=output)
        model.compile(optimizer= "Nadam", loss = focal_loss(),
                           metrics=['accuracy', metrics.recall, metrics.precision])

        X_train = np.concatenate([X_train, X_train, X_train], axis=-1)
        X_val = np.concatenate([X_val, X_val, X_val], axis=-1)
        history = model.fit(X_train, y_train,
                            validation_data=[X_val, y_val],
                            epochs=n_epochs,
                            callbacks=[EarlyStopping(patience=es_patience)])

        X_train = None
        X_val = None
        X_test = np.concatenate([X_test, X_test, X_test], axis=-1)

        print("fold " + str(i_fold))

        y_test = [np.argmax(truth) for truth in y_test]
        test(model, X_test, y_test)
        test_pred = probability_predict(model, X_test)
        test_ans = y_test

        if(i_fold == 0):
            total_test_pred = test_pred
            total_test_ans = test_ans
        else:
            total_test_pred = np.concatenate([total_test_pred, test_pred], axis = 0)
            total_test_ans = np.concatenate([total_test_ans, test_ans], axis = 0)

        test_pred = None
        test_ans = None

        # summarize history for accuracy
        plt.plot(history.history['acc'])
        plt.plot(history.history['val_acc'])
        plt.title('model accuracy')
        plt.ylabel('accuracy')
        plt.xlabel('epoch')
        plt.legend(['train', 'test'], loc='upper left')
        plt.show()

        # summarize history for loss
        plt.plot(history.history['loss'])
        plt.plot(history.history['val_loss'])
        plt.title('model loss')
        plt.ylabel('loss')
        plt.xlabel('epoch')
        plt.legend(['train', 'test'], loc='upper left')
        plt.show()

        save_command = input("Save this model? (yes/no)")
        if save_command == 'yes':
            model_name = input("Name your model: ")
            model.save(model_name + str(i_fold) + ".h5")
            print("The model is saved!")

        i_fold += 1

    print("Overall Performance using Probabilities Method")
    analyze(total_test_pred, total_test_ans)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-

"""

"""

import gzip
import typing
import pandas as pd
from pathlib_mate import PathCls as Path

def make_gzip(this_file, *parts):
    """
    将文件名为 <filename> 的 ``.tsv`` 文件 (后缀名必须是tsv, 否则出粗) 压缩成 ``.tsv.gz``
    的压缩包文件. 执行该操作的 python 脚本必须和 ``.tsv`` 文件在同一目录下.

    :type this_file: str
    :param this_file: 当前 python 脚本的文件全路径

    :type parts: str
    :param filename: ``.tsv`` 文件名
    """
    if not parts[-1].endswith(".tsv"):
        raise ValueError("You have to specify a '.tsv' file")

    src = Path(this_file).parent.append_parts(*parts)
    dst = src.change(new_basename=src.basename + ".gz")

    if not dst.exists():
        with gzip.open(dst.abspath, "wb") as f:
            f.write(src.read_bytes())

def read_compressed_tsv(this_file, *parts):
    """
    从当前脚本所在的文件夹读取压缩的 ``.tsv.gz`` 文件. 读成 ``pd.DataFrame``.

    :type this_file: str
    :param this_file:

    :type parts: str
    :param parts: ``.tsv.gz`` 文件

    :rtype: pd.DataFrame
    """
    if not parts[-1].endswith(".tsv.gz"):
        raise ValueError("You have to specify a '.tsv.gz' file")
    src = Path(this_file).parent.append_parts(*parts)
    return pd.read_csv(
        src.abspath,
        sep="\t", compression="gzip",
    )

if __name__ == "__main__":
    make_gzip(__file__, "test.tsv")
    df = read_compressed_tsv(__file__,"test.tsv.gz")
    print(df)


import yaml
from pytest_inmanta.plugin import Project

def test_basics(project: Project) -> None:
    test_content = {"a": ["a", "b", "c"]}
    yaml_content = yaml.safe_dump(test_content)
    project.add_mock_file("files", "test.yaml", yaml_content)
    project.compile(
        f"""
            import yaml
            import unittest

            a = yaml::load("unittest/test.yaml")
            b = yaml::loads(\"""{yaml_content}\""")

            entity Out:
                dict a
                dict b
            end

            Out(a=a, b=b)

            implement Out using std::none
        """
    )

    instance = project.get_instances("__config__::Out")[0]
    assert instance.a == test_content
    assert instance.b == test_content

def test_readme(project):

    project.compile(
        """
    import yaml
    yaml_dict_c = yaml::loads(\"""key: value\""")
    yaml_dict_c = {"key":"value"}
    """
    )


import json
import pymongo
from pprint import pprint
from pymongo import MongoClient
from config import configuration as cfg
from playerinfo import playerinfo

client = MongoClient(cfg['deployurl'])
db = client[cfg['deploydatabase']]
cUsers = db.Users
cPlayers = db.Players
cGames = db.Games
cPlayers.create_index("playername", unique=True)
def initializeplayers():
    for playername, info in playerinfo.items():
        player = {
        'playername': playername,
        'teamname': info[1],
        'role': info[0],
        'statistics': {},
        'latestgame': {},
        'gamesplayed': []
        }
        cPlayers.update({'playername':playername}, player, upsert=True)

def updateplayerdata():
    for playername, info in playerinfo.items():
        gamesplayed = []
        gamesplayedarray = []
        latestgameplayedid = 0
        gamesplayedquery = cGames.find({'playerlist': playername},)
        for game in gamesplayedquery:
            gamesplayed.append(game)
        for game in gamesplayed:
            gamesplayedarray.append(game['gameID'])
        for game in gamesplayed:
            if game['gameID'] == max(gamesplayedarray):
                latestgame = game

        cPlayers.update({'playername':playername}, {'$set':{'gamesplayed':gamesplayed, 'latestgame':latestgame}})

def updateplayerdatabygame(game):
    for playername, player in game['players'].items():
        for player in cPlayers.find({'playername':playername}):
            if not any(game1['gameID'] == game['gameID'] for game1 in player[gamesplayed]):
                cPlayers.update({'playername':playername}, {'$set':{'gamesplayed':gamesplayed}})

exports = {
    'initializeplayers': initializeplayers,
    'updateplayerdata': updateplayerdata,
    'updateplayerdatabygame': updateplayerdatabygame
}


#!/usr/bin/env python
# coding: utf-8

# ##  Movie Recommendation

# In[2]:

import numpy as np
import pandas as pd

#  The two dataset and its merge

# In[3]:

x = pd.read_csv(r'C:\Users\Admin\Desktop\ANUSHKA\MovieRecommendation\ml-latest-small\ml-latest-small\ratings.csv')

y = pd.read_csv(r'C:\Users\Admin\Desktop\ANUSHKA\MovieRecommendation\ml-latest-small\ml-latest-small\movies.csv')

df = pd.merge(x, y, on='movieId')
df.head()

# In[4]:

import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
import seaborn as sns
sns.set_style('white')

# In[5]:

df.groupby('title')['rating'].mean().sort_values(ascending=False) #ratings ka mean from all users with title in descending order(it isnt written descending=True cause sort_value dosent have a keyword named descending)

df.groupby('title')['rating'].count().sort_values(ascending=False) #title with the count of ratings

ratings = pd.DataFrame(df.groupby('title')['rating'].mean()) #title and ratings in order of dataset

ratings['num of ratings'] = pd.DataFrame(df.groupby('title')['rating'].count())
ratings

#  Countplot

# In[27]:

plt.figure(figsize=(8,10)) #8=width,10=length
x = sns.countplot( y='rating', data=df)

#  Histogram

# In[21]:

plt.figure(figsize = (10,4))
ratings['num of ratings'].hist(bins=70)

# In[7]:

plt.figure(figsize=(10,4))
ratings['rating'].hist(bins=70)  #graph of ratings to num of users

#  Jointplot

# In[8]:

sns.jointplot(x='rating', y='num of ratings', data=ratings,alpha=0.5) #aplha=0.5 is rgba value which is from 0 to 1, transperancy increases from 0 to 1

# In[28]:

df1 = df.pivot_table(index='userId',columns='title', values='rating')  #which userid has given rating to which movie using pivottable

# In[22]:

ToyStory_user_ratings = df1['Toy Story (1995)'] #toystory's ratings given by all userid's

# In[11]:

similar_to_ToyStory = df1.corrwith(ToyStory_user_ratings) #pairwise correlate between rows or columns

# In[23]:

corr_ToyStory = pd.DataFrame(similar_to_ToyStory, columns=['Correlation'])
 #correlating with all movies

# In[24]:

corr_ToyStory.dropna(inplace=True) #max correlation value is 1 higher the value it gets recommended the -1 correlation is said to be the strongest

#  Correlation with movies

# In[14]:

corr_ToyStory.sort_values('Correlation', ascending = False)

# In[25]:

corr_ToyStory = corr_ToyStory.join(ratings['num of ratings'])

#  Recommended Movies

# In[26]:

corr_ToyStory.head(10)[corr_ToyStory.head(10)['num of ratings']>5].sort_values('Correlation',ascending=False) #if num of ratings is >1 it will display and the correlations values will be in descng order

# In[17]:

Babe_user_ratings = df1['Babe (1995)']
similar_to_Babe = df1.corrwith(Babe_user_ratings)
corr_Babe = pd.DataFrame(similar_to_Babe, columns=['Correlation'])
corr_Babe.dropna(inplace=True)
corr_Babe = corr_Babe.join(ratings['num of ratings'])
corr_Babe.head(5)[corr_Babe.head(5)['num of ratings']>1].sort_values('Correlation',ascending=False)

# In[ ]:


import math
import yaml
import os
import matplotlib.pyplot as plt
from datetime import datetime, timedelta

from robotino_core.Graph import Graph
from robotino_core.agv.AGV_Routing import Routing
from robotino_core.agv.AGV_Main import AGV
from robotino_core.Comm import Comm
from robotino_core.solvers.astar_solver import find_shortest_path

def calculate_euclidean_distance(a, b):
	return math.sqrt(math.pow(b[0] - a[0], 2) + math.pow(b[1] - a[1], 2))

def cost_estimator(executing_task, local_task_list, current_node, current_loc, start_time, routing):

	# Start point
	current_loc = current_loc
	current_node = current_node

	# Outputs
	task_slots = []
	total_cost = timedelta()

	if routing:

		# Evaluate executing task
		end_time = reserved_slots[executing_task][-1][0] + reserved_slots[executing_task][-1][1]
		duration = end_time - start_time

		# Update executing task
		task_slots.append((start_time, end_time, duration))
		total_cost += duration

		# Evaluate local task list
		for task in local_task_list:
			start_time = reserved_slots[task][0][0]
			end_time = reserved_slots[task][-1][0] + reserved_slots[task][-1][1]
			duration = end_time - start_time
			task_slots.append((start_time, end_time, duration))
			total_cost += duration

	else:

		# Evaluate executing task
		dist = calculate_euclidean_distance((current_loc[0], current_loc[1]), graph.nodes[current_node].pos)
		dist += find_shortest_path(graph, current_node, executing_task)[1]
		cost = timedelta(seconds=dist / 1.0)

		# End time
		end_time = start_time + cost
		duration = end_time - start_time

		# Update executing task
		task_slots.append((start_time, end_time, duration))
		total_cost += cost

		# Evaluate local task list
		start_time = end_time
		start_node = executing_task
		for task in local_task_list:
			dist = find_shortest_path(graph, start_node, task)[1]
			cost = timedelta(seconds=dist / 1.0)
			end_time = start_time + cost
			duration = end_time - start_time
			task_slots.append((start_time, end_time, duration))
			total_cost += cost
			start_time = end_time
			start_node = task

	return task_slots, total_cost

if __name__== "__main__":

	# Params
	robot_id = 15

	# Start time
	start_time_init = datetime(2000,1,1,0,0,0,0)

	# AGV
	agv = AGV('localhost', 10015, 15, 'localhost', 'matthias', 'matthias', 'kb', (10, 30), False, False, False)

	# Open connection to database
	comm = Comm('localhost', 10015, 'localhost', 'matthias', 'matthias', 'kb')
	comm.sql_open()

	# Calculate estimated end time and duration
	routing = Routing(agv)

	# Read params
	this_dir = os.path.dirname(os.path.dirname(__file__))
	data_path = os.path.join(this_dir, "src/robotino_core/params", "setup.yaml")
	with open(data_path, 'r') as file:
		params = yaml.load(file, Loader=yaml.FullLoader)

	# Create graph
	graph = Graph()
	node_neighbors = params['node_neighbors']
	node_locations = params['node_locations']
	graph.create_nodes(list(node_locations.values()), list(node_locations.keys()))
	graph.create_edges(list(node_neighbors.keys()), list(node_neighbors.values()))
	graph.plot()

	# Current loc
	current_loc = (20, 30)

	# Current node
	current_node = 'pos_5'

	# Executing_task
	executing_task = 'pos_6'

	# Get tasks in local task list
	local_task_list = ['pos_11', 'pos_14','pos_16']

	# Dmas update
	reserved_paths = {}
	reserved_slots  = {}
	total_path = []

	# Update paths for executing task
	start_time = start_time_init + timedelta(seconds=calculate_euclidean_distance((current_loc[0], current_loc[1]), graph.nodes[current_node].pos) / 1.0)
	best_path, best_slots = routing.dmas(current_node, [executing_task], start_time, comm)
	reserved_paths[executing_task] = best_path
	reserved_slots[executing_task] = best_slots

	# Update paths for each task in local task list
	estimated_start_time = best_slots[-1][0] + best_slots[-1][1]
	start_node = executing_task
	for task in local_task_list:

		# Do dmas
		best_path, best_slots = routing.dmas(start_node, [task], estimated_start_time, comm)

		# If a solution exist
		if best_path:

			# Set total planned path
			total_path.extend(best_path)

			# Estimated end time and duration
			estimated_end_time = best_slots[-1][0] + best_slots[-1][1]
			estimated_duration = estimated_end_time - estimated_start_time

			# Set paths towards all tasks
			reserved_paths[task] = best_path
			reserved_slots[task] = best_slots

			# Estimated start time for next task
			estimated_start_time = estimated_end_time

			# Start node for next task is end node of previous task
			start_node = task

	# Print
	print()
	for key, value in reserved_paths.items():
		print(key + ': ' + str(value))
	print()
	for key, value in reserved_slots.items():
		print(key + ': ' + str(value))

	# Evaluate current route
	task_slots, cost = cost_estimator(executing_task, local_task_list, current_node, current_loc, start_time_init, False)

	# Print results
	print()
	print("Astar: ")
	for task in task_slots:
		print(task)
	print(cost.total_seconds())

	# Evaluate current route
	task_slots, cost = cost_estimator(executing_task, local_task_list, current_node, current_loc, start_time_init, True)

	# Print results
	print()
	print("Dmas: ")
	for task in task_slots:
		print(task)
	print(cost.total_seconds())

	# Plot graph
	plt.show()