'''
Created on Feb 11, 2013

@package: ally base
@copyright: 2012 Sourcefabric o.p.s.
@license: http://www.gnu.org/licenses/gpl-3.0.txt
@author: Gabriel Nistor

Provides the attributes support.
'''

from .spec import IAttribute, AttrError, IResolver, ResolverError, \
    ContextMetaClass, LIST_UNAVAILABLE, LIST_UNUSED, CREATE_DEFINITION
from ally.design.processor.spec import LIST_CLASSES
from ally.support.util_spec import IGet, ISet
from ally.support.util_sys import locationStack
from collections import Iterable
from inspect import isclass
import logging
from ally.support.util import FlagSet

# --------------------------------------------------------------------

log = logging.getLogger(__name__)

# --------------------------------------------------------------------

def defines(*types, doc=None):
    '''
    Construct a defining attribute for the context. The defines attribute means that the context can provide a value
    for the attribute, but is not mandatory also whenever managing an attribute if this type is a good idea to check
    if there aren't already values provided.

    @param types: arguments[class]
        The types of the defined attribute.
    @param doc: string|None
        The documentation associated with the attribute.
    '''
    return Attribute(Specification(DEFINED, types, doc=doc))

def definesIf(*types, doc=None):
    '''
    Construct a defining attribute for the context. The defines attribute means that the context can provide a value
    for the attribute, but is not mandatory also whenever managing an attribute if this type is a good idea to check
    if there aren't already values provided. Whenever using this type of attributes always check if the context has them since
    if they are optional they might not event be populated if there is no definition for them, so always to a check like:
        MyContext.myAttribute in myInstance
    , otherwise you might get attribute error.

    @param types: arguments[class]
        The types of the defined attribute.
    @param doc: string|None
        The documentation associated with the attribute.
    '''
    return Attribute(Specification(DEFINED | OPTIONAL, types, doc=doc))

def optional(*types, doc=None):
    '''
    Construct an optional attribute for the context. The optional attribute means that the context is valid even if
    there is no value for the attribute. Whenever using this type of attributes always check if the context has them since
    if they are optional they might not event be populated if there is no definition for them, so always to a check like:
        MyContext.myAttribute in myInstance
    , otherwise you might get attribute error.

    @param types: arguments[class]
        The types of the optional attribute, the attribute value can be any one of the provided attributes.
    @param doc: string|None
        The documentation associated with the attribute.
    '''
    return Attribute(Specification(OPTIONAL, types, doc=doc))

def requires(*types, doc=None):
    '''
    Construct a required attribute for the context. The requires attribute means that the context is valid only if
    there is a value for the attribute.

    @param types: arguments[class]
        The types of the required attribute, the attribute value can be any one of the provided attributes.
    @param doc: string|None
        The documentation associated with the attribute.
    '''
    return Attribute(Specification(REQUIRED, types, doc=doc))

def attribute(*types, doc=None):
    '''
    Construct a simple attribute for the context that will be used by the processor that define it and is also available
    as a defined attribute.

    @param types: arguments[class]
        The types of the defined attribute.
    @param doc: string|None
        The documentation associated with the attribute.
    '''
    return Attribute(Specification(DEFINED | USED, types, doc=doc))

# --------------------------------------------------------------------

DEFINED = 1 << 1
# Status flag for defined attributes.
REQUIRED = 1 << 2
# Status flag for required attributes.
OPTIONAL = 1 << 3
# Status flag for optional attributes.
USED = 1 << 4
# Status flag for attributes that should be considered always used.

# --------------------------------------------------------------------

class Specification:
    '''
    Provides attribute specifications.
    '''

    def __init__(self, status, types, definedIn=None, doc=None, defined=None):
        '''
        Construct the attribute specification.

        @param status: integer
            The status of the attribute specification.
        @param types: Iterable(class)
            The type(s) for the attribute specification.
        @param definedIn: class|None
            The class that defines the specification, this is just in case the specification was created based on a definer.
        @param doc: string|None
            The documentation associated with the attribute specification.
        @param defined: Iterable(class)|None
            The defined classes.
        '''
        assert isinstance(status, int), 'Invalid status %s' % status
        types = tuple(reduce(types))
        assert types, 'At least a type is required'
        assert definedIn is None or isclass(definedIn), 'Invalid defined in %s' % definedIn

        if defined is None:
            if status & DEFINED: defined = types
            else: defined = ()
        else: assert isinstance(defined, Iterable), 'Invalid defined classes %s' % defined

        self.status = status
        self.types = types
        self.definedIn = definedIn
        self.doc = doc
        self.defined = frozenset(defined)

        self.usedIn = {}

    def __str__(self):
        status = []
        if self.status & DEFINED: status.append('DEFINES')
        if self.status & REQUIRED: status.append('REQUIRED')
        if self.status & OPTIONAL: status.append('OPTIONAL')

        return ''.join(('|'.join(status), '[', ','.join(t.__name__ for t in self.types), ']'))

class Resolver(IResolver):
    '''
    Implementation for a @see: IResolver that manages contexts with @see: Attribute.
    '''

    def __init__(self, context):
        '''
        Construct the attribute resolver.

        @param context: ContextMetaClass|dictionary{string: Specification}
            The context class to construct the resolver based on, or the specifications dictionary.
        '''
        if isinstance(context, ContextMetaClass):
            assert isinstance(context, ContextMetaClass)
            specifications = {}
            for name, attribute in context.__attributes__.items():
                assert isinstance(name, str), 'Invalid name %s' % name
                assert isinstance(attribute, Attribute), 'Invalid attribute %s' % attribute
                specifications[name] = attribute.specification

        else:
            assert isinstance(context, dict), 'Invalid context %s' % context
            if __debug__:
                for name, spec in context.items():
                    assert isinstance(name, str), 'Invalid name %s' % name
                    assert isinstance(spec, Specification), 'Invalid specification %s' % spec
            specifications = context

        self.specifications = specifications

    def copy(self, names=None):
        '''
        @see: IResolver.copy
        '''
        specifications = {}
        if names is None:
            specifications.update(self.specifications)

        else:
            assert isinstance(names, Iterable), 'Invalid names %s' % names
            for name in names:
                assert isinstance(name, str), 'Invalid name %s' % name
                spec = self.specifications.get(name)
                if spec: specifications[name] = spec

        return self.__class__(specifications)

    def merge(self, other, isFirst=True):
        '''
        @see: IResolver.merge
        '''
        assert isinstance(other, IResolver), 'Invalid other resolver %s' % other
        assert isinstance(isFirst, bool), 'Invalid is first flag %s' % isFirst

        if self is other: return self

        if not issubclass(self.__class__, other.__class__):
            if isFirst: return other.merge(self, False)
            raise ResolverError('Cannot merge %s with %s' % (self, other))
        assert isinstance(other, Resolver), 'Invalid other resolver %s' % other

        return self.__class__(self.mergeSpecifications(self.specifications, other.specifications))

    def solve(self, other):
        '''
        @see: IResolver.solve
        '''
        assert isinstance(other, IResolver), 'Invalid other resolver %s' % other
        if self is other: return self

        if not issubclass(self.__class__, other.__class__): return other.solve(self)
        assert isinstance(other, Resolver), 'Invalid other resolver %s' % other

        return self.__class__(self.solveSpecifications(self.specifications, other.specifications))

    def list(self, *flags):
        '''
        @see: IResolver.list
        '''
        flags, attributes = FlagSet(flags), {}

        listed = False
        if flags.checkOnce(LIST_UNAVAILABLE):
            listed = True
            for name, spec in self.specifications.items():
                assert isinstance(spec, Specification), 'Invalid specification %s' % spec
                if spec.status == REQUIRED: attributes[name] = None

        listClasses = flags.checkOnce(LIST_CLASSES)

        if flags.checkOnce(LIST_UNUSED):
            listed = True
            for name, spec in self.specifications.items():
                assert isinstance(spec, Specification), 'Invalid specification %s' % spec
                if spec.status & USED: continue
                if spec.status & OPTIONAL: continue
                for status in spec.usedIn.values():
                    if not status & DEFINED:
                        if spec.definedIn is not None: attributes[name] = (spec.definedIn,) if listClasses else None
                        break
                else: attributes[name] = spec.usedIn.keys() if listClasses else None
            listClasses = False

        if not listed:
            for name in self.specifications: attributes[name] = None

        if listClasses:
            for name in attributes: attributes[name] = self.specifications[name].usedIn.keys()

        assert not flags, 'Unknown flags: %s' % ', '.join(flags)
        return attributes

    def create(self, *flags):
        '''
        @see: IResolver.create
        '''
        flags = FlagSet(flags)

        attributes = {}
        if flags.checkOnce(CREATE_DEFINITION): attributes = self.createDefinitions(self.specifications)
        else: attributes = self.createDescriptors(self.specifications)

        assert not flags, 'Unknown flags: %s' % ', '.join(flags)
        return attributes

    # ----------------------------------------------------------------

    def mergeSpecification(self, mergeSpec, withSpec, **keyargs):
        '''
        Merges the provided specifications.

        @param mergeSpec: Specification
            The specification to be merged.
        @param withSpec: Specification
            The specification to merge with.
        @param keyargs: key arguments
            Additional key arguments to be used in constructing the merged specification.
        @return: Specification
            The merged specification.
        '''
        assert isinstance(mergeSpec, Specification), 'Invalid merge specification %s' % mergeSpec
        assert isinstance(withSpec, Specification), 'Invalid with specification %s' % withSpec

        if mergeSpec is withSpec: return mergeSpec

        if mergeSpec.status == REQUIRED:
            if withSpec.status & DEFINED:
                raise AttrError('Improper order for %s, it should be before %s' % (withSpec, mergeSpec))
            status = REQUIRED
            types = intersect(mergeSpec.types, withSpec.types)
            if not types:
                raise AttrError('Incompatible required types of %s, with required types of %s' % (mergeSpec, withSpec))

        elif mergeSpec.status == OPTIONAL:
            if withSpec.status & DEFINED:
                status = DEFINED
                types = set(mergeSpec.types)
            else:
                status = withSpec.status
                types = intersect(mergeSpec.types, withSpec.types)
                if not types:
                    raise AttrError('Incompatible required types of %s, with required types of %s' % (mergeSpec, withSpec))

        elif mergeSpec.status & DEFINED:
            status = DEFINED
            if mergeSpec.status & USED: status |= USED
            if withSpec.status & DEFINED:
                types = set(mergeSpec.types)
                types.update(withSpec.types)
                # In case both definitions are optional we need to relate that to the merged attribute
                if withSpec.status & USED: status |= USED
                if mergeSpec.status & OPTIONAL and withSpec.status & OPTIONAL: status |= OPTIONAL
            else:
                types = set(withSpec.types)

        keyargs['status'] = status

        defined = set(mergeSpec.defined)
        defined.update(withSpec.defined)
        defined, types = reduce(defined), reduce(types)
        if defined != types and not types.issuperset(defined):
            raise ResolverError('Invalid types %s and defined types %s, they cannot be joined, for %s, and %s, '
                                'from merged:%s\n, with:%s' % (', '.join('\'%s\'' % typ.__name__ for typ in types),
                                 ', '.join('\'%s\'' % typ.__name__ for typ in defined), mergeSpec, withSpec,
                                 ''.join(locationStack(clazz) for clazz in mergeSpec.usedIn),
                                 ''.join(locationStack(clazz) for clazz in withSpec.usedIn)))

        keyargs['types'] = types
        keyargs['defined'] = defined

        docs = []
        if mergeSpec.doc is not None: docs.append(mergeSpec.doc)
        if withSpec.doc is not None: docs.append(withSpec.doc)
        keyargs['doc'] = '\n'.join(docs) if docs else None

        spec = mergeSpec.__class__(**keyargs)
        assert isinstance(spec, Specification), 'Invalid specification %s' % spec
        spec.usedIn.update(mergeSpec.usedIn)
        spec.usedIn.update(withSpec.usedIn)

        return spec

    def mergeSpecifications(self, mergeSpecs, withSpecs):
        '''
        Merges the provided specifications.

        @param mergeSpecs: dictionary{string: Specification}
            The specifications to be merged.
        @param withSpecs: dictionary{string: Specification}
            The specifications to merge with.
        @return: dictionary{string: Specification}
            The merged specifications.
        '''
        assert isinstance(mergeSpecs, dict), 'Invalid specifications %s' % mergeSpecs
        assert isinstance(withSpecs, dict), 'Invalid specifications %s' % withSpecs

        specifications = dict(mergeSpecs)
        for name, spec in withSpecs.items():
            ownSpec = specifications.get(name)
            if ownSpec is None: specifications[name] = spec
            else:
                assert isinstance(spec, Specification), 'Invalid specification %s' % spec
                try: specifications[name] = self.mergeSpecification(ownSpec, spec, definedIn=spec.definedIn)
                except AttrError:
                    raise AttrError('Cannot merge attribute \'%s\', from:%s\n, with:%s' %
                                    (name, ''.join(locationStack(clazz) for clazz in ownSpec.usedIn),
                                     ''.join(locationStack(clazz) for clazz in spec.usedIn)))

        return specifications

    def solveSpecifications(self, mergeSpecs, withSpecs):
        '''
        Solve the provided specifications.

        @param mergeSpecs: dictionary{string: Specification}
            The specifications to be solved.
        @param withSpecs: dictionary{string: Specification}
            The specifications to solve with.
        @return: dictionary{string: Specification}
            The solved specifications.
        '''
        assert isinstance(mergeSpecs, dict), 'Invalid specifications %s' % mergeSpecs
        assert isinstance(withSpecs, dict), 'Invalid specifications %s' % withSpecs

        specifications = dict(self.specifications)
        for name, spec in withSpecs.items():
            assert isinstance(spec, Specification), 'Invalid specification %s' % spec
            ownSpec = specifications.get(name)
            if ownSpec is None: specifications[name] = spec
            elif ownSpec.status & DEFINED:
                assert isinstance(ownSpec, Specification), 'Invalid specification %s' % ownSpec
                specifications[name] = self.mergeSpecification(ownSpec, spec)
            else:
                if spec.definedIn is not None and ownSpec.definedIn is not None:
                    specifications[name] = self.mergeSpecification(spec, ownSpec)
                else:
                    specifications[name] = self.mergeSpecification(spec, ownSpec)

        return specifications

    def createDefinitions(self, specifications):
        '''
        Create the definitions attributes.

        @param specifications: dictionary{string: Specification}
            The specifications to create the definitions for.
        @return: dictionary{string: IAttribute}
            The created attributes.
        '''
        assert isinstance(specifications, dict), 'Invalid specifications %s' % specifications
        attributes = {}
        for name, spec in specifications.items():
            assert isinstance(name, str), 'Invalid name %s' % name
            assert isinstance(spec, Specification), 'Invalid specification %s' % spec
            attributes[name] = Attribute(spec)

        return attributes

    def createDescriptors(self, specifications):
        '''
        Create the descriptors attribute.

        @param specifications: dictionary{string: Specification}
            The specifications to create the descriptors for.
        @return: dictionary{string: IAttribute}
            The created attributes.
        '''
        assert isinstance(specifications, dict), 'Invalid specifications %s' % specifications
        attributes = {}
        for name, spec in specifications.items():
            assert isinstance(name, str), 'Invalid name %s' % name
            assert isinstance(spec, Specification), 'Invalid specification %s' % spec
            if spec.status == REQUIRED:
                raise AttrError('Cannot generate attribute %s=%s, used in:%s' %
                                (name, spec, ''.join(locationStack(clazz) for clazz in spec.usedIn)))
            if spec.status & OPTIONAL: continue  # If is optional then no need to create it
            attributes[name] = AttributeObject(spec)

        return attributes

    def __str__(self):
        if not self.specifications: return '%s empty' % self.__class__.__name__
        return '%s[%s]' % (self.__class__.__name__, ', '.join('%s=%s' % (name, self.specifications.get(name))
                                                              for name in sorted(self.specifications)))

class Attribute(IAttribute):
    '''
    Base attribute implementation for a @see: IAttribute that manages a attributes by status.
    '''

    def __init__(self, specification, Resolver=Resolver):
        '''
        Construct the attribute.

        @param specification: Specification
            The attribute specification.
        @param Resolver: class
            The resolver class for the attribute.
        '''
        assert isinstance(specification, Specification), 'Invalid specification %s' % specification
        assert isclass(Resolver), 'Invalid resolver class %s' % Resolver

        self.specification = specification
        self.Resolver = Resolver

        self.isPlaced = False

    def resolver(self):
        '''
        @see: IAttribute.resolver
        '''
        return self.Resolver

    def place(self, clazz, name):
        '''
        @see: IAttribute.place
        '''
        if not self.isPlaced:
            assert isinstance(clazz, ContextMetaClass), 'Invalid class %s' % clazz
            assert isinstance(name, str), 'Invalid name %s' % name
            self.isPlaced, self.__objclass__, self.__name__ = True, clazz, name
            self.specification.usedIn[clazz] = self.specification.status
            if self.specification.definedIn is None and self.specification.status == DEFINED:
                self.specification.definedIn = clazz
        elif not issubclass(clazz, self.__objclass__) or self.__name__ != name:
            raise AttrError('%s\n, is already placed in:%s as attribute %s' %
                            (self, locationStack(self.__objclass__), self.__name__))

    def isValid(self, clazz):
        '''
        @see: IAttribute.isValid
        '''
        if not self.isPlaced: return False
        assert isclass(clazz), 'Invalid class %s' % clazz

        if not isinstance(clazz, ContextMetaClass): return False
        assert isinstance(clazz, ContextMetaClass)

        other = clazz.__attributes__.get(self.__name__)
        if other is None:
            if self.specification.status != REQUIRED: return True
            return False

        if not isinstance(other, Attribute): return False
        assert isinstance(other, Attribute)

        for typ in self.specification.types:
            if typ in other.specification.types: break
        else: return False

        return True

    def isIn(self, clazz):
        '''
        @see: IAttribute.isIn
        '''
        if not self.isPlaced: return False
        assert isclass(clazz), 'Invalid class %s' % clazz

        if not isinstance(clazz, ContextMetaClass): return False
        assert isinstance(clazz, ContextMetaClass)

        other = clazz.__attributes__.get(self.__name__)
        if other is None: return False

        if not isinstance(other, Attribute): return False
        assert isinstance(other, Attribute)

        for typ in self.specification.types:
            if typ in other.specification.types: break
        else: return False

        return True

    # ----------------------------------------------------------------

    def __get__(self, obj, owner=None):
        '''
        Descriptor get.
        '''
        if obj is not None: raise TypeError('Operation not allowed')
        assert self.isPlaced, 'Attribute %s, is not placed in a class' % self
        return self

    def __set__(self, obj, value):
        '''
        Descriptor set.
        '''
        raise TypeError('Operation not allowed')

    def __str__(self):
        st = ''.join((self.__class__.__name__, '.', str(self.specification)))
        if self.isPlaced:
            return ''.join((st, ' in:', locationStack(self.__objclass__), ' as attribute ', self.__name__))
        return ''.join((st, ' unplaced'))

class AttributeObject(Attribute):
    '''
    Object descriptor implementation for a @see: Attribute.
    '''

    def __init__(self, specification, Resolver=Resolver):
        '''
        @see: Attribute.__init__
        '''
        super().__init__(specification, Resolver)
        self.descriptor = None

    def place(self, clazz, name):
        '''
        @see: IAttribute.place
        '''
        if not self.isPlaced:
            assert isinstance(clazz, ContextMetaClass), 'Invalid class %s' % clazz
            assert isinstance(name, str), 'Invalid name %s' % name

            if __debug__:
                assert hasattr(clazz, name), 'Invalid class %s has no descriptor for %s' % (clazz, name)
                self.descriptor = getattr(clazz, name)
                assert isinstance(self.descriptor, IGet), 'Invalid descriptor %s' % self.descriptor
                assert isinstance(self.descriptor, ISet), 'Invalid descriptor %s' % self.descriptor
                setattr(clazz, name, self)
            self.isPlaced, self.__objclass__, self.__name__ = True, clazz, name
        elif not issubclass(clazz, self.__objclass__) or self.__name__ != name:
            raise AttrError('%s\n, is already placed in:%s as attribute %s' %
                            (self, locationStack(self.__objclass__), self.__name__))

    # ----------------------------------------------------------------

    def __get__(self, obj, owner=None):
        '''
        @see: Attribute.__get__
        '''
        if obj is None: return self
        assert self.descriptor, 'Attribute %s, is not placed in a class' % self
        try: return self.descriptor.__get__(obj, owner)
        except AttributeError: return None

    def __set__(self, obj, value):
        '''
        @see: Attribute.__set__
        '''
        assert value is None or isinstance(value, self.specification.types), \
        'Invalid value \'%s\' for %s' % (value, self.specification.types)
        assert self.descriptor, 'Attribute %s, is not placed in a class' % self
        self.descriptor.__set__(obj, value)

# --------------------------------------------------------------------

def reduce(types):
    '''
    Reduces the provided types to only the classes that are the top classes.

    @param types: Iterable(class)
        The types to reduce.
    @return: set(class)
        The reduced types.
    '''
    assert isinstance(types, Iterable), 'Invalid types %s' % types
    reduced = []
    for clazz in types:
        assert isclass(clazz), 'Invalid class %s' % clazz
        k, solved = 0, False
        while k < len(reduced):
            rclazz = reduced[k]

            if rclazz == clazz:
                solved = True
                break
            elif issubclass(clazz, rclazz):
                solved = True
                reduced[k] = clazz
            elif issubclass(rclazz, clazz):
                solved = True
                break

            k += 1

        if not solved: reduced.append(clazz)
    return set(reduced)

def intersect(first, second):
    '''
    Reduces the provided types to only the classes that are the top classes.

    @param first: Iterable(class)
        The first types to reduce.
    @param second: Iterable(class)
        The second types to reduce.
    @return: set(class)
        The intersected types.
    '''
    assert isinstance(first, Iterable), 'Invalid first types %s' % first
    assert isinstance(second, Iterable), 'Invalid types %s' % second
    if not isinstance(second, (list, tuple, set)): second = list(second)
    intersect = []
    for fclazz in first:
        for sclazz in second:
            if issubclass(fclazz, sclazz): intersect.append(sclazz)
            elif issubclass(sclazz, fclazz): intersect.append(fclazz)
    return reduce(intersect)


#from distutils.core import setup
import codecs
import os
from setuptools import setup
import re

#Allow single version in source file to be used here
#From https://packaging.python.org/guides/single-sourcing-package-version/
def read(*parts):
    # intentionally *not* adding an encoding option to open
    # see here: https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690
    here = os.path.abspath(os.path.dirname(__file__))
    return codecs.open(os.path.join(here, *parts), 'r').read()
def find_version(*file_paths):
    version_file = read(*file_paths)
    version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
                              version_file, re.M)
    if version_match:
        return version_match.group(1)
    raise RuntimeError("Unable to find version string.")

setup(name="SparseSC",
      version=find_version('SparseSC', '__init__.py'),
      description="Sparse Synthetic Controls",
      author="Microsoft Research",
      url="https://github.com/Microsoft/SparseSyntheticControls",
      packages=['SparseSC'],
	  license='MIT',
      install_requires=["numpy", "Scipy", "scikit-learn"])


import joblib
import tensorflow as tf
import numpy as np
import argparse
import copy
from matplotlib import pyplot as plt

parser = argparse.ArgumentParser()
parser.add_argument("path", type=str)
parser.add_argument("--level", type=int, default=None)
parser.add_argument("--supervised", type=bool, default=False)
args = parser.parse_args()

# EXAMPLES

# Policy which generally follows the teacher, except that it turns right when it says turn left.
# If the teacher said something different in that situation, the agent would follow if it's a common command (straight
# or right), but if it was an uncommon command like toggling, it would ignore the teacher.
# SUPERVISED_teacherPreActionAdvice_persistgoa_droptypestep_dropgoal0_ent0.001_lr0.01corr0_currfnsmooth_4/latest.pkl

# (seems similar to the policy above)
# THRESHOLD++_teacherPreActionAdvice_persistgoa_droptypestep_dropincNone_dropgoal0_disc0.9_thresh0.99_ent0.001_lr0.01corr0_currfnsmooth_4

# Kinda sorta follows the teacher, but gets stuck on toggling. Consider more entropy???
# "THRESHOLD++_teacherPreActionAdvice_persistgoa_droptypemeta_rollout_start_dropinc(0.8, 0.2)_dropgoal0_disc0.9_thresh0.95_ent0.001_lr0.01corr0_currfnsmooth_4/latest.pkl"

with tf.Session() as sess:

    # ACTION SPACE
    # left = 0
    # right = 1
    # forward = 2
    # pickup = 3
    # drop = 4
    # toggle = 5
    # done = 6
    # no feedback = 7 // -1
    action_meanings = ["left", "right", "forward", "pickup", "drop", "toggle", "done", "no feedback"]

    # Turn left policy
    base_path = "/home/olivia/Documents/Teachable/babyai/meta-mb-internal/data/"
    pkl_path = args.path

    pkl_path = base_path + pkl_path
    data = joblib.load(pkl_path)
    if args.supervised:
        agent = data['supervised_model']
    else:
        agent = data['policy']
    env = data['env']
    if args.level is not None:
        env.set_level_distribution(args.level)

    agent.reset(dones=[True])
    env.set_task(None)

    env.set_dropout_proportion(1)
    obs = env.reset()
    obs = np.expand_dims(obs, 0)

    use_one_hot = True

    count = 0
    skip_to_done = False
    skip_to_meta_rollout = False
    teacher_recommendations = []
    agent_actions = []
    while True:
        # Interaction
        ready = ""
        hidden_state = copy.deepcopy(agent._hidden_state)
        obs_orig = copy.deepcopy(obs)
        while (not skip_to_done) and (not skip_to_meta_rollout):
            print("please type a command below")
            ready = input()
            if ready in ['h', 'help']:
                print("Type 'h' for help, 'c' to continue to the next timestep, 'm' to continue with the modified "
                      "version of the observation with the alt teacher action (note: if none has been specified, it"
                      "functions the same as 'c'.  Type a number to make the teacher suggest the action at that index.")
            elif ready in ['r', 'render']:
                env.render(mode='human')
            elif ready in ['c', 'continue']:
                break
            elif ready in ['sd', 'skip_done']:
                skip_to_done = True
                break
            elif ready in ['sm', 'skip_meta']:
                skip_to_meta_rollout = True
                break
            elif ready in ['m', 'modified']:
                obs_orig = obs
                break
            else:
                try:
                    agent.set_hidden_state(hidden_state)
                    num = int(ready)
                    assert num >= 0
                    assert num <= 6
                    agent.reset(dones=[True])
                    if use_one_hot:
                        obs[0, 160:168] = 0
                        obs[0, 160 + num] = 1
                        print("MODIFIED Teacher suggested:", np.argmax(obs[0, 160:168]))
                    else:
                        if num == 7:
                            num = -1
                        obs[0, 160] = num
                        print("MODIFIED Teacher suggested:", obs[0, 160])
                    a, agent_info = agent.get_actions(obs)
                    a = a[0]
                    print("MODIFIED most likely action is", np.argmax(agent_info[0][0]['probs']), agent_info[0][0]['probs'])
                except:
                    print("Invalid index", ready)

        # Advance env
        obs = obs_orig
        agent.set_hidden_state(hidden_state)
        a, agent_info = agent.get_actions(obs)
        a = a[0]
        if use_one_hot:
            print("Teacher suggested:", np.argmax(obs[0, 160:168]))
            teacher_recommendations.append(np.argmax(obs[0, 160:168]))
        else:
            print("Teacher suggested:", obs[0, 160])
            teacher_recommendations.append(obs[0, 160])
        if obs[0, 160] == 6:
            print("what????")
        print("Agent took", a[0][0], agent_info[0][0]['probs'])
        agent_actions.append(a[0][0])
        obs, r, d, env_info = env.step(a)
        obs = np.expand_dims(obs, 0)
        print("Done?", d)
        ready = ''

        print("Success?", r)
        if d:
            skip_to_done = False
            if count % 2 == 1:
                agent.reset(dones=[True])
                env.set_task(None)
                skip_to_meta_rollout = False
            obs = env.reset()
            obs = np.expand_dims(obs, 0)
            count += 1

            plt.figure(1)
            plt.hist([teacher_recommendations, agent_actions], bins=list(range(len(action_meanings))), label=action_meanings, color=["blue", "orange"])
            plt.legend(["Teacher Suggested", "Agent took"])
            plt.title("Teacher's suggestions vs agent's actions")
            plt.show()

            agent_actions = np.array(agent_actions)
            teacher_recommendations = np.array(teacher_recommendations)
            percent_listened = []
            for i in range(len(action_meanings)):
                indices = np.where(teacher_recommendations == i)
                actions = agent_actions[indices]
                proportion_correct = np.mean(actions == i)
                percent_listened.append(proportion_correct)

            plt.figure(3)
            percent_listened = [0 if np.isnan(p) else p for p in percent_listened]
            plt.bar(action_meanings, percent_listened)
            plt.title("Proportion of the time each action was followed")
            plt.show()

            teacher_recommendations = []
            agent_actions = []


# -*- coding: utf-8 -*-
#
# Copyright (C) 2016 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

TOPOLOGY = """
#
# +-------+
# |  sw1  |
# +-------+
#

# Nodes
[type=openswitch name="Switch 1"] sw1
"""

def test_sftp_client_configuration(topology, step):
    sw1 = topology.get('sw1')

    assert sw1 is not None

    step('Test SFTP client username max limit')
    max_username_len = 256
    username = 'u'
    srcpath = ' /etc/ssh/sshd_config'
    dstpath = ' /home/admin/'

    for x in range(1, max_username_len+1):
        username = username+'u'
    print(len(username))
    copy = "copy sftp "
    hostip = ' 127.0.0.1'
    cmd = copy+username+hostip+srcpath+dstpath

    sw1._shells['vtysh']._prompt = (
        'switch#'
    )
    out = sw1(cmd)
    assert 'Username should be less than 256 characters' in out

    sw1._shells['vtysh']._prompt = (
        '(^|\n)switch(\\([\\-a-zA-Z0-9]*\\))?#'
    )
    sw1(' ')

    step('Test SFTP client hostname max limit')
    max_hostname_len = 256
    hostname = ' h'
    username = "root"
    srcpath = ' /etc/ssh/sshd_config'
    dstpath = ' /home/admin/'
    copy = "copy sftp "

    for y in range(1, max_hostname_len+1):
        hostname = hostname+'h'

    cmd = copy+username+hostname+srcpath+dstpath

    sw1._shells['vtysh']._prompt = (
        'switch#'
    )
    out = sw1(cmd)
    assert 'Hostname should be less than 256 characters' in out

    sw1._shells['vtysh']._prompt = (
        '(^|\n)switch(\\([\\-a-zA-Z0-9]*\\))?#'
    )
    sw1(' ')


import csv
import codecs
import pprint
import re
import xml.etree.cElementTree as ET

import cerberus

import schema

OSM_PATH = "houston_texas.osm"

NODES_PATH = "nodes.csv"
NODE_TAGS_PATH = "nodes_tags.csv"
WAYS_PATH = "ways.csv"
WAY_NODES_PATH = "ways_nodes.csv"
WAY_TAGS_PATH = "ways_tags.csv"

LOWER_COLON = re.compile(r'^([a-z]|_)+:([a-z]|_)+')
PROBLEMCHARS = re.compile(r'[=\+/&<>;\'"\?%#$@\,\. \t\r\n]')

SCHEMA = schema.schema

# Make sure the fields order in the csvs matches the column order in the sql table schema
NODE_FIELDS = ['id', 'lat', 'lon', 'user', 'uid', 'version', 'changeset', 'timestamp']
NODE_TAGS_FIELDS = ['id', 'key', 'value', 'type']
WAY_FIELDS = ['id', 'user', 'uid', 'version', 'changeset', 'timestamp']
WAY_TAGS_FIELDS = ['id', 'key', 'value', 'type']
WAY_NODES_FIELDS = ['id', 'node_id', 'position']

mapping = { 'Ave': 'Avenue',
			'Ave.': 'Avenue',
			'Blvd': 'Boulevard',
			'Blvd.': 'Bouelvard',
			'Dr': 'Drive',
			'Frwy': 'Freeway',
			'Fwy': 'Freeway',
			'HIGHWAY': 'Highway',
			'Ln': 'Lane',
			'Pkwy': 'Parkway',
			'Rd': 'Road',
			'St': 'Street',
			'Stree': 'Street',
			'street': 'Street'
            }

def update_name(name, mapping):

    list_name = name.split(' ')
    last_type = list_name.pop()
    if last_type in mapping:
    	new_type = mapping[last_type]
    	list_name.append(new_type)
    	name = ' '.join(list_name)

    	return name
    else:
    	return name

def fix_postcode(code):
	if len(code) >= 5:
		start_postcode = code.find('7')
		end_postcode = start_postcode + 5
		new_code = code[start_postcode:end_postcode]
		return new_code
	else:
		return code

def shape_element(element, node_attr_fields=NODE_FIELDS, way_attr_fields=WAY_FIELDS,
                  problem_chars=PROBLEMCHARS, default_tag_type='regular'):
    """Clean and shape node or way XML element to Python dict"""

    node_attribs = {}
    way_attribs = {}
    way_nodes = []
    tags = []  # Handle secondary tags the same way for both node and way elements

    if element.tag == 'node':
        node_attribs['id'] = element.attrib['id']
        node_attribs['lat'] = element.attrib['lat']
        node_attribs['lon'] = element.attrib['lon']
        node_attribs['user'] = element.attrib['user']
        node_attribs['uid'] = element.attrib['uid']
        node_attribs['version'] = element.attrib['version']
        node_attribs['changeset'] = element.attrib['changeset']
        node_attribs['timestamp'] = element.attrib['timestamp']

        for tag in element.iter("tag"):
            tag_dict = {}
            m = PROBLEMCHARS.search(tag.attrib['k'])
            if not m:
                n = LOWER_COLON.search(tag.attrib['k'])
                if n:
                    tag_k = n.group()
                    k_split = tag.attrib['k'].split(':', 1)
                    tag_dict['id'] = element.attrib['id']
                    tag_dict['key'] = k_split[1]
                    if tag.attrib['k'] == 'addr:street':
                    	tag_dict['value'] = update_name(tag.attrib['v'],mapping)
                    elif tag.attrib['k'] == 'addr:postcode':
                    	tag_dict['value'] = fix_postcode(tag.attrib['v'])
                    else:
                    	tag_dict['value'] = tag.attrib['v']

                    tag_dict['type'] = k_split[0]

                    tags.append(tag_dict)
                else:
                    tag_dict['id'] = element.attrib['id']
                    tag_dict['key'] = tag.attrib['k']
                    tag_dict['value'] = tag.attrib['v']
                    tag_dict['type'] = 'regular'

                    tags.append(tag_dict)
            else:
                continue

        return {'node': node_attribs, 'node_tags': tags}

    elif element.tag == 'way':
        way_attribs['id'] = element.attrib['id']
        way_attribs['user'] = element.attrib['user']
        way_attribs['uid'] = element.attrib['uid']
        way_attribs['version'] = element.attrib['version']
        way_attribs['changeset'] = element.attrib['changeset']
        way_attribs['timestamp'] = element.attrib['timestamp']

        count = 0
        for tag in element.iter("nd"):
            tag_dict1 = {}
            tag_dict1['id'] = element.attrib['id']
            tag_dict1['node_id'] = tag.attrib['ref']
            tag_dict1['position'] = count

            way_nodes.append(tag_dict1)
            count += 1

        for tag in element.iter("tag"):
            tag_dict = {}
            m = PROBLEMCHARS.search(tag.attrib['k'])
            if not m:
                n = LOWER_COLON.search(tag.attrib['k'])
                if n:
                    tag_k = n.group()
                    k_split = tag.attrib['k'].split(':', 1)
                    tag_dict['id'] = element.attrib['id']
                    tag_dict['key'] = k_split[1]
                    if tag.attrib['k'] == 'addr:street':
                    	tag_dict['value'] = update_name(tag.attrib['v'],mapping)
                    elif tag.attrib['k'] == 'addr:postcode':
                    	tag_dict['value'] = fix_postcode(tag.attrib['v'])
                    else:
                    	tag_dict['value'] = tag.attrib['v']
                    tag_dict['type'] = k_split[0]

                    tags.append(tag_dict)
                else:
                    tag_dict['id'] = element.attrib['id']
                    tag_dict['key'] = tag.attrib['k']
                    tag_dict['value'] = tag.attrib['v']
                    tag_dict['type'] = 'regular'

                    tags.append(tag_dict)
            else:
                continue

        return {'way': way_attribs, 'way_nodes': way_nodes, 'way_tags': tags}

# ================================================== #
#               Helper Functions                     #
# ================================================== #
def get_element(osm_file, tags=('node', 'way', 'relation')):
    """Yield element if it is the right type of tag"""

    context = ET.iterparse(osm_file, events=('start', 'end'))
    _, root = next(context)
    for event, elem in context:
        if event == 'end' and elem.tag in tags:
            yield elem
            root.clear()

def validate_element(element, validator, schema=SCHEMA):
    """Raise ValidationError if element does not match schema"""
    if validator.validate(element, schema) is not True:
        field, errors = next(validator.errors.iteritems())
        message_string = "\nElement of type '{0}' has the following errors:\n{1}"
        error_string = pprint.pformat(errors)

        raise Exception(message_string.format(field, error_string))

class UnicodeDictWriter(csv.DictWriter, object):
    """Extend csv.DictWriter to handle Unicode input"""

    def writerow(self, row):
        super(UnicodeDictWriter, self).writerow({
            k: (v.encode('utf-8') if isinstance(v, unicode) else v) for k, v in row.iteritems()
        })

    def writerows(self, rows):
        for row in rows:
            self.writerow(row)

# ================================================== #
#               Main Function                        #
# ================================================== #
def process_map(file_in, validate):
    """Iteratively process each XML element and write to csv(s)"""

    with codecs.open(NODES_PATH, 'w') as nodes_file, \
         codecs.open(NODE_TAGS_PATH, 'w') as nodes_tags_file, \
         codecs.open(WAYS_PATH, 'w') as ways_file, \
         codecs.open(WAY_NODES_PATH, 'w') as way_nodes_file, \
         codecs.open(WAY_TAGS_PATH, 'w') as way_tags_file:

        nodes_writer = UnicodeDictWriter(nodes_file, NODE_FIELDS)
        node_tags_writer = UnicodeDictWriter(nodes_tags_file, NODE_TAGS_FIELDS)
        ways_writer = UnicodeDictWriter(ways_file, WAY_FIELDS)
        way_nodes_writer = UnicodeDictWriter(way_nodes_file, WAY_NODES_FIELDS)
        way_tags_writer = UnicodeDictWriter(way_tags_file, WAY_TAGS_FIELDS)

        nodes_writer.writeheader()
        node_tags_writer.writeheader()
        ways_writer.writeheader()
        way_nodes_writer.writeheader()
        way_tags_writer.writeheader()

        validator = cerberus.Validator()

        for element in get_element(file_in, tags=('node', 'way')):
            el = shape_element(element)
            if el:
                if validate is True:
                    validate_element(el, validator)

                if element.tag == 'node':
                    nodes_writer.writerow(el['node'])
                    node_tags_writer.writerows(el['node_tags'])
                elif element.tag == 'way':
                    ways_writer.writerow(el['way'])
                    way_nodes_writer.writerows(el['way_nodes'])
                    way_tags_writer.writerows(el['way_tags'])

if __name__ == '__main__':
    process_map(OSM_PATH, validate=True)


'''
Write a function called two_list_dictionary which accepts two lists of varying lengths.The first list consists of keys and the second one consists of values. Your function should return a dictionary created from the keys and values. If there are not enough values, the remaining keys should have a value of null. If there not enough keys, just ignore the remaining values.

two_list_dictionary(['a', 'b', 'c', 'd'], [1, 2, 3])    # {'a': 1, 'b': 2, 'c': 3, 'd': None}
two_list_dictionary(['a', 'b', 'c']     , [1, 2, 3, 4]) # {'a': 1, 'b': 2, 'c': 3}
two_list_dictionary(['x', 'y', 'z']     , [1,2])    # {'x': 1, 'y': 2, 'z': None}
'''

def two_list_dictionary(keys, values):

    result = {}
    key_count = len(keys)
    value_count = len(values)

    i = 0
    while i < key_count:
        value = values[i] if i < value_count else None
        result[keys[i]] = value
        i += 1

    return result


import base64
import json
from Crypto.Cipher import AES
from Crypto.Util.Padding import unpad

key = "assignmentToSetu".encode('utf-8')

# Encryption class and methods
class Encryption:
    def encrypt(self, info, format):
        encryption_type = get_encryption_format(format)
        return encryption_type(info)

def get_encryption_format(format):
    if format == 'ase128ECB':
        return _aes128ecb_encryption
    else:
        raise ValueError(format)

def _aes128ecb_encryption(info):
    block_size = 16
    cipher = AES.new(key, AES.MODE_ECB)
    padded = lambda s: s + (block_size - len(s) % 16) * chr(16 - len(s) % 16)
    info = padded(info)
    response = base64.b64encode(cipher.encrypt(info.encode('utf-8')))
    return response

# Decryption class and methods
class Decryption:
    def decrypt(self, info, format):
        decryption_type = get_decryption_format(format)
        return decryption_type(info)

def get_decryption_format(format):
    if format == 'ase128ECB':
        return _aes128ecb_decryption
    else:
        raise ValueError(format)

def _aes128ecb_decryption(info):
    block_size = 16
    enc = base64.b64decode(info)
    cipher = AES.new(key, AES.MODE_ECB)
    response = unpad(cipher.decrypt(enc), block_size=block_size)
    new_response = (response.decode('utf-8')).replace("\'", "\"")
    resp = json.loads(new_response)
    return resp


import requests
from bs4 import BeautifulSoup

START_URL = 'http://m.zmnysn.61qt.cn/home/user/107067/?from=r_f_timeline&isappinstalled=0'
UserAgent = 'mozilla/5.0 (iphone; cpu iphone os 5_1_1 like mac os x) applewebkit/534.46 (khtml, like gecko) mobile/9b206 micromessenger/5.0'
headers = {'User-Agent':UserAgent}
def parse_url(url,header):
	req = requests.get(url, params=header)
	if req.raise_for_status() == 200:
		print('you are right')
	else:
		print(req.raise_for_status())

if __name__ == '__main__':
	parse_url(START_URL,headers)


import json
import time

from eigen_util import KafkaData, Singleton

class JiQiZhiXin(Singleton):

    def __init__(self, project, data, datatype):
        self.project = project
        self.data = data
        self.datatype = datatype

    def run(self, *args, **kwargs):
        if self.datatype == 'article' and 'articleid' in self.data.keys():
            self.data['postive'] = self.data['positive']
            del self.data['positive']

            today = time.strftime('%Y-%m-%d', time.localtime(time.time()))
            item_id = 'AlmostHumanSpiderItem/{}/{}'.format(today, self.data['articleid'])
            self.data['item_id'] = item_id
            del self.data['articleid']

            if not self.data['thumbnailurl'].startswith('http'):
                self.data['thumbnailurl'] = 'https://image.jiqizhixin.com' + self.data['thumbnailurl']
            send = KafkaData(self.data)
            send.push_data('AlmostHumanSpiderItem', key=b'AlmostHumanSpiderItem', value=json.dumps(self.data),
                           partition=0)
            return 'success'
        elif self.datatype == 'daily' and 'dailyid' in self.data.keys():
            today = time.strftime('%Y-%m-%d', time.localtime(time.time()))
            item_id = 'AlmostHumanNewSpiderItem/{}/{}'.format(today, self.data['dailyid'])
            self.data = {'item_id': item_id, 'meta': self.data}
            send = KafkaData(self.data)
            send.push_data('AlmostHumanNewSpiderItem', key=b'AlmostHumanNewSpiderItem', value=json.dumps(self.data),
                           partition=0)
            return 'success'
        else:
            return 'error'


import sys

def expand(filename):
    for dir in ('.', '../common', '../anstests/'):
        try:
            f = open(dir + "/" + filename)
        except IOError:
            continue
        for line in f:
            line = line.replace('\r', '')
            if line.strip().startswith('#bye'):
                sys.exit(0)
            if line.strip().startswith('include '):
                expand(line.split()[1])
            else:
                sys.stdout.write(line)
        sys.stdout.write('\n')
        return
    assert 0, filename + 'not found'

if __name__ == '__main__':
    for a in sys.argv[1:]:
        expand(a)


import numpy as np
from Neighbors import Neighbors

def FrequentWordsWithMismatches(Text, k, d):
    """
    Frequent Words with Mismatches Problem: Find the most frequent k-mers with mismatches in a string.
        Input: A string Text as well as integers k and d. (You may assume k ≤ 12 and d ≤ 3.)
        Output: All most frequent k-mers with up to d mismatches in Text.

    Sample Input:
        ACGTTGCATGTCGCATGATGCATGAGAGCT
        4
        1
    Sample Output:
        ATGC ATGT GATG
    """
    frequentPatterns = set()
    frequencyArray = ApproximateFrequencies(Text, k, d)
    maxCount = max(frequencyArray.values())

    for key in frequencyArray:
        if frequencyArray[key] == maxCount:
            frequentPatterns.add(key)

    return frequentPatterns

def ApproximateFrequencies(Text, k, d):
    """
    Count the k-mers in text, with d Hamming Distance
    """
    frequencyArray = {}

    for i in range(0, len(Text) - k + 1):
        p = Text[i:i+k]
        neighbors = Neighbors(p, d)
        for neighbor in neighbors:
            if neighbor in frequencyArray:
                frequencyArray[neighbor] += 1
            else:
                frequencyArray[neighbor] = 1

    return frequencyArray

# %%
data = np.loadtxt('./data/dataset_9_7.txt', dtype='str')
print('Text: {0}\n\nk: {1}\n\nd: {2}\n\nMost Frequent Words With Mismatches: {3}'
      .format(data[0], data[1], data[2], FrequentWordsWithMismatches(data[0], int(data[1]), int(data[2]))))


#!/usr/bin/python
# -*- coding: utf-8 -*-
# Created by dorayo@2017.4.21
'''
批量重命名文件夹下的文件
'''

import os
import sys
import argparse

def batch_rename(work_dir, old_ext, new_ext):
    '''
    This will batch rename a group of files in a given directory,
    once you pass the current and new extensions
    '''
    for filename in os.listdir(work_dir):
        # Get the file extension
        file_ext = os.path.splitext(filename)[1]
        # Start of the logic to check the file extensions, if old_ext == file_ext
        if file_ext == old_ext:
            # Returns changed name of the file with new extension
            basefilename = filename.split('.')[0]
            newname = basefilename+new_ext
            # Rename
            os.rename(
                os.path.join(work_dir, filename),
                os.path.join(work_dir, newname)
            )

def get_parser():
    parser = argparse.ArgumentParser(description='change extension of files in a working directory')
    parser.add_argument('work_dir', metavar='WORK_DIR', type=str, nargs=1, help='the directory where to change extension')
    parser.add_argument('old_ext', metavar='OLD_EXT', type=str, nargs=1, help='old extension')
    parser.add_argument('new_ext', metavar='NEW_EXT', type=str, nargs=1, help='new extension')
    return parser

def main():
    '''
    This will be called if the script is directly invoked.
    '''
    # adding command line argument
    parser = get_parser()
    args = vars(parser.parse_args())

    # Set the variable work_dir with the first argument passed
    work_dir = args['work_dir'][0]
    # Set the variable old_ext with the second argument passed
    old_ext = args['old_ext'][0]
    # Set the variable new_ext with the third argument passed
    new_ext = args['new_ext'][0]

    batch_rename(work_dir, old_ext, new_ext)

if __name__ == '__main__':
    main()


from OpenSSL import crypto

from certificates.core.models import Certificates

__all__ = ['parse_certificate', 'write_certificate_to_db']

PEM_EXT = 'pem'
COMPONENTS_MAPPING = {
    'C': 'country',
    'O': 'organization',
    'CN': 'common_name',
    'SN': 'surname',
    'emailAddress': 'email'
}

def get_certificate_type(file_):
    ext = file_.name.lower().split('.')[-1]
    if ext == PEM_EXT:
        return crypto.FILETYPE_PEM
    return crypto.FILETYPE_ASN1

def get_subject(file_):
    cert = crypto.load_certificate(get_certificate_type(file_), file_.read())
    return cert.get_subject().get_components()

def parse_components(components):
    result = {}
    for key, value in components:
        try:
            # it's possibly needs string decoding for value
            result[COMPONENTS_MAPPING[key]] = value
        except KeyError:
            continue
    return result

def parse_certificate(file_):
    try:
        subject_components = get_subject(file_)
    except Exception:
        subject_components = []

    return parse_components(subject_components)

def write_certificate_to_db(certificate):
    if not certificate:
        return
    cert = Certificates(**{k: v for k, v in certificate.iteritems()})
    cert.save()
    return cert


#!/usr/bin/env python3
#Skript zur Erstellung des Spektrums der 40min Messung inklusive Gaussfit
import sys
import argparse
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from numpy import genfromtxt
from scipy.optimize import curve_fit
import scipy
from scipy import stats
import uncertainties.unumpy as unp
from uncertainties import ufloat

heV = 4.135667696e-15
h = 6.62607017e-34
cpm = 299792458e12
c = 299792458
E_R=2.179e-18

plt.figure(dpi=300)

def wellenlänge(x):
    d=5.6402/2*1e-10
    y= 2*d*np.sin(np.radians(x))#*1e12
    return y

def gauss(x, *p):
	b, c, d,f = p
	y =  stats.skewnorm.pdf(x , 0, b, c)*d+f
	return y

file = sys.argv[1]
data = genfromtxt(file, delimiter=';')
x = data[80:150, 0]
y= data[80:150, 1]
plt.plot(wellenlänge(x)*1e12, y, 'x', ms=3)

def makegauss(min1, min2, max):
    x = data[min1:min2, 0]
    #x=wellenlänge(x)
    y = data[min1:min2, 1]
    p_initial = [max,1,4000,500]
    #e = np.array([5 for _ in y])
    popt, pcov = curve_fit(gauss, x, y, p0=p_initial)
    x = np.linspace(x[0],x[-1],num=100)
    y_fit = gauss(x, *popt)
    plt.plot(wellenlänge(x)*1e12, y_fit, color='black')
    perr = np.sqrt(np.diag(pcov))
    print('%.4f;%.4f' %(wellenlänge(popt[0])*1e12, wellenlänge(perr[0])*1e12))
    return wellenlänge(popt[0]), wellenlänge(perr[0])
    return popt[0], perr[0]

def deltawellenlänge(x):
    d=2.7e-10
    y= np.abs(2*d*np.cos(np.radians(x))*np.radians(0.1))
    return y

print('lambda')

a,b=makegauss(87,97,11)
b=deltawellenlänge(a)
Ky=ufloat(a,b)
a,b=makegauss(102,120,13)
b=deltawellenlänge(a)
Kß=ufloat(a,b)
a,b=makegauss(125,140, 15)
b=deltawellenlänge(a)
Ka=ufloat(a,b)

Zy=unp.sqrt(1/(1-1/4**2)*h*c/E_R/Ky)
Zß=unp.sqrt(1/(1-1/3**2)*h*c/E_R/Kß)
Za=unp.sqrt(1/(1-1/2**2)*h*c/E_R/Ka)
Z=(Za+Zß+Zy)*1/3
print("Z")
print(Ka*1e12)
print(Kß*1e12)
print(Ky*1e12)
#print(Z)

def energie(Z,n):
    e=Z*Z*E_R*(1/4-(1/n**2.))*6.2415096471204e18
    return e

# print(energie(65,3))
# print(energie(65,4))
# print(energie(65,5))
Ea=heV*c/Ka
Eß=heV*c/Kß
Ey=heV*c/Ky
print('{:.2f}'.format(Ea))
print('{:.2f}'.format(Eß))
print('{:.2f}'.format(Ey))

def mouse_move(event):
    x, y = event.xdata, event.ydata
    print(x, y)

#plt.connect('motion_notify_event', mouse_move)
#plt.axis('equal')
plt.ylabel('# / s$^{-1}$')
plt.xlabel('$\lambda$ / pm')
plt.savefig('bild.jpg')
plt.savefig('unbekannteAnode.svg')
#plt.show()


"""
    src.utilities
    Helper functions
"""

def slidingAverage(history, newValue, smoothingLevel):
    if len(history) == smoothingLevel:
        history.pop(0)
    history.append(newValue)
    return mean(history), history

def mean(inputList):
    cumulativeSum = 0
    for val in inputList:
        cumulativeSum += val
    if cumulativeSum != 0:
        return cumulativeSum / len(inputList)
    else:
        return 0


import sys
import fcntl
import struct
import socket
import ctypes

if sys.platform == 'darwin':
    from . import bsd_ifreqioctls as ioctls
else:
    from . import linux_ifreqioctls as ioctls

# Buffer constants
IFNAMESIZE = 16

class sockaddr_gen(ctypes.Structure):
    _fields_ = [("sa_family", ctypes.c_uint16),
                ("sa_data", (ctypes.c_uint8 * 22))]

class in_addr(ctypes.Structure):
    # AF_INET / IPv4
    _pack_ = 1
    _fields_ = [("s_addr", ctypes.c_uint32)]

class sockaddr_in(ctypes.Structure):
    _pack_ = 1
    _fields_ = [("sin_family", ctypes.c_ushort),
                ("sin_port", ctypes.c_ushort),
                ("sin_addr", in_addr),
                ("sin_zero", (ctypes.c_uint8 * 16))]

class in6_u(ctypes.Union):
    # AF_INET6 / IPv6
    _pack_ = 1
    _fields_ = [("u6_addr8", (ctypes.c_uint8 * 16)),
                ("u6_addr16", (ctypes.c_uint16 * 8)),
                ("u6_addr32", (ctypes.c_uint32 * 4))]

class in6_addr(ctypes.Structure):
    _pack_ = 1
    _fields_ = [("in6_u", in6_u)]

class sockaddr_in6(ctypes.Structure):
    _pack_ = 1
    _fields_ = [("sin6_family", ctypes.c_short),
                ("sin6_port", ctypes.c_ushort),
                ("sin6_flowinfo", ctypes.c_uint32),
                ("sin6_addr", in6_addr),
                ("sin6_scope_id", ctypes.c_uint32)]

class sockaddr_dl(ctypes.Structure):
    # AF_LINK / BSD|OSX
    _fields_ = [("sdl_len", ctypes.c_uint8),
                ("sdl_family", ctypes.c_uint8),
                ("sdl_index", ctypes.c_uint16),
                ("sdl_type", ctypes.c_uint8),
                ("sdl_nlen", ctypes.c_uint8),
                ("sdl_alen", ctypes.c_uint8),
                ("sdl_slen", ctypes.c_uint8)]

class sockaddr(ctypes.Union):
    _pack_ = 1
    _fields_ = [('gen', sockaddr_gen),
                ('in4', sockaddr_in),
                ('in6', sockaddr_in6)]

class ifmap(ctypes.Structure):
    _pack_ = 1
    _fields_ = [('mem_start', ctypes.c_ulong),
                ('mem_end', ctypes.c_ulong),
                ('base_addr', ctypes.c_ushort),
                ('irq', ctypes.c_ubyte),
                ('dma', ctypes.c_ubyte),
                ('port', ctypes.c_ubyte)]

class ifr_data(ctypes.Union):
    _pack_ = 1
    _fields_ = [('ifr_addr', sockaddr),
                ('ifr_dstaddr', sockaddr),
                ('ifr_broadaddr', sockaddr),
                ('ifr_netmask', sockaddr),
                ('ifr_hwaddr', sockaddr),
                ('ifr_flags', ctypes.c_short),
                ('ifr_ifindex', ctypes.c_int),
                ('ifr_ifqlen', ctypes.c_int),
                ('ifr_metric', ctypes.c_int),
                ('ifr_mtu', ctypes.c_int),
                ('ifr_map', ifmap),
                ('ifr_slave', (ctypes.c_ubyte*IFNAMESIZE)),
                ('ifr_newname', (ctypes.c_ubyte*IFNAMESIZE)),
                ('ifr_data', ctypes.c_void_p)]

class ifreq(ctypes.Structure):
    _pack_ = 1
    _fields_ = [('ifr_name', (ctypes.c_ubyte * IFNAMESIZE)),
                ('data', ifr_data)]

def set_ifaddress(iface, ip):
    # bin_ip = socket.inet_aton(ip)

    ifr = ifreq()
    ifr.ifr_name = (ctypes.c_ubyte * IFNAMESIZE)(*bytearray(iface))

    sin4 = sockaddr()
    sin4.in4.sin_family = socket.AF_INET
    s_addr_data = socket.inet_pton(socket.AF_INET, ip)
    sin4.in4.sin_addr.s_addr = struct.unpack('<L', s_addr_data)[0]

    ifr.data.ifr_addr = sin4

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    fcntl.ioctl(sock, ioctls.SIOCSIFADDR, ifr)


'''
The is the main program
Run python3 start.py to play the game!
'''

import pygame , random
from pygame.locals import *
from maze_gen import *
from helper import *
from Startmenu import startMenu
import time
import sys

# helper functions
def find_random_spot(listmaze):
    s = []
    for i in range(len(listmaze)):
        for j in range(len(listmaze[0])):
            if listmaze[i][j] == 1:
                s.append([j,i])
    random.shuffle(s)
    return s[0]

# make sure objects are away from exit and Starting position
def find_spot_center(listmaze):
    s = []
    for i in range(len(listmaze)):
        for j in range(len(listmaze[0])):
            if listmaze[i][j] == 1:
                s.append([j,i])
    while distance(s[0], [0,1])<6 or distance(s[0], [len(listmaze)-1,len(listmaze[0])-2])<6:
        random.shuffle(s)

    return s[0]

# keep updating the maze
def draw_maze(screen):
   for row in range(len(maze)):
       for column in range(len(maze[0])):
           screen.fill(cell_colors[maze[column][row]], get_cell_rect((row, column), screen))
           # if (maze[row][column] == 'x'):
           #     score_value += 10

def get_cell_rect(coordinates, screen):
   row, column = coordinates
   cell_width = screen.get_width() / len(maze)
   adjusted_width = cell_width - cell_margin
   return pygame.Rect(row * cell_width + cell_margin / 2, column * cell_width + cell_margin / 2, adjusted_width, adjusted_width)

# drawing all the objects
def draw_player(playerImg, screen):
   rect = playerImg.get_rect()
   rect.center = get_cell_rect(current_position, screen).center
   screen.blit(playerImg, rect)

def draw_star(star,screen,position):
    rect = star.get_rect()
    rect.center = get_cell_rect(position,screen).center
    screen.blit(star,rect)

def drawScore(score, screen):
    scoreSurf = BASICFONT.render('score: %s'%(score),True,(0,0,0))
    scoreRect = scoreSurf.get_rect()
    scoreRect.topleft = (700,5)
    screen.blit(scoreSurf,scoreRect)

# moving Character
def move(dx, dy):
    x, y = current_position
    nx, ny = x + dx, y + dy
    if nx >= 0 and nx < len(maze) and ny >= 0 and ny < len(maze[0]) and maze[ny][nx]:
       current_position[0] = nx
       current_position[1] = ny

# moving enemy
def move_enemy(enemy_position):
    tmp = Arandom_move()
    x = enemy_position[0]
    y = enemy_position[1]
    nx, ny = x + tmp[0], y + tmp[1]
    if nx >= 0 and nx < len(maze) and ny >= 0 and ny < len(maze[0]) and maze[ny][nx]:
       enemy_position[0] = nx
       enemy_position[1] = ny
    return enemy_position

# ========================================== main ==========================================
'''
A sample maze, return from maze_gen()
maze = [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
        [1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0],
        [0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0],
        [0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0],
        [0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0],
        [0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0],
        [0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0],
        [0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1],
        [0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 0],
        [0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0],
        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]
'''

# game play
def main(level, enemies):
    global score, BASICFONT, FPSCLOCK, maze, resolution

    pygame.init()

    # loading custom images for all objects
    icon = pygame.image.load('img/icon.png')
    star = pygame.image.load('img/shovel.png')
    enemy = pygame.image.load('img/enemy.png')
    playerImg = pygame.image.load('img/mushroom.png')
    exitlogo = pygame.image.load('img/exit.png')
    hint = pygame.image.load('img/hint.png')

    # set size for each object
    playerImg = pygame.transform.scale(playerImg, object_size)
    star = pygame.transform.scale(star,object_size)
    enemy = pygame.transform.scale(enemy,object_size)
    exitlogo = pygame.transform.scale(exitlogo,object_size)
    hint = pygame.transform.scale(hint, object_size)
    pygame.display.set_icon(icon)

    #print("1: {}".format(pygame.display.get_init()))
    FPSCLOCK = pygame.time.Clock()
    screen = pygame.display.set_mode(resolution)
    #print("2: {}".format(pygame.display.get_init()))
    screen.fill(cell_colors[1])
    #print("3: {}".format(pygame.display.get_init()))
    #BASICFONT = pygame.font.SysFont("comicsansms", 18)
    BASICFONT = pygame.font.Font('freesansbold.ttf', 18)
    #print("4: {}".format(pygame.display.get_init()))
    pygame.display.set_caption("Amazing Maze")
    #star_store = find_random_spot(maze)
    #hint_store = find_random_spot(maze)

    # Initial positions for hint bell and Shovel
    star_store = find_spot_center(maze)
    hint_store = find_spot_center(maze)

    hintTimer = False
    enemyTimer = True

    exit_location = [len(maze)-1,len(maze[0])-2]
    #print(len(maze), len(maze[0]))
    #print(exit_location)
    over = 0
    win = 0

    #print("start loop")

    # GAME LOOP
    while True:
        for event in pygame.event.get():
            if event.type == KEYDOWN and (win or over):
                print("key press!")
                #time.sleep(1)
                pygame.quit()
                return
            elif event.type == KEYDOWN:
                key = event.key
                # moving Character by pressing arrow keys
                if key == K_UP:
                    move(0, -1)
                elif key == K_RIGHT:
                    move(1, 0)
                elif key == K_DOWN:
                    move(0, 1)
                elif key == K_LEFT:
                    move(-1, 0)
            elif event.type == QUIT:
                # save game when quit. For load game funciton
                save_game(maze, level, current_position)
                print("maze saved")
                pygame.quit()
                sys.exit()
                #return

        # when player pick up Shovel, Destroyed a random Surrounding wall
        # Find another spot for the Shovel
        if current_position==star_store:
            star_store = find_spot_center(maze)
            #delete_random_wall(maze)
            delete_random_surround_wall(maze, current_position)
            score+=1

        # when the player get hint position, show the right path for 5 seconds
        # then Find another spot for the the hint bell
        if current_position==hint_store:
            hint_store = find_spot_center(maze)
            show_path(maze, current_position[1], current_position[0])
            start_ticks = pygame.time.get_ticks()
            hintTimer = True

        # timer start
        if (hintTimer == True):
            hintSeconds = (pygame.time.get_ticks()-start_ticks)/1000
            if (hintSeconds > 5):
                clear_path(maze)
                hintTimer = False

        # maze
        draw_maze(screen)

        # player
        draw_player(playerImg, screen)

        # star
        draw_star(star,screen,star_store)

        # score
        drawScore(score,screen)

        # hint
        draw_star(hint, screen, hint_store)

        # exit
        draw_star(exitlogo, screen, exit_location)

        # enemies move
        if (enemyTimer == True):
            enemy_start_ticks = pygame.time.get_ticks()
            enemyTimer = False

        if ((pygame.time.get_ticks()-enemy_start_ticks)/1000 > 0.2):
            enemyTimer = True
            for e in enemies:
                enemy_position = e
                enemy_position = move_enemy(enemy_position)

        # display enemies
        for e in enemies:
            draw_star(enemy,screen,e)

        if current_position in enemies:
            if not over:
                print("GAME OVER\n")
            over = 1

        if current_position == exit_location:
            if not win:
                print("WIN\n")
            win = 1

        # End-game message
        # gameover or win
        if over:
            fontB = pygame.font.SysFont("comicsansms", 150)
            fontS = pygame.font.SysFont("comicsansms", 45)
            GAMEOVER = fontB.render("GAME OVER!!!", True, (255,127,80))
            ANYKEYS = fontS.render("Press any key to quit", True, (0, 128, 0))
            screen.blit(GAMEOVER,(resolution[0]/2 - GAMEOVER.get_width() // 2, resolution[0]/2.5 - GAMEOVER.get_height() // 2))
            screen.blit(ANYKEYS,(resolution[0]/2 - ANYKEYS.get_width() // 2, resolution[0]/2 - ANYKEYS.get_height() // 2))
        elif win:
            fontB = pygame.font.SysFont("comicsansms", 150)
            fontS = pygame.font.SysFont("comicsansms", 45)
            WIN = fontB.render("GOOD JOB!!!", True, (255,127,80))
            ANYKEYS = fontS.render("Press any key to quit", True, (0, 128, 0))
            screen.blit(WIN,(resolution[0]/2 - WIN.get_width() // 2, resolution[0]/2.5 - WIN.get_height() // 2))
            screen.blit(ANYKEYS,(resolution[0]/2 - ANYKEYS.get_width() // 2, resolution[0]/2 - ANYKEYS.get_height() // 2))

        pygame.display.update()
        FPSCLOCK.tick(60)

if __name__ == "__main__":

    #levels = ['easy', 'medium', 'hard', 'insane']
    #stat = [0]
    #level = levels[1]
    #win = 0

    # Run Start Menu
    print('StartMenu')
    stat, level = startMenu()
    #print(stat, level)

    if stat[0]:
        # maze
        resolution = (850, 850)
        current_position = [0, 1]
        score = 0
        enemy_num = 0
        enemies = []

        # if play press load game
        # continue last game play
        if(stat[0] == 2):
            temp = read_maze()

            if(temp != 0):
                maze = temp[0]
                level = temp[1]
                current_position = temp[2]

        # setup objects' sizes
        # setup #Enemies beased on level
        # setup color beased on level
        if level=='easy':
            if(stat[0] == 1):
                maze = maze_gen(6, 6)

            object_size = (37, 37)
            cell_margin = 0.5
            # pink white
            cell_colors = (255, 192, 203), (255, 255, 255), (255, 255, 0)
        elif level=='medium':
            if(stat[0] == 1):
                maze = maze_gen(10, 10)

            object_size = (30, 30)
            cell_margin = 0.5
            # green white
            cell_colors = (144,238,144), (255, 255, 255), (255, 255, 0)
            #enemy_position = find_random_spot(maze)
            enemy_num = 2
        elif level=='hard':
            if(stat[0] == 1):
                maze = maze_gen(18, 18)

            object_size = (20, 20)
            cell_margin = 0.5
            # blue white
            cell_colors = (0,191,255), (255, 255, 255), (255, 255, 0)
            #enemy_position = find_random_spot(maze)
            enemy_num = 5
        elif level=='insane':
            if(stat[0] == 1):
                maze = maze_gen(26, 26)

            object_size = (15, 15)
            cell_margin = 0.5
            # red white
            cell_colors = (220,20,60), (169,169,169), (0, 0, 0)
            enemy_num = 10

        for i in range(enemy_num):
            #print(i)
            #enemies.append(find_random_spot(maze))
            enemies.append(find_spot_center(maze))

        # RUN the game
        main(level, enemies)
        print('end')

    pygame.quit()

    sys.exit()


# -*- coding: utf-8 -*-
from __future__ import print_function

import os
import sys
import random
import jieba
from time import strftime, gmtime

import pickle
import codecs
from keras.optimizers import RMSprop, Adam
from scipy.stats import rankdata
from heapq import nlargest
from gensim.models import Word2Vec

from keras_models_v3 import *

random.seed(42)

class Evaluator:
    def __init__(self, conf=None):
        serialization_data_path = 'serialization_data'
        self.path = serialization_data_path
        self.conf = dict() if conf is None else conf
        self.params = conf.get('training_params', dict())
        self.type = conf.get('type', None)
        self.answers = self.load(self.type + '_answers_v3.pkl')
        self._vocab = None
        self._reverse_vocab = None
        self._eval_sets = None

    ##### Resources #####

    def load(self, name):
        return pickle.load(open(os.path.join(self.path, name), 'rb'))

    def vocab(self):
        if self._vocab is None:
            self._vocab = self.load(self.type + '_vocabulary_v3.pkl')
        return self._vocab

    def reverse_vocab(self):
        if self._reverse_vocab is None:
            vocab = self.vocab()
            self._reverse_vocab = dict((v.lower(), k) for k, v in vocab.items())
        return self._reverse_vocab

    ##### Loading / saving #####

    def save_epoch(self, model, epoch):
        if not os.path.exists('models/'):
            os.makedirs('models/')
        model.save_weights('models/' + self.type + '_weights_epoch_%d_v3.h5' % epoch, overwrite=True)

    def load_epoch(self, model, epoch):
        assert os.path.exists('models/' + self.type + '_weights_epoch_%d_v3.h5' % epoch), 'Weights at epoch %d not found' % epoch
        model.load_weights('models/' + self.type + '_weights_epoch_%d_v3.h5' % epoch)

    ##### Converting / reverting #####

    def convert(self, words):
        rvocab = self.reverse_vocab()
        if type(words) == str:
            words = words.strip().lower().split(' ')
        return [rvocab.get(w, 0) for w in words]

    def revert(self, indices):
        vocab = self.vocab()
        return [vocab.get(i, 'X') for i in indices]

    ##### Padding #####

    def padq(self, data):
        return self.pad(data, self.conf.get('question_len', None))

    def pade(self, data):
        return self.pad(data, self.conf.get('entity_len', None))

    def padd(self, data):
        return self.pad(data, self.conf.get('des_len', None))

    def pad(self, data, len=None):
        from keras.preprocessing.sequence import pad_sequences
        return pad_sequences(data, maxlen=len, padding='post', truncating='post', value=0)

    ##### Training #####

    def print_time(self):
        print(strftime('%Y-%m-%d %H:%M:%S :: ', gmtime()), end='')

    def train(self, model):
        eval_every = self.params.get('eval_every', None)
        save_every = self.params.get('save_every', None)
        batch_size = self.params.get('batch_size', 128)
        nb_epoch = self.params.get('nb_epoch', 10)
        split = self.params.get('validation_split', 0)

        training_set = self.load(self.type + '_train_v3.pkl')

        questions = list()
        good_entities = list()
        good_descriptions = list()
        bad_answer_candidates = list()

        for q in training_set:
            questions += [q['question']] * len(q['good_answers'])
            good_entities += [self.answers[i]['entity'] for i in q['good_answers']]
            good_descriptions += [self.answers[i]['des'] for i in q['good_answers']]
            bad_answer_candidates += [q['bad_answers']] * len(q['good_answers'])

        questions = self.padq(questions)
        good_entities = self.pade(good_entities)
        good_descriptions = self.padd(good_descriptions)
        print("Questions: ", len(questions))
        print("Good answers: ", len(good_entities))

        val_loss = {'loss': 1., 'epoch': 0}

        for i in range(1, nb_epoch+1):
            # bad_answers = np.roll(good_answers, random.randint(10, len(questions) - 10))
            # bad_answers = good_answers.copy()
            # random.shuffle(bad_answers)
            # bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers)))
            neg_samples = [random.choice(candidate) for candidate in bad_answer_candidates]
            bad_entities = self.pade([self.answers[neg_sample]['entity'] for neg_sample in neg_samples])
            bad_descriptions = self.padd([self.answers[neg_sample]['des'] for neg_sample in neg_samples])
            # print("Bad answers: ", len(bad_entity_candidates))

            # shuffle questions
            # zipped = zip(questions, good_answers)
            # random.shuffle(zipped)
            # questions[:], good_answers[:] = zip(*zipped)

            print('Epoch %d :: ' % i, end='')
            self.print_time()
            model.fit([good_entities, bad_entities, questions, good_descriptions, bad_descriptions],
                      nb_epoch=1, batch_size=batch_size, validation_split=split)

            # if hist.history['val_loss'][0] < val_loss['loss']:
            #     val_loss = {'loss': hist.history['val_loss'][0], 'epoch': i}
            # print('Best: Loss = {}, Epoch = {}'.format(val_loss['loss'], val_loss['epoch']))

            if eval_every is not None and i % eval_every == 0:
                self.get_map(model)

            if save_every is not None and i % save_every == 0:
                self.save_epoch(model, i)

    ##### Evaluation #####
    def get_map(self, model, evaluate_all=False):
        top1s = list()
        top5s = list()
        top10s = list()
        maps = list()
        for name, data in self.eval_sets().items():
            if evaluate_all:
                self.print_time()
                print('----- %s -----' % name)

            random.shuffle(data)

            if not evaluate_all and 'n_eval' in self.params:
                data = data[:self.params['n_eval']]

            ap, h1, h5, h10 = 0, 0, 0, 0

            for i, d in enumerate(data):
                if evaluate_all:
                    self.prog_bar(i, len(data))

                indices = d['good_answers'] + d['bad_answers']
                entities = self.pade([self.answers[index]['entity'] for index in indices])
                descriptions = self.padd([self.answers[index]['des'] for index in indices])
                question = self.padq([d['question']] * len(indices))

                n_good = len(d['good_answers'])
                sims = model.predict([entities, question, descriptions], batch_size=len(indices)).flatten()
                r = rankdata(sims, method='ordinal')

                target_rank = np.asarray(r[:n_good])
                num_candidate = len(sims)
                ground_rank = num_candidate - target_rank + 1
                ground_rank.sort()
                one_ap = 0
                for rank in xrange(n_good):
                    one_ap += (rank + 1) / float(ground_rank[rank])
                one_ap /= n_good

                ap += one_ap
                h1 += 1 if np.argmax(sims) < n_good else 0
                h5 += 1 if set(list(ground_rank - 1)).intersection(set(range(5))) else 0
                h10 += 1 if set(list(ground_rank - 1)).intersection(set(range(10))) else 0

                # max_r = np.argmax(sims)
                # max_n = np.argmax(sims[:n_good])

                # print(' '.join(self.revert(d['question'])))
                # print(' '.join(self.revert(self.answers[indices[max_r]])))
                # print(' '.join(self.revert(self.answers[indices[max_n]])))

                # c_1 += 1 if max_r == max_n else 0
                # c_2 += 1 / float(r[max_r] - r[max_n] + 1)

            top1 = h1 / float(len(data))
            top5 = h5 / float(len(data))
            top10 = h10 / float(len(data))
            mean_ap = ap / float(len(data))

            del data

            if evaluate_all:
                print('Top-1 Precision: %f' % top1)
                print('Hit@5 Precision: %f' % top5)
                print('Hit@10 Precision: %f' % top10)
                print('MAP: %f' % mean_ap)

            top1s.append(top1)
            top5s.append(top5)
            top10s.append(top10)
            maps.append(mean_ap)

        # rerun the evaluation if above some threshold
        if not evaluate_all:
            print('Top-1 Precision: {}'.format(top1s))
            print('Hit@5 Precision: {}'.format(top5s))
            print('Hit@10 Precision: {}'.format(top10s))
            print('MAP: {}'.format(maps))
            evaluate_all_threshold = self.params.get('evaluate_all_threshold', dict())
            evaluate_mode = evaluate_all_threshold.get('mode', 'all')
            map_threshold = evaluate_all_threshold.get('map', 1)
            top1_threshold = evaluate_all_threshold.get('top1', 1)

            if evaluate_mode == 'any':
                evaluate_all = evaluate_all or any([x >= top1_threshold for x in top1s])
                evaluate_all = evaluate_all or any([x >= map_threshold for x in maps])
            else:
                evaluate_all = evaluate_all or all([x >= top1_threshold for x in top1s])
                evaluate_all = evaluate_all or all([x >= map_threshold for x in maps])

            if evaluate_all:
                return self.get_map(model, evaluate_all=True)

        return top1s, top5s, top10s, maps

    def prog_bar(self, so_far, total, n_bars=20):
        n_complete = int(so_far * n_bars / total)
        if n_complete >= n_bars - 1:
            print('\r[' + '=' * n_bars + ']', end='')
        else:
            s = '\r[' + '=' * (n_complete - 1) + '>' + '.' * (n_bars - n_complete) + ']'
            print(s, end='')

    def eval_sets(self):
        if self._eval_sets is None:
            self._eval_sets = dict([(s, self.load(s)) for s in [self.type + '_test_v3.pkl']])
        return self._eval_sets

    def max_sim(self, left_list, right_list, w2v_model):
        max_cos = -1
        for i in range(len(left_list)):
            for j in range(len(right_list)):
                try:
                    local_sim = w2v_model.similarity(left_list[i], right_list[j])
                    max_cos = local_sim if local_sim > max_cos else max_cos
                except KeyError:
                    pass
        return max_cos

    def make_submit(self, model, dev, submit_file):
        lines = dev.readlines()
        target_lines = list()
        data = self.eval_sets().values()[0]
        for i, d in enumerate(data):
            terms = lines[i].strip('\n').split('\t')
            query = terms[0]
            entities = terms[1:]
            num_candidate = len(entities)
            index_entities = xrange(num_candidate)

            indices = d['answers']
            answers = self.pada([self.answers[i] for i in indices])
            question = self.padq([d['question']] * len(indices))

            sims = model.predict([question, answers], batch_size=100).flatten()
            print(len(sims))
            r = rankdata(sims, method='ordinal')
            index_candidates = nlargest(num_candidate, index_entities, key=lambda j: r[j])
            for index_candidate in index_candidates:
                query = query + '\t' + entities[index_candidate]
            query += '\n'
            target_lines.append(query)
        submit_file.writelines(target_lines)

        del data

    def make_submit_v2(self, model, dev, submit_file, w2v_model):
        lines = dev.readlines()
        target_lines = list()
        data = self.eval_sets().values()[0]
        for i, d in enumerate(data):
            terms = lines[i].strip('\n').split('\t')
            query = terms[0]
            query_word_list = [x for x in jieba.cut(query)]
            entities = terms[1:]
            num_candidate = len(entities)
            sim_candidate = list()

            for entity in entities:
                sim_candidate.append(self.max_sim(query_word_list, [y for y in jieba.cut(entity)], w2v_model))

            sim_candidate = np.asarray(sim_candidate)

            index_entities = xrange(num_candidate)

            indices = d['answers']
            answers = self.pada([self.answers[i] for i in indices])
            question = self.padq([d['question']] * len(indices))

            sims = model.predict([question, answers], batch_size=100).flatten()
            sims = sim_candidate * 0.6 + sims * 0.4
            print(len(sims))
            r = rankdata(sims, method='ordinal')
            index_candidates = nlargest(num_candidate, index_entities, key=lambda j: r[j])
            for index_candidate in index_candidates:
                query = query + '\t' + entities[index_candidate]
            query += '\n'
            target_lines.append(query)
        submit_file.writelines(target_lines)

        del data

if __name__ == '__main__':
    conf = {
        'type': 'tvShow',
        'question_len': 8,
        'entity_len': 4,
        'des_len': 2,
        'n_words': 9843,  # len(vocabulary) + 1
        'margin': 0.02,

        'training_params': {
            'save_every': 1000,
            'eval_every': 10,
            'batch_size': 32,
            'nb_epoch': 3000,
            'validation_split': 0,
            'optimizer': 'adam',
            # 'optimizer': Adam(clip_norm=0.1),
            # 'n_eval': 100,

            'evaluate_all_threshold': {
                'mode': 'all',
                'top1': 0.5,
            },
        },

        'model_params': {
            'n_embed_dims': 300,
            'n_hidden': 200,

            # convolution
            'nb_filters': 1000, # * 4
            'conv_activation': 'relu',

            # recurrent
            'n_lstm_dims': 141, # * 2

            'initial_embed_weights': np.load('embeddings/tvShow_300_dim_v3.embeddings'),
        },

        'similarity_params': {
            'mode': 'cosine',
            'gamma': 1,
            'c': 1,
            'd': 2,
        }
    }

    evaluator = Evaluator(conf)

    ##### Define model ######
    model = EmbeddingModel(conf)
    optimizer = conf.get('training_params', dict()).get('optimizer', 'adam')
    model.compile(optimizer=optimizer)

    import numpy as np

    # save embedding layer
    # evaluator.load_epoch(model, 33)
    # embedding_layer = model.prediction_model.layers[2].layers[2]
    # evaluator.load_epoch(model, 100)
    # evaluator.train(model)
    # weights = embedding_layer.get_weights()[0]
    # np.save(open('models/embedding_1000_dim.h5', 'wb'), weights)

    # train the model
    evaluator.train(model)
    # evaluate mrr for a particular epoch
    evaluator.load_epoch(model, 3000)
    evaluator.get_map(model, evaluate_all=True)

    # celebrity
    # evaluator.load_epoch(model, 3000)
    # dev_set = codecs.open('data/celebrity.TESTSET.txt', 'rb', 'gb18030')
    # submit = codecs.open('data/celebrity-final-v1.txt', 'wb', 'gb18030')
    # wor2vec_model = Word2Vec.load_word2vec_format('data/baike_vector.bin', binary=True)
    # evaluator.make_submit_v2(model, dev_set, submit, wor2vec_model)

    # movie
    # evaluator.load_epoch(model, 3000)
    # dev_set = codecs.open('data/movie.TESTSET.txt', 'rb', 'gb18030')
    # submit = codecs.open('data/movie-final-v1.txt', 'wb', 'gb18030')
    # evaluator.make_submit(model, dev_set, submit)

    # restaurant
    # evaluator.load_epoch(model, 3000)
    # dev_set = codecs.open('data/restaurant.TESTSET.txt', 'rb', 'gb18030')
    # submit = codecs.open('data/restaurant-final-v1.txt', 'wb', 'gb18030')
    # evaluator.make_submit(model, dev_set, submit)

    # tvShow
    # evaluator.load_epoch(model, 3000)
    # dev_set = codecs.open('data/tvShow.TESTSET.txt', 'rb', 'gb18030')
    # submit = codecs.open('data/tvShow-final-v1.txt', 'wb', 'gb18030')
    # evaluator.make_submit(model, dev_set, submit)


#coding=utf-8

import sys
import socket
import json
import base64
from binascii import unhexlify

from lorawanPkt import LoRaWANPkt
from consts import *
from utils import loraPktPrettyPrint

def recvThread(NwkSKey, AppSKey):
	'''
	Thread para recepção de mensagens de dados e de status
	'''
	count = 1
	print('Iniciando thread de recepcao')
	# Cria socket UDP para recepção de mensagens
	recvSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
	try:
		recvSocket.bind(('', 13509))
	except OSError:
		print('Endereco utilizado. Saindo...')
		sys.exit()
	while True:
		data, host = recvSocket.recvfrom(1024)
		# print('Got: ' + str(data) + ' from: ' + str(host))
		if data[3] == PKT_PUSH_DATA:
			# Dados enviados para o servidor. 12 primeiros bytes são informações do GW
			try:
				message = json.loads(data[12:].decode('utf-8'))
				if 'stat' in message:
					ackMsg = bytes([PROTOCOL_VERSION]) + data[1:3] + bytes([PKT_PUSH_ACK])
					print('Mensagem de status recebida. Mandando ACK: ' + ackMsg.hex() + ' de ' + str(host))
					recvSocket.sendto(ackMsg, host)
				elif 'rxpk' in message:
					# print('RXPK pkt received' + '\n' + json.dumps(message, indent=4))
					for i in message['rxpk']:
						# data está em base64. Necessario transformar para hexadecimal
						print('\n\tMensagem do tipo RXPK recebida: ' + base64.b64decode(i['data']).hex())
						pkt = LoRaWANPkt( base64.b64decode(i['data']).hex(), NwkSKey, AppSKey )
						loraPktPrettyPrint(pkt)
						# Montar uma mensagem para transmissao numa das janelas 1 -> 1s, 2 -> 2s
						pkt.setDownlinkLoRaPktMsg(UNCONFIRMED_DATA_DOWN, pkt.getDevAddr(), False, False, False, 0, count, '', 2, 'Hello from network server', NwkSKey, AppSKey)
						# Janela 1
						downLinkMessage = dict()
						downLinkMessage['txpk'] = dict()
						# Assumindo dispositivos classe A
						downLinkMessage['txpk']['imme'] = True
						downLinkMessage['txpk']['tmst'] = i['tmst'] + 1000000
						downLinkMessage['txpk']['freq'] = 923.3
						# downLinkMessage['txpk']['rfch']
						downLinkMessage['txpk']['powe'] = 20
						downLinkMessage['txpk']['modu'] = 'LORA'
						downLinkMessage['txpk']['datr'] = i['datr']
						downLinkMessage['txpk']['codr'] = i['codr']
						downLinkMessage['txpk']['ipol'] = False
						downLinkMessage['txpk']['size'] = len(pkt.getLoRaPktMsg()) // 2
						downLinkMessage['txpk']['data'] = base64.b64encode(unhexlify(pkt.getLoRaPktMsg())).decode('utf-8')
						# print('We will send this json: \n' + json.dumps(downLinkMessage, indent=4))
						count += 1
						sendSocket.sendto( bytes([PROTOCOL_VERSION, token_h, token_l, PKT_PULL_RESP]) + json.dumps(downLinkMessage).encode('utf-8'), sendHost )
						# Janela 2
						downLinkMessage['txpk']['tmst'] = i['tmst'] + 2000000
						downLinkMessage['txpk']['freq'] = 923.3
						downLinkMessage['txpk']['datr'] = 'SF12BW500'

						sendSocket.sendto( bytes([PROTOCOL_VERSION, token_h, token_l, PKT_PULL_RESP]) + json.dumps(downLinkMessage).encode('utf-8'), sendHost )

				else:
					print('Outros tipos de mensagem precisam de tratamento...')

			except UnicodeDecodeError:
				print('JSON nao eh decodificavel')

			except Exception as e:
				print('Ocorreu erro' + str(e))

def sendThread():
	'''
	Thread para recepção de mensagens de downlink
	'''
	print('Iniciando thread de transmissao...')
	# Cria socket UDP para transmissão de mensagens e atualização de status
	global sendSocket
	global sendHost
	global token_h
	global token_l
	sendSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
	try:
		sendSocket.bind(('', 13510))
	except OSError:
		print('Endereco utilizado. Saindo...')
		sys.exit()
	while True:
		data, sendHost = sendSocket.recvfrom(64)
		token_h = data[1]
		token_l = data[2]
		# print('Got: ' + str(data) + ' from: ' + str(sendHost))
		if data[3] == PKT_PULL_DATA:
			# Mandando infomacao do GW para o servidor de aplicacao. Necessario mandar um ACK
			# com os tokens corretos
			print('Mensagem com informacao de token recebida: ' + data[1:3].hex() + ' de ' + str(sendHost))
			ackMsg = bytes([PROTOCOL_VERSION]) + data[1:3] + bytes([PKT_PULL_ACK])
			sendSocket.sendto(ackMsg, sendHost)
		elif data[3] == PKT_TX_ACK:
			print('Ack recebido')
		else:
			print('Outros tipos de mensagem precisam de tratamento 2...')


# -*- coding: utf-8 -*-
"""
Created on Thu Sep  5 23:15:03 2019

@author: ASUS
"""

import pandas  as pd
import tushare as tu
import matplotlib
import matplotlib.pyplot as plt
import datetime

#买卖函数
def buy(code,money,day):
    global total
    global handle
    global cost
    global df
    global handles

    str(code)
    str(day)
    data=tu.get_hist_data(code,day,day)
    buy_price=float(data['open'])
    cost = money/buy_price
    handles = handles+cost
    handle = {code:handles*buy_price}
    total = total - money
    print('buy',buy_price)
    print('handle',handle[code])
    print('total',total)
    print('final',total+handle[code])
    return buy_price

def sell(code,percent,day):
    global total
    global handle
    global cost
    global df
    global handles

    str(code)
    str(day)
    data=tu.get_hist_data(code,day,day)
    sell_price=float(data['close'])
    handle = {code:handles*(1-percent)*sell_price}
    total = total + sell_price*handles*percent
    handles = handles*(1-percent)
    print('sell',sell_price)
    print('handle',handle[code])
    print('total',total)
    print('final',total+handle[code])
    return sell_price

total = 100
buy_money = total*0.1
sell_percent = 1
day_start = '2016-01-01'
day_end   = '2019-10-21'
code = '600438'

#draw = pd.DataFrame()
buydata = pd.DataFrame()
selldata = pd.DataFrame()
flag = 0
index = 0
buyp = []
sellp = []
buyt = []
sellt = []
handle = pd.DataFrame()
truerange = []
i = 0
flag = 'out'
handles = 0
fil = []
key=1

#交易逻辑
def get_datas(code):
    data=tu.get_hist_data(code,day_start,day_end)
    data = data.sort_index()
    truerange = []
    maxdata = []
    mindata = []
    high=data['high']
    low=data['low']
    close=data['close']
    index = 0

    for item in data.index:
        tr = max(high[index-1]-low[index-1],high[index]-close[index],close[index]-low[index])
        truerange.append(tr)
        trueranges = pd.DataFrame(data=truerange)
        peranges = trueranges.rolling(window=5,center=False).mean()
        if index<21:
            maxd = max(close[0:index+1])
            mind = min(close[0:index+1])
        else:
            maxd = max(close[index-20:index])
            mind = min(close[index-20:index])
        maxdata.append(maxd)
        mindata.append(mind)
        index = index+1
    #print(perange)
    return peranges,data,maxdata,mindata

peranges,data,maxdata,mindata = get_datas(code)
for item in data.index:
    if  data.loc[item,'close'] >  maxdata[i] and flag == 'out':
       buy_money = total*0.7
       bp = buy(code,buy_money,item)
       buyp.append(bp)
       buyt.append(item)
       flag = 1
       flag = 'in0'
    if  data.loc[item,'close'] <  mindata[i] and flag != 'out':
       sell_percent = 1
       sp=sell(code,sell_percent,item)
       sellp.append(sp)
       sellt.append(item)
       flag = 'out'
       key = 1

    if data.loc[item,'p_change'] > float(peranges.loc[i]) and flag == 'in0' :
       buy_money = uint
       bp = buy(code,buy_money,item)
       key = key+1
       if key >5:
         flag = 'in1'
         print('d')
       buyp.append(bp)
       buyt.append(item)

    if data.loc[item,'p_change'] < float(-peranges.loc[i]) and flag != 'out':
       sell_percent = 1
       sp=sell(code,sell_percent,item)
       sellp.append(sp)
       sellt.append(item)
       flag = 'out'
       key = 1

    handle = {code:handles*data.loc[item,'close']}
    final = total+handle[code]
    uint = float(abs(final*0.01/peranges.loc[i]))
    i = i+1
    fil.append(total+handle[code]-100)

#数据整理与绘图
data_test = {'buy_price':buyp,'sellprice':sellp}
draw = data['close']
buydata['data'] = buyt
buydata['price'] = buyp
selldata['data'] = sellt
selldata['price'] = sellp

datab_list = []
for i in buydata['data']:
    dateb_time = datetime.datetime.strptime(i,'%Y-%m-%d')
    t = matplotlib.dates.date2num(dateb_time)
    datab_list.append(t)
buydata['data'] = datab_list

datas_list = []
for i in selldata['data']:
    dates_time = datetime.datetime.strptime(i,'%Y-%m-%d')
    t = matplotlib.dates.date2num(dates_time)
    datas_list.append(t)
selldata['data'] = datas_list

draw = draw.reset_index()
draw_list = []
for i in draw['date']:
    dates_time = datetime.datetime.strptime(i,'%Y-%m-%d')
    t = matplotlib.dates.date2num(dates_time)
    draw_list.append(t)
draw['date'] = draw_list
opr = plt.subplot2grid((2,1), (0,0))
res = plt.subplot2grid((2,1), (1,0))

opr.plot(buydata['data'],buydata['price'],'bo')
opr.plot(selldata['data'],selldata['price'],'bo',color='red')
opr.plot(draw['date'],draw['close'])
res.plot(fil)


#calss header
class _TORNADO():
	def __init__(self,):
		self.name = "TORNADO"
		self.definitions = [u'a strong, dangerous wind that forms itself into an upside-down spinning cone and is able to destroy buildings as it moves across the ground']

		self.parents = []
		self.childen = []
		self.properties = []
		self.jsondata = {}

		self.specie = 'nouns'

	def run(self, obj1 = [], obj2 = []):
		return self.jsondata


import numpy as np

class Norm:
    def __init__(self, Xtr_features, Xte_features):
        self.NUMBER_OF_FEATURES = 7

        self.Xtr_features = Xtr_features
        self.Xte_features = Xte_features

        self.means = None  # NUMBER_OF_FEATURES x 1, vector of means
        self.sigmas = None  # NUMBER_OF_FEATURES x 1, vector of standard deviations

        # Normalized features
        self.nXtr_features = None
        self.nXte_features = None

        # Classification
        self.classify = None

        # Do normalization
        self.determine_means()
        self.determine_sigmas()

        self.normalize_training_data()
        self.normalize_testing_data()

    def determine_means(self):
        means = np.zeros((1, self.NUMBER_OF_FEATURES))
        for sample in self.Xtr_features:
            means = np.add(means, sample)

        means = np.multiply(means, 1.0/len(self.Xtr_features))
        self.means = np.transpose(means)

    def determine_sigmas(self):
        means_transpose = np.transpose(self.means)
        sigmas = np.zeros((1, self.NUMBER_OF_FEATURES))

        for sample in self.Xtr_features:
            diff = sample - means_transpose
            sigmas = np.add(sigmas, np.square(diff))

        sigmas = np.multiply(sigmas, 1.0 / len(self.Xtr_features))
        sigmas = np.sqrt(sigmas)
        self.sigmas = np.transpose(sigmas)

    def normalize(self, features):
        normalized_features = []
        means_transpose = np.transpose(self.means)
        sigmas_transpose = np.transpose(self.sigmas)
        for sample in features:
            normalized_features.append(np.divide((sample - means_transpose), sigmas_transpose))

        return normalized_features

    def normalize_training_data(self):
        self.nXtr_features = self.normalize(self.Xtr_features)

    def normalize_testing_data(self):
        self.nXte_features = self.normalize(self.Xte_features)