#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Matrix Decomposition: latent factor model
"""

#==============================================================================
# Module metadata variables
#==============================================================================

#==============================================================================
# Imports
#==============================================================================

import logging
import sys
import copy
import numpy as np
from scipy.optimize import fmin, fmin_powell

from .recommenders import BaseEventNIScorePredictor
from pyrecsys.md import latent_factor

#==============================================================================
# Public symbols
#==============================================================================

__all__ = ['EventNIScorePredictor']

#==============================================================================
# Constants
#==============================================================================

#==============================================================================
# Module variables
#==============================================================================

#==============================================================================
# Classes
#==============================================================================

class EventNIScorePredictor(BaseEventNIScorePredictor):
    """
    Information neutralized pyrecsys.md.latent_factor.EventScorePredictor

    Parameters
    ----------
    C : float, optional
        regularization parameter (= :math:`\lambda`), default=1.0
    eta : float, optional
        parameter of information neutrality term (= :math:`\eta`),
        default=1.0
    k : int, optional
        the number of latent factors (= sizes of :math:`\mathbf{p}_u` or
        :math:`\mathbf{q}_i`), default=1

    Attributes
    ----------
    `mu_` : array_like
        grobal biases
    `bu_` : array_like
        users' biases
    `bi_` : array_like
        items' biases
    `p_` : array_like
        latent factors of users
    `q_` : array_like
        latent factors of items
    `n_trgvars_` : int
        the numbers of possible target variables
    `i_loss_` : float
        the loss value after initialization
    `f_loss_` : float
        the loss value after fitting
    """

    def __init__(self, C=1.0, eta=1.0, k=1):
        super(EventNIScorePredictor, self).__init__()

        self.C = np.float(C)
        self.eta = np.float(eta)
        self.k = np.int(k)
        self.coef_ = None
        self.mu_ = None
        self.bu_ = None
        self.bi_ = None
        self.p_ = None
        self.q_ = None
        self.n_trgvars_ = 0
        self.i_loss_ = np.inf
        self.f_loss_ = np.inf

    def _init_coef(self, orig_data, tev, tsc, n_objects, **kwargs):
        """
        Initialize model parameters

        Parameters
        ----------
        orig_data : pyrecsys.data.EventWithScoreData
            data before separated depending on vals of target vars
        tev : array, shape(n_events, 2)
            separated event data
        tsc : array, shape(n_events,)
            scores attached to separated events
        n_objects : array, shape(2,)
            vector of numbers of objects
        kwargs : keyword arguments
            parameters for optmizers
        """

        # constants
        n_users = n_objects[0]
        n_items = n_objects[1]
        k = self.k
        n_trgvars = self.n_trgvars_

        # define dtype for parameters
        self._dt = np.dtype([
            ('mu', np.float, (1,)),
            ('bu', np.float, (n_users,)),
            ('bi', np.float, (n_items,)),
            ('p', np.float, (n_users, k)),
            ('q', np.float, (n_items, k))
        ])

        # memory allocation
        dt_itemsize = 1 + n_users + n_items +\
                      n_users * k + n_items * k
        self.coef_ = np.zeros(dt_itemsize * n_trgvars, dtype=np.float)

        # set array's view
        self.mu_ = self.coef_.view(self._dt)['mu']
        self.bu_ = self.coef_.view(self._dt)['bu']
        self.bi_ = self.coef_.view(self._dt)['bi']
        self.p_ = self.coef_.view(self._dt)['p']
        self.q_ = self.coef_.view(self._dt)['q']

        # init model by normal recommenders
        data = copy.copy(orig_data)
        recommender = latent_factor.EventScorePredictor(C=self.C, k=k)
        for t in xrange(n_trgvars):
            data.event = tev[t]
            data.n_events = tev[t].shape[0]
            data.score = tsc[t]
            recommender.fit(data, **kwargs)

            self.mu_[t][:] = recommender.mu_
            self.bu_[t][:] = recommender.bu_[:n_users]
            self.bi_[t][:] = recommender.bi_[:n_items]
            self.p_[t][:, :] = recommender.p_[:n_users, :]
            self.q_[t][:, :] = recommender.q_[:n_items, :]

        # scale a regularization term by the number of parameters
        self._reg = self.C / np.float((dt_itemsize - 2 - 2 * k) * n_trgvars)

    def loss(self, coef, tev, tsc, n_objects, score_bins):
        """
        loss function to optimize

        main loss function: same as the pyrecsys.md.latent_factor.

        information neutrality term:

        To estimate distribution of estimated scores, we adopt a histogram
        model. Estimated scores are first discretized into bins according to
        `score_bins`. Distributions are derived from the frequencies of
        events in these bins.

        \sum_{u, i, t} \sum_{s \in bins} \
            I(bin(\hat{s}(u, i, t)) == s) log p[s | t] - log p[s]

        u, i, t are user_index, item_index, information neutral target var,
        respectively. \hat(s)(u, i, t) is estimated score, bin(s) is a
        function that returns index of the bin to which s belongs.
        Distribution p[y | t] is derived by generating histogram from a data
        set that is consists of events whose target values equal to t.
        p[s] = \sum_t \p[t] p[s | t].

        Parameters
        ----------
        coef : array_like, shape=(variable,)
            coefficients of this model
        tev : array_like, shape(n_events, 2), dtype=int
            user and item indexes
        tsc : array_like, shape(n_events,), dtype=float
            target scores
        n_objects : array_like, shape(2,), dtype=int
            numbers of users and items
        score_bins : array_like, shape=(variable,) dtype=float, optional
            thresholds to discretize scores.

        Returns
        -------
        loss : float
            value of loss function
        """
        # constants
        n_events = np.array([ev.shape[0] for ev in tev])
        n_users = n_objects[0]
        n_items = n_objects[1]
        k = self.k

        # set array's view
        mu = coef.view(self._dt)['mu']
        bu = coef.view(self._dt)['bu']
        bi = coef.view(self._dt)['bi']
        p = coef.view(self._dt)['p']
        q = coef.view(self._dt)['q']

        # basic stats
        esc = np.empty(self.n_trgvars_, dtype=object)
        pesct = np.empty((self.n_trgvars_, len(score_bins) - 1),
                         dtype=np.float)
        for t in xrange(self.n_trgvars_):
            ev = tev[t]
            esc[t] = mu[t][0] + bu[t][ev[:, 0]] + bi[t][ev[:, 1]] +\
                     np.sum(p[t][ev[:, 0], :] * q[t][ev[:, 1], :], axis=1)
            pesct[t, :] = np.histogram(esc[t], score_bins)[0]
        pesc = np.sum(pesct, axis=0) / np.sum(n_events)
        pesct = pesct / n_events[:, np.newaxis]

        # loss term and information neutrality term
        loss = 0.0
        for t in xrange(self.n_trgvars_):
            loss += np.sum((tsc[t] - esc[t]) ** 2)

        # information neutrality term
        in_term = 0.0
        for t in xrange(self.n_trgvars_):
            # NOTE: to avoid 0 in log
            pos = np.nonzero(pesct[t, :] > 0.0)[0]
            in_term += np.dot(pesct[t, pos],
                              np.log(pesct[t, pos]) - np.log(pesc[pos])) *\
                       n_events[t]

        # regularization term
        reg = 0.0
        for t in xrange(self.n_trgvars_):
            reg += np.sum(bu[t] ** 2) + np.sum(bi[t] ** 2) +\
                   np.sum(p[t] ** 2) + np.sum(q[t] ** 2)

        return loss / np.float(np.sum(n_events)) +\
               self.eta * in_term / np.float(np.sum(n_events)) +\
               self._reg * reg

    def fit(self, data, trg, n_trgvars=2,
            user_index=0, item_index=1, score_index=0,
            score_bins=None, **kwargs):
        """
        fitting model

        Parameters
        ----------
        data : :class:`pyrecsys.data.EventWithScoreData`
            data to fit
        trg : array_like, shape=(n_events,)
            a variable to be neutral in recommendation
        n_trgvars : int
            the numbers of possible target variables
        user_index : optional, int
            Index to specify the position of a user in an event vector.
            (default=0)
        item_index : optioanl, int
            Index to specify the position of a item in an event vector.
            (default=1)
        score_index : optional, int
            Ignored if score of data is a single criterion type. In a multi-
            criteria case, specify the position of the target score in a score
            vector. (default=0)
        score_bins : array_like, ndim=1
            a vector of thresholds to separate scores to bins
            default=[-np.inf, 1.5, 2.5, 3.5, 4.5, np.inf],
            for 1 to 5 rating scores
        kwargs : keyowrd arguments
            keyword arguments passed to optimizers
        """

        # set parameters
        self.n_trgvars_ = n_trgvars
        if score_bins is None:
            score_bins = np.array([-np.inf, 1.5, 2.5, 3.5, 4.5, np.inf])

        # get input data
        ev, sc, n_objects =\
        self._get_event_and_score(data,
            (user_index, item_index), score_index)

        # divide events and scores according to the corresponding target
        # variables
        tev = np.empty(self.n_trgvars_, dtype=np.object)
        tsc = np.empty(self.n_trgvars_, dtype=np.object)
        for t in xrange(self.n_trgvars_):
            tev[t] = ev[trg == t, :]
            tsc[t] = sc[trg == t]

        # initialize coefficients
        self._init_coef(data, tev, tsc, n_objects,
                        gtol=kwargs.pop('gtol', 1e-05),
                        maxiter=kwargs.get('maxiter', None),
                        disp=kwargs.get('disp', False))

        # check optimization parameters
        if not 'disp' in kwargs:
            kwargs['disp'] = False
        if 'gtol' in kwargs:
            del kwargs['gtol']

        # get final loss
        self.i_loss_ = self.loss(self.coef_, tev, tsc, n_objects, score_bins)

        # optimize model
        self.coef_[:] = fmin_powell(self.loss,
                                    self.coef_,
                                    args=(tev, tsc, n_objects, score_bins),
                                    **kwargs)

        # add parameters for unknown users and items
        self.bu_ = np.empty(self.n_trgvars_, dtype=object)
        self.bi_ = np.empty(self.n_trgvars_, dtype=object)
        self.p_ = np.empty(self.n_trgvars_, dtype=object)
        self.q_ = np.empty(self.n_trgvars_, dtype=object)
        for t in xrange(self.n_trgvars_):
            self.bu_[t] = np.r_[self.coef_.view(self._dt)['bu'][t], 0.0]
            self.bi_[t] = np.r_[self.coef_.view(self._dt)['bi'][t], 0.0]
            self.p_[t] = np.r_[self.coef_.view(self._dt)['p'][t],
                               np.zeros((1, self.k), dtype=np.float)]
            self.q_[t] = np.r_[self.coef_.view(self._dt)['q'][t],
                               np.zeros((1, self.k), dtype=np.float)]

        # get final loss
        self.f_loss_ = self.loss(self.coef_, tev, tsc, n_objects, score_bins)

        # clean up temporary instance variables
        del self.coef_
        del self._reg
        del self._dt

    def raw_predict(self, ev, trg):
        """
        predict score of given one event represented by internal ids

        Parameters
        ----------
        (user, item) : array_like
            a target user's and item's ids. unknwon objects assumed to be
            represented by n_object[event_otype]

        Returns
        -------
        ev : array_like, shape=(s_events,) or (variable, s_events)
            events for which scores are predicted
        trg : int or array_like, dtype=int
            target values to enhance information neutrality

        Raises
        ------
        TypeError
            shape of an input array is illegal
        """

        if ev.ndim == 1:
            return self.mu_[trg][0] +\
                   self.bu_[trg][ev[0]] + self.bi_[trg][ev[1]] +\
                   np.dot(self.p_[trg][ev[0]], self.q_[trg][ev[1]])
        elif ev.ndim == 2:
            return np.array([self.mu_[t][0] +\
                             self.bu_[t][ev[i, 0]] + self.bi_[t][ev[i, 1]] +\
                             np.dot(self.p_[t][ev[i, 0], :],
                                    self.q_[t][ev[i, 1], :])
                             for i, t in enumerate(trg)])
        else:
            raise TypeError

#==============================================================================
# Functions
#==============================================================================

#==============================================================================
# Module initialization
#==============================================================================

# init logging system ---------------------------------------------------------

logger = logging.getLogger('pyrecsys')
if not logger.handlers:
    logger.addHandler(logging.NullHandler)

#==============================================================================
# Test routine
#==============================================================================

def _test():
    """ test function for this module
    """

    # perform doctest
    import sys
    import doctest

    doctest.testmod()

    sys.exit(0)

# Check if this is call as command script -------------------------------------

if __name__ == '__main__':
    _test()


import numpy as np

def main():
    a = 1.0 / np.float32(0.0) # 無限大の生成
    print(f"a={a} ")
    b = 1.0 / a               # 無限大による除算
    print(f"b={b} ")

if __name__ == "__main__":
    main()


import time
import pytest
from pytest import raises, mark
import pydle
from .fixtures import with_client
from .mocks import Mock

pydle.client.PING_TIMEOUT = 10

## Initialization.

@pytest.mark.asyncio
@with_client(invalid_kwarg=False)
def test_client_superfluous_arguments(server, client):
    assert client.logger.warning.called

## Connection.
@pytest.mark.asyncio
@with_client()
async def test_client_reconnect(server, client):
    await client.disconnect(expected=True)
    assert not client.connected

    await client.connect(reconnect=True)
    assert client.connected

@pytest.mark.asyncio
@mark.slow
@with_client()
async def test_client_unexpected_disconnect_reconnect(server, client):
    client._reconnect_delay = Mock(return_value=0)
    await client.disconnect(expected=False)
    assert client._reconnect_delay.called

    time.sleep(0.1)
    assert client.connected

@pytest.mark.asyncio
@with_client()
async def test_client_unexpected_reconnect_give_up(server, client):
    client.RECONNECT_ON_ERROR = False
    await client.disconnect(expected=False)
    assert not client.connected

@pytest.mark.asyncio
@mark.slow
@with_client()
async def test_client_unexpected_disconnect_reconnect_delay(server, client):
    client._reconnect_delay = Mock(return_value=1)
    await client.disconnect(expected=False)

    assert not client.connected
    time.sleep(1.1)
    assert client.connected

@pytest.mark.asyncio
@with_client()
def test_client_reconnect_delay_calculation(server, client):
    client.RECONNECT_DELAYED = False
    assert client._reconnect_delay() == 0

    client.RECONNECT_DELAYED = True
    for expected_delay in client.RECONNECT_DELAYS:
        delay = client._reconnect_delay()
        assert delay == expected_delay

        client._reconnect_attempts += 1

    assert client._reconnect_delay() == client.RECONNECT_DELAYS[-1]

@pytest.mark.asyncio
@with_client()
async def test_client_disconnect_on_connect(server, client):
    client.disconnect = Mock()

    await client.connect("mock://local", 1337)
    assert client.connected
    assert client.disconnect.called

@pytest.mark.asyncio
@with_client(connected=False)
async def test_client_connect_invalid_params(server, client):
    with raises(ValueError):
        await client.connect()
    with raises(ValueError):
        await client.connect(port=1337)

@pytest.mark.asyncio
@mark.slow
@with_client()
async def test_client_timeout(server, client):
    client.on_data_error = Mock()
    time.sleep(pydle.client.BasicClient.READ_TIMEOUT + 1)

    assert client.on_data_error.called
    assert isinstance(client.on_data_error.call_args[0][0], TimeoutError)

@pytest.mark.asyncio
@with_client(connected=False)
async def test_client_server_tag(server, client):
    assert client.server_tag is None

    await client.connect("Mock.local", 1337)
    assert client.server_tag == "mock"
    await client.disconnect()

    await client.connect("irc.mock.local", 1337)
    assert client.server_tag == "mock"
    await client.disconnect()

    await client.connect("mock", 1337)
    assert client.server_tag == "mock"
    await client.disconnect()

    await client.connect("127.0.0.1", 1337)
    assert client.server_tag == "127.0.0.1"

    client.network = "MockNet"
    assert client.server_tag == "mocknet"
    await client.disconnect()

## Messages.

@pytest.mark.asyncio
@with_client()
async def test_client_message(server, client):
    client.on_raw_install = Mock()
    await server.send("INSTALL", "gentoo")
    assert client.on_raw_install.called

    message = client.on_raw_install.call_args[0][0]
    assert isinstance(message, pydle.protocol.Message)
    assert message.command == "INSTALL"
    assert message.params == ("gentoo",)

@pytest.mark.asyncio
@with_client()
async def test_client_unknown(server, client):
    client.on_unknown = Mock()
    await server.send("INSTALL", "gentoo")
    assert client.on_unknown.called


'''
Created on Jul 11, 2018

@author: root
'''

from PyQt5 import QtWidgets
from pydm import PyDMApplication

class EmaApp (object):
    def __init__(self):
        ''' Ema App'''
        from interface_i import Ui_MainWindow
        self.window_MW = QtWidgets.QMainWindow()
        self.ui_MW = Ui_MainWindow()
        self.ui_MW.setupUi(self.window_MW)
        self.window_MW.show()
        '''
        from interface_i import Ui_MainWindow
        self.window_MW = QtWidgets.QMainWindow()
        self.ui_MW = Ui_MainWindow()
        self.ui_MW.setupUi(self.window_MW)
        self.window_MW.show()

        self.__setFlowControl()
        '''
        from OceanOptics import OceanOptics
        self.Ui_MainWindow_Ocean = QtWidgets.QMainWindow()
        self.ui_ocean = OceanOptics()
        self.ui_ocean.setupUi(self.Ui_MainWindow_Ocean)
        self.ui_ocean.setFlowControl()
        #self.Ui_MainWindow_Ocean.show()

        self.__setFlowControl()

    def  __setFlowControl(self):

        self.ui_MW.PyDMPushButton_pressuresystem.clicked.connect(self.__openOceanOptics)

    def __openOceanOptics(self):
        print('__openOceanOptics')
        if(self.ui_MW.PyDMCheckbox.isChecked()):
            self.Ui_MainWindow_Ocean.show()
            print('__openOceanOptics')
            '''
            from OceanOptics import OceanOptics
            self.Ui_MainWindow_Ocean = QtWidgets.QMainWindow()
            self.ui_ocean = OceanOptics()
            self.ui_ocean.setupUi(self.Ui_MainWindow_Ocean)
            self.ui_ocean.setFlowControl()
            self.Ui_MainWindow_Ocean.show()
            '''
        else:
            print('Box OceanOptics: Not Checked')

if __name__ == '__main__':
    pass
    import sys
    from OceanOptics import OceanOptics
    app = PyDMApplication()
    '''

    Ui_MainWindow_Ocean = QtWidgets.QMainWindow()
    ui_ocean = OceanOptics()
    ui_ocean.setupUi(Ui_MainWindow_Ocean)
    ui_ocean.setFlowControl()
    Ui_MainWindow_Ocean.show()
    '''
    emapp = EmaApp()
    sys.exit(app.exec_())
    '''
    app = PyDMApplication()
    emapp = EmaApp()
    sys.exit(app.exec_())
    QtWidgets.QApplication.closeAllWindows()

    '''


import os
import sys

sys.path.append(os.getcwd())

from Controller.Common import common_log
from Controller.ControllerManager import Manager

_DEBUG = True

# ------------------------------------------------------------------------------------------
def _log(prefix, msg):
    common_log(_DEBUG, '', 'Multi Nox start', prefix, msg)

def emulators_test(vm):
    manager = Manager()
    manager._DEBUG = True
    manager._mdb._DEBUG = True
    # t = {
    #     'taskId': 2,
    #     'app_name': 'miaopai',
    #     'docker_name': 'nox-2',
    #     'timer_no': 2,
    #     'script': 'script_miaopai.py'
    # }
    # manager.run_script(task_info=t)
    # manager.start_tasks()
    # manager.vm_draw_cardiogram()
    # manager.vm_reset(vm_name='vm4')
    # host_ip = '172.16.253.37'
    return

def emulators_main():
    manager = Manager()
    manager._DEBUG = True
    manager.nox_run_tasks()

# ------------------------------------------------------------------------------------------
if __name__ == '__main__':
    # mnox_test()
    emulators_main()


# coding: utf-8
# @author: hongxin
# @date: 9/8/18

class TextProcessor(object):
    Plugins = {}

    def process(self, text, plugins=()):
        if plugins is ():
            for plugin_name in self.Plugins.keys():
                text = self.Plugins[plugin_name]().process(text)
                return text
        else:
            for plugin_name in plugins:
                # self.Plugins[plugin_name] 为函数名, 加上括号代表函数实例
                text = self.Plugins[plugin_name]().process(text)
                return text

    @classmethod
    def plugin_register(cls, plugin_name):
        def wrapper(plugin):
            """
            传入参数作为插件名, 函数及wrapper的参数作为插件值
            :param plugin:
            :return:
            """
            cls.Plugins.update({plugin_name: plugin})
            return plugin
        return wrapper

@TextProcessor.plugin_register('plugin1')
class CleanMarkdownBolds(object):
    def process(self, text):
        return text.replace('**', '')

processor = TextProcessor()
print(processor.Plugins)
processed = processor.process(text="**foo bar**", plugins=('plugin1',))
print(processed)
# processor


#!/usr/bin/env python3

import sys

def is_letter(a):
    return 'a'<=a<='z' or 'A'<=a<='Z' or '0'<=a<='9' or a=='_'

def parse_newick(L,T):
    S = []
    curr_name = []
    curr_children = []
    for i in range(len(L)):
        if L[i]=='(':
            assert(not curr_name)
            S.append('(')
        elif L[i] in '),;':
            if not curr_name:
                name = 'node%d'%(len(T))
            else:
                name = ''.join(curr_name)
            T[name] = curr_children
            S.append(name)
            curr_children = []
            if L[i]==')':
                while S[-1]!='(':
                    curr_children.append(S[-1])
                    S.pop()
                S.pop()
            elif L[i]==';':
                assert(len(S)==1)
                return S[0]
            curr_name = []
        else:
            assert(is_letter(L[i]))
            curr_name.append(L[i])

def unroot(T,u,u0=None):
    for v in T[u]:
        assert(v!=u0)
        unroot(T,v,u)
    if u0!=None:
        T[u].append(u0)

def tree_dist_dfs(T,Dist,u,u0=None):
    for v in T[u]:
        if v!=u0:
            Dist[v] = Dist[u]+1
            tree_dist_dfs(T,Dist,v,u)

def main():
    L = sys.stdin.readlines()
    res = []
    for i in range(0,len(L),3):
        NW = L[i].strip  ()
        T = {}
        root = parse_newick(NW,T)
        unroot(T,root)
        u,v = L[i+1].split()
        D = {u:0}
        tree_dist_dfs(T,D,u)
        res.append(D[v])
    print(' '.join(map(str,res)))

main()


# <copyright>
# Solipsis, a peer-to-peer serverless virtual world.
# Copyright (C) 2002-2005 France Telecom R&D
#
# This software is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this software; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
# </copyright>
"""Contains preferences for profile"""

__revision__ = "$Id$"

import os.path

import ConfigParser

DOWNLOAD_REPO = os.sep.join([os.path.expanduser("~"), ".solipsis", "download"])
PROFILE_DIR = os.sep.join([os.path.expanduser("~"), ".solipsis", "profiles"])
ENCODING = "iso-8859-1"

PREFS_FILE = os.path.join(PROFILE_DIR, ".preferences")
MAIN_SECTION = "General"
DIALOG_SECTION = "Dialogs"

def get_prefs(option):
    """load config from file PREFS_FILE creating it if necessary"""
    if not Preferences.conf:
        Preferences.conf = Preferences(PREFS_FILE)
        Preferences.conf.load()
    return Preferences.conf.get(option)

def set_prefs(option, value=None):
    """load config from file PREFS_FILE creating it if necessary"""
    if not Preferences.conf:
        Preferences.conf = Preferences(PREFS_FILE)
        Preferences.conf.load()
    Preferences.conf.set(option, value)

class Param:
    """simple structure to define a parameter"""

    def __init__(self, name, section, default, var_type, convertor):
        self.name = name
        self.section = section
        self.default = default
        self.var_type = var_type
        self.convertor = convertor

    def __str__(self):
        return "%s: %s"% (self.name, self.default)

    def __repr__(self):
        return self.name

class BoolParam(Param):
    """simple structure to define a boolean parameter"""

    def __init__(self, name, section, default):
        convertor = lambda value_str: not value_str in ["False", "0", "no"]
        Param.__init__(self, name, section, default, bool, convertor)

class IntParam(Param):
    """simple structure to define a boolean parameter"""

    def __init__(self, name, section, default):
        Param.__init__(self, name, section, default, int, int)

class StringParam(Param):
    """simple structure to define a boolean parameter"""

    def __init__(self, name, section, default):
        Param.__init__(self, name, section, default, str, str)

class Preferences:
    """manage all options providing setters and getters for every params"""

    conf = None
    # {param_name : [section,
    params = {"disclaimer": BoolParam("Disclaimer", MAIN_SECTION, True),
              "display_dl": BoolParam("Download Dialog", MAIN_SECTION, True),
              "download_repo": StringParam("Download directory",
                                           MAIN_SECTION, DOWNLOAD_REPO),
              "profile_dir": StringParam("Profile directory",
                                           MAIN_SECTION, PROFILE_DIR),
              "profile_width": IntParam("Profile width", DIALOG_SECTION, 460),
              "profile_height": IntParam("Profile height", DIALOG_SECTION, 600),
              "match_width": IntParam("Match width", DIALOG_SECTION, 460),
              "match_height": IntParam("Match height", DIALOG_SECTION, 600),
              "filter_width": IntParam("Filter width", DIALOG_SECTION, 460),
              "filter_height": IntParam("Filter height", DIALOG_SECTION, 600),
              "simple_mode": BoolParam("Simple mode", DIALOG_SECTION, True),
              "encoding": StringParam("Encoding", MAIN_SECTION, ENCODING),
              "log": BoolParam("Enable logging", MAIN_SECTION, False),
              }

    def __init__(self, file_name):
        self.file_name = file_name
        self.config = None

    def __str__(self):
        if not self.config:
            return "not initialized"
        else:
            return ", ".join(self.params.keys())

    def load(self):
        """load params from file"""
        self.config = ConfigParser.ConfigParser()
        if not os.path.exists(self.file_name):
            for option in Preferences.params:
                self.set(option)
        else:
            pref_file = open(self.file_name)
            self.config.readfp(pref_file)
            pref_file.close()

    def _save(self):
        """save params to file"""
        pref_file = open(self.file_name, 'w')
        self.config.write(pref_file)
        pref_file.close()

    def set(self, option, value=None):
        """generic setter. Will save options each time called. Wrapped
        for every params under appropriate name"""
        assert self.config, "Preferences not loaded"
        assert option in Preferences.params, "no option %s. Available: %s"\
               % (option, Preferences.params)
        param = Preferences.params[option]
        # set value
        if value is None:
            value = param.default
        assert isinstance(value, param.var_type)
        if not self.config.has_section(param.section):
            self.config.add_section(param.section)
        self.config.set(param.section, param.name, str(value))
        self._save()

    def get(self, option):
        """generic getter. Wrapped for every params under appropriate name"""
        assert self.config, "Preferences not loaded"
        assert option in Preferences.params, "no option %s. Available: %s"\
               % (option, Preferences.params)
        param = Preferences.params[option]
        try:
            return param.convertor(self.config.get(param.section, param.name))
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
            self.set(option)
            return param.default


import os
import yaml

def parse_config(filepath):
    if not os.path.exists(filepath):
        raise FileNotFoundError(f'Config file {filepath} does not exist.')

    # Not wrapping in a try/except so that we get the
    # yaml library to raise any issues on parsing
    with open(filepath, 'r') as f:
        config = yaml.safe_load(f)

    spatial = config.get('spatial', None)
    if spatial == None:
        raise KeyError('Config is missing the root "spatial" key.')
    for k in ['latitudinal_srid', 'dem']:
        if k not in spatial.keys():
            raise KeyError(f'Missing key: {k} in the spatial section of the config.')

    database = config.get('database', None)
    if database == None:
        raise KeyError('Config is missing the root "database" key.')

    redis = config.get('redis', None)
    if redis == None:
        raise KeyError('Config is missing the root "redis" key.')

    for k in ['type', 'username', 'password', 'host', 'pgbouncer_port', 'name']:
        if k not in database.keys():
            raise KeyError(f'Missing key: "{k}" in the database section of the config.')

    return config


import csv
import urllib.request
import time
import multiprocessing
import concurrent.futures

with open('/opt/splunk/etc/apps/Kaspersky_Threat_Feed_App_for_Splunk/lookups/Botnet_CnC_URL_Exact_Data_Feed_DOMAIN.csv') as csvfile:
    readCSV = csv.reader(csvfile, delimiter=',')
    ipList = ["apple", "banana"]
    for row in readCSV:
        print(row[0])
        try:
            url = "http://" + row[0]
            print(url)
            urlA = row[0]
            ipList.append(url)
            #print(ipList)
            #wp = urllib.request.urlopen(url)
            #print(wp)
        except:
            print("Something else went wrong!")
    print(ipList)
    listLength = len(ipList)
    print(listLength)
    print(ipList[100])
    print(round(listLength/10))

#    for y in range(0, 11):
#        for x in range(0*y, 136*y):
#            print(x)
#            print(ipList[x])

    start = time.perf_counter()

    def do_something(num1):
        length = round(listLength/10)
        for x in range(num1*length, length*(num1+1)):
            try:
                print(ipList[x])
                time.sleep(0.1)
                print(x)
                url = ipList[x]
                wp = urllib.request.urlopen(url,timeout=5)
                print(wp)
            except:
                print("Somthing is not correct.")
    numList=[0,1,2,3,4,5,6,7,8,9]

    with concurrent.futures.ProcessPoolExecutor() as executor:
        executor.map(do_something, numList)

    finish = time.perf_counter()

    print(f'Finished in {round(finish-start, 2)} second(s)')


var_names = [\
    #--- Leloup et al.,Toward a detailed computational model for the mammalian circadian clock,PNAS(2003) ---#
    'MP',
    'MC',
    'MB',
    'PC',
    'CC',
    'PCP',
    'CCP',
    'PCC',
    'PCN',
    'PCCP',
    'PCNP',
    'BC',
    'BCP',
    'BN',
    'BNP',
    'IN',
    #
    'len_f_vars'\
]

for idx,name in enumerate(var_names):
  exec('%s=%d'%(name,idx))


import unittest
from unittest import mock

from unittest.mock import patch, MagicMock

from dataprep.bpepkg import merge
from dataprep.bpepkg.merge import MergeList, Merge

@patch('dataprep.bpepkg.merge.open')
class ReadMergesTest(unittest.TestCase):
    def test_simple(self, open_mock):
        open_mock.return_value = MagicMock(spec=['__enter__', '__exit__'])
        handle = open_mock.return_value.__enter__.return_value
        handle.__iter__.return_value = iter(['a b 67', 'b c 34', 'c d 94'])

        actual = merge.read_merges('file', 2)
        expected = MergeList().append(Merge(('a', 'b'), 67, 0)).append(Merge(('b', 'c'), 34, 1))

        self.assertEqual(expected, actual)

    def test_wrong_delimiter(self, open_mock):
        with self.assertRaises(ValueError):
            open_mock.return_value = MagicMock(spec=['__enter__', '__exit__'])
            handle = open_mock.return_value.__enter__.return_value
            handle.__iter__.return_value = iter(['a\tb\t67'])

            merge.read_merges('file')

@patch('dataprep.bpepkg.merge.open')
class WriteMergesTest(unittest.TestCase):
    def test_simple(self, open_mock):
        open_mock.return_value = MagicMock(spec=['__enter__', '__exit__'])
        handle = open_mock.return_value.__enter__.return_value

        merges = MergeList().append(Merge(('a', 'b'), 67, 0)).append(Merge(('b', 'c'), 34, 1))
        merge.dump_merges(merges, 'file')

        handle.write.assert_has_calls([
            mock.call('a b 67\n'),
            mock.call('b c 34\n')
        ])

class MergeTest(unittest.TestCase):
    def test_merge_as_key(self):
        merge1 = Merge(('a', 'b'), 34, 0)
        merge2 = Merge(('a', 'b'), 34, 0)
        map = {merge1: 3}

        self.assertEqual(3, map[merge2])

class MergeListTest(unittest.TestCase):
    def test_append_wrong_priority(self):
        with self.assertRaises(ValueError):
            merges = MergeList()
            merges.append(Merge(('a', 'b'), 34, 3))

    def test_get_priority(self):
        merges = MergeList()
        merges.append(Merge(('a', 'b'), 34, 0)).append(Merge(('b', 'c'), 44))
        actual = merges.get_priority(('b', 'c'))

        self.assertEqual(1, actual)

class MergeListInclusionTest(unittest.TestCase):
    def test_present(self):
        merges = MergeList()
        merges.append(Merge(('a', 'b'), 34, 0)).append(Merge(('b', 'c'), 44, 1))
        self.assertTrue(('a', 'b') in merges)

    def test_absent(self):
        merges = MergeList()
        merges.append(Merge(('a', 'b'), 34, 0)).append(Merge(('b', 'c'), 44, 1))
        self.assertFalse(('g', 'r') in merges)

class MergeListIndexTest(unittest.TestCase):
    def test_simple(self):
        merges = MergeList()
        merges.append(Merge(('a', 'b'), 34, 0)).append(Merge(('b', 'c'), 44))

        actual = merges[1]
        expected = (Merge(('b', 'c'), 44, 1))

        self.assertEqual(expected, actual)

    def test_out_of_range(self):
        with self.assertRaises(IndexError):
            merges = MergeList()
            merges.append(Merge(('a', 'b'), 34, 0)).append(Merge(('b', 'c'), 44, 1))

            actual = merges[2]

    def test_negative(self):
        merges = MergeList()
        merges.append(Merge(('a', 'b'), 34, 0)).append(Merge(('b', 'c'), 44))

        actual = merges[-1]
        expected = (Merge(('b', 'c'), 44, 1))

        self.assertEqual(expected, actual)

    def test_range_with_negative(self):
        merges = MergeList()
        merges.append(Merge(('a', 'b'), 34)).append(Merge(('b', 'c'), 44))

        actual = merges[0:-1]
        expected = [Merge(('a', 'b'), 34, 0)]

        self.assertEqual(expected, actual)

class MergeListPlusOperationTest(unittest.TestCase):
    def test_simple(self):
        merges1 = MergeList()
        merges1.append(Merge(('a', 'b'), 34, 0)).append(Merge(('b', 'c'), 44, 1))

        merges2 = MergeList()
        merges2.append(Merge(('c', 'd'), 64, 0)).append(Merge(('d', 'e'), 84, 1))

        expected = MergeList()
        expected.append(Merge(('a', 'b'), 34, 0)).append(Merge(('b', 'c'), 44, 1))\
            .append(Merge(('c', 'd'), 64, 2)).append(Merge(('d', 'e'), 84, 3))

        actual = merges1 + merges2

        self.assertEqual(expected, actual)

    def test_add_wrong_type(self):
        with self.assertRaises(TypeError):
            merges = MergeList()
            merges + [(('d', 'e'), 84, 1)]

class MergeListLenTest(unittest.TestCase):
    def test_simple(self):
        merges = MergeList()
        merges.append(Merge(('a', 'b'), 34, 0)).append(Merge(('b', 'c'), 44, 1))

        actual = len(merges)

        self.assertEqual(2, actual)

class MergeListIteratorTest(unittest.TestCase):
    def test_simple(self):
        merges = MergeList()
        merges.append(Merge(('a', 'b'), 34, 0)).append(Merge(('b', 'c'), 44, 1))

        for idx, merge in enumerate(merges):
            self.assertEqual(merges[idx], merge)

if __name__ == '__main__':
    unittest.main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import datetime
from django.utils.timezone import utc

class Migration(migrations.Migration):

    dependencies = [
        ('members', '0001_initial'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='areaofstudy',
            options={'verbose_name_plural': 'Program', 'verbose_name': 'Program'},
        ),
        migrations.AlterModelOptions(
            name='member',
            options={'verbose_name_plural': 'Medlemmar', 'verbose_name': 'Medlem'},
        ),
        migrations.AddField(
            model_name='member',
            name='member_since',
            field=models.DateTimeField(default=datetime.datetime(2016, 11, 22, 0, 26, 49, 355201, tzinfo=utc), auto_now=True),
            preserve_default=False,
        ),
        migrations.AlterField(
            model_name='member',
            name='email',
            field=models.EmailField(unique=True, max_length=255, verbose_name='E-postadress'),
        ),
    ]


import logging

from aiohttp import web
from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup
from servicelib.aiohttp.rest_routing import (
    iter_path_operations,
    map_handlers_with_operations,
)

from ..constants import APP_CONFIG_KEY, APP_OPENAPI_SPECS_KEY
from ..login.decorators import login_required
from .handlers_redirects import get_redirection_to_viewer
from .handlers_rest import rest_handler_functions

logger = logging.getLogger(__name__)

@app_module_setup(
    "simcore_service_webserver.studies_dispatcher", ModuleCategory.ADDON, logger=logger
)
def setup_studies_dispatcher(app: web.Application) -> bool:
    cfg = app[APP_CONFIG_KEY]["main"]

    # Redirects routes
    redirect_handler = get_redirection_to_viewer
    if not cfg["studies_access_enabled"]:
        redirect_handler = login_required(get_redirection_to_viewer)
        logger.warning(
            "'%s' config explicitly disables anonymous users from this feature",
            __name__,
        )
    app.router.add_routes(
        [web.get("/view", redirect_handler, name="get_redirection_to_viewer")]
    )

    # Rest-API routes: maps handlers with routes tags with "viewer" based on OAS operation_id
    specs = app[APP_OPENAPI_SPECS_KEY]
    rest_routes = map_handlers_with_operations(
        rest_handler_functions,
        filter(lambda op: "viewer" in op.tags, iter_path_operations(specs)),
        strict=True,
    )
    app.router.add_routes(rest_routes)

    return True


# -*- coding: utf-8 -*-

import requests
import pandas as pd
import numpy as np
import json

class StockModel():
    def __init__(self, date, price):
        self.date = date
        self.price = price

class StockData:

    def __init__(self, trade_code):
        self.code = trade_code

    def fetch(self, months_count, date_index = False):
        url = 'https://www.dsebd.org/php_graph/monthly_graph.php?inst=' + self.code + '&duration=' + str(months_count) + '&type=price'
        r = requests.get(url)
        r.text
        data = r.text
        #companyCode = data.split("Closing Price Graph of ",1)[1].split('\'')[0]
        d = data.split('\\n')
        del d[0]
        del d[-1]
        for i in range(0, len(d)):
            if(i == 0):
                d[i] = d[i].split('" +\r\n    "')[1]
            else:
                d[i] = d[i].split('"+"')[1]

        stock2dList = []
        for i in range(len(d)):
            arr = d[i].split(",")
            arr[1] = float(arr[1])
            stock2dList.append(arr)

        df = pd.DataFrame(stock2dList)
        df.columns =['Date','Price']

        print("---------------------------------")
        print("Trading Code: " + self.code)
        print("---------------------------------")
        print("Days: " + str(df.shape[0]))
        print("---------------------------------")
        if date_index:
            df = df.set_index(pd.to_datetime(df['Date']))
            del df['Date']
        return df

    def format_df_json(self, df, format_json = True):
        arr = np.array(df)
        stock_list = []
        for i in range(len(arr)):
            sm = StockModel(arr[i][0], arr[i][1])
            stock_list.append(sm.__dict__)
        if format_json:
            return json.dumps(stock_list)
        else:
            return stock_list


import chainer
import nnoir
import numpy as np
import util
from nnoir_chainer import NNOIRFunction

def test_pad():
    inputs = [nnoir.Value(b"v0", np.zeros((2, 3, 4, 5)).astype("float32"))]
    outputs = [nnoir.Value(b"v2", np.zeros((3, 5, 7, 10)).astype("float32"))]
    pads = ((1, 0), (1, 1), (1, 2), (0, 5))
    nodes = inputs + outputs
    input_names = [x.name for x in inputs]
    output_names = [x.name for x in outputs]
    function = nnoir.functions.ConstantPadding(input_names, output_names, pads=pads, value=1.0)
    result = nnoir.NNOIR(
        b"ConstantPadding",
        b"nnoir2chainer_test",
        "0.1",
        input_names,
        output_names,
        nodes,
        [function],
    )
    result.dump("pad.nnoir")

    x = np.random.randn(2, 3, 4, 5).astype("float32")
    ref = function.run(x)
    with chainer.using_config("train", False):
        m = NNOIRFunction("pad.nnoir")
        y = m(x)
        assert np.all(abs(y - ref).data < util.epsilon)


import json
import pandas as pd

def export_json(result):
    """ Serializing json """
    with open('src/sample/output.json', 'w') as f:
        # json_object = json.dumps(result, ensure_ascii=False, indent=4)
        json.dump(result, f, ensure_ascii=False, indent=4)

def export_exel(result):
    """ Export exel with new format """
    excel_result = []
    for r in result:
        for comment in r['Comments']:
            r_dict = {
                'ProductPageLink': r['ProductPageLink'],
                'ProductName': r['ProductName'],
                'Productcode': r['Productcode'],
                'BrandNameFa': r['BrandNameFa'],
                'BrandNameEn': r['BrandNameEn'],
                'Group': r['Group'],
                'CommentOwnerId': comment['CommentOwnerId'],
                'CommentDate': comment['CommentDate'],
                'CommentDescription': comment['CommentDescription'],
            }
            excel_result.append(r_dict)
    df = pd.DataFrame.from_dict(excel_result)
    print(df)
    df.to_excel('src/sample/sample.xlsx')


# coding=gb2312

import re,string
import time

LINE_ANY_STR = "__LINE_ANY_STR"
ANY_NUM_LINE = "__ANY_NUM_LINE-"

RULE_KEY_INCLUDE = "include"
RULE_KEY_INCLUDEOR = "include-or"
RULE_KEY_EQUAL = "equal"
RULE_KEY_PATTERN = "pattern"

ANY_NUM_LINE_FLAG = 1

class SmartScript():
	rules = []
	result = []
	def __init__(self):
		self.rules = []
		self.result = []
		self.matched_line_format = 1 #0表示以文本形式返回，1表示以列表形式返回
		self.obj = None
		self.do_equal = 0
		self.do_pattern = 0
		return

	#添加规则行
	def add_rule_line(self, rule):
		self.rules.append(rule.strip())
		return

	def clear_rule(self):
		del self.rules[0:len(self.rules)]
		return

	#检查参数s是否是一个数字
	def is_integer(self, s):
		c = r'\d+'
		d = re.match(c, s)
		if None == d:
			return False, s
		c = r'\D+'
		if None != re.search(c, s):
			return False, s
		return True, string.atoi(s)

	#正则表达式匹配
	def pattern_line_match(self, rule, line, values):
		if None == rule or None == line:
			return False

		rule_pattern = re.compile(rule)
		if None == rule_pattern:
			return False
		matched = re.match(rule_pattern, line)
		if None == matched:
			return False
		if None == values:#如果values为空，表示只比较不取数据
			return True
		keys = []
		self.get_key(keys, rule)
		for k in keys:
			a1,a2 = self.is_integer(matched.group(k))
			values[k] = a2
		return True

	#比较一行是否匹配，支持equal和pattern
	def is_match(self, line, rule, type, values):
		if LINE_ANY_STR == rule:
			return True
		if RULE_KEY_EQUAL == type and line == rule:
			self.do_equal = self.do_equal + 1
			return True
		if RULE_KEY_PATTERN == type and self.pattern_line_match(rule, line, values):
			self.do_pattern = self.do_pattern + 1
			return True
		return False

	#从规则中取出关键字
	def get_key(self, value_list, pattern_str):
		f1 = ''
		f2 = 0
		f3 = 0
		for i in range(0, len(pattern_str)):
			if pattern_str[i] == '(':
				f1 = '('
				f2 = i
				continue
			if f2 == i - 1:
				if ('(' == f1) and ('?' == pattern_str[i]):
					f1 = '?'
					f2 = i
				elif ('?' == f1) and ('P' == pattern_str[i]):
					f1 = 'P'
					f2 = i
				elif ('P' == f1) and ('<' == pattern_str[i]):
					f3 = i + 1
			elif f3 != 0:
				if '>' == pattern_str[i]:
					value_list.append(pattern_str[f3:i])
					f3 = 0
			else:
				f3 = 0
				f1 = ''
				f2 = 0
		return

	#多行匹配
	def do_match(self, contents, match_type):
		rule_tmp = self.rules
		lines_tmp = []
		match_first_rule = False
		rule_key_tmp = []

		#对规则进行预处理
		for r in rule_tmp:
			if -1 !=  r.find(ANY_NUM_LINE):
				t = r.split("-")
				if len(t) >= 2:
					t2 = [ANY_NUM_LINE_FLAG, string.atoi(t[1])]
					rule_key_tmp.append(t2)
			else:
				rule_key_tmp.append([0,0])

		for line in contents:
			line = line.strip()

			#进行首规则匹配，以增加匹配速度
			if False == match_first_rule:
				if False == self.is_match(line, rule_tmp[0], match_type, None):
					continue
				else:
					match_first_rule = True

			lines_tmp.append(line)
			any_num_line = 0
			next_match_rule_line = 0
			if len(lines_tmp) >= len(rule_tmp):
				c = 0
				j = 0
				matched = False

				arg_item = {"matched_line":[], "values":{}}

				#开始匹配
				for i in range(0,len(lines_tmp)):
					#先做ANY_NUM_LINE匹配，如果rule为ANY_NUM_LINE，则获取最大跳过的行数，
					#直接匹配下一个rule行，如果匹配成功则跳到下下个rule行比较
					if 0 == any_num_line and ANY_NUM_LINE_FLAG == rule_key_tmp[j][0]:
						#t = rule_tmp[j].split("-")
						#if len(t) >= 2:
						any_num_line = rule_key_tmp[j][1] + 2  #获取最大跳过行数
						next_match_rule_line = j + 1  #下一个rule行的下标
						if next_match_rule_line >= len(rule_tmp):
							next_match_rule_line = 0
					if any_num_line > 0: #控制跳过行数
						any_num_line = any_num_line - 1

					if 0 != any_num_line and 0 != next_match_rule_line: #条件满足的情况下进行下一个rule行匹配
						if True == self.is_match(lines_tmp[i], rule_tmp[next_match_rule_line], match_type, arg_item["values"]):
							j = j + 2
							next_match_rule_line = 0
							any_num_line = 0
							if j == len(rule_tmp): #命中最后一个rule行，则认为完全匹配成功
								matched = True
								break
							#下一个rule行匹配匹配成功，则清空ANY_NUM_LINE
						continue

					#非ANY_NUM_LINE比较，这个是逐行比较，必须每行都一致
					if False == self.is_match(lines_tmp[i], rule_tmp[j], match_type, arg_item["values"]):
						lines_tmp.pop(0)
						c = -1
						break
					else:
						if j == len(rule_tmp) - 1:
							matched = True
							break
						j = j + 1 #匹配成功一个，rule就向后移动一个
					c = c + 1

				if matched: #匹配成功
					m = 0
					for tmp in lines_tmp:
						arg_item["matched_line"].append(tmp)
						if None != self.obj:
							if 0 == m:
								self.obj.info = tmp
							else:
								self.obj.info = self.obj.info + "\n" + tmp
							m = m + 1
					self.result.append(arg_item)
					del lines_tmp[0:len(lines_tmp)] #清空lines_tmp

		return

	def pattern_match(self, lines):
		self.do_match(lines,RULE_KEY_PATTERN)
		return

	def equal_match(self, lines):
		self.do_match(lines, RULE_KEY_EQUAL)
		return

	def match_key_line(self, lines, key):
		matched_lines = []
		for line in lines:
			if -1 != line.find(key):
				matched_lines.append(line)
		return matched_lines

	def cut_range_lines(self, lines, flag):
		step = 0
		c = 0
		info = ""
		start = 0
		end = 0
		arg_item = {"matched_line":[], "values":{}}

		#先预查找一次，以避免存取速度太慢或太费内存
		for line in lines:
			line = line.strip()
			if 0 == step and self.rules[0] == line:
				step = 1
				start = c
			elif 1 == step and self.rules[1] == line:
				step = -1
				end = c + 1
				break
			c = c + 1

		if -1 != step:
			return False

		#找到后，直接获取
		self.info = ""
		for i in range(start, end):
			arg_item["matched_line"].append(lines[i])
			if None != self.obj:
				if start == i:
					self.obj.info = lines[i].strip()
				else:
					self.obj.info = self.obj.info + "\n" + lines[i].strip()
		self.result.append(arg_item)
		return True


#! python3
#  coding=utf-8
from urllib.parse import urljoin

import scrapy

import Columbus.utils.datetime_util as datetime_util
from Columbus.items import OfficialListItem
from Columbus.repository.weixin_dao.keyword_monitor_dao import KeywordMonitorDao

class AccountCrawlSpider(scrapy.Spider):
    name = "account_crawl"
    allowed_domains = ['sogou.com']

    keyword_monitor = KeywordMonitorDao()
    keyword_monitors = keyword_monitor.query_by_crawl_date(datetime_util.curDate())

    keywords = []
    for keyword_monitor in keyword_monitors:
        keywords.append(keyword_monitor[1])

    keyword_count = 0
    keyword = keywords[keyword_count]

    page_count = 1
    base_url = ""
    if page_count == 1:
        base_url = "http://weixin.sogou.com/weixin?type=1&s_from=input&query=" + keyword + "&ie=utf8&_sug_=n&_sug_type_="
    else:
        base_url = "http://weixin.sogou.com/weixin?query=" + keyword + "&_sug_type_=&s_from=input&_sug_=n&type=1&page=" \
                   + str(page_count) + "&ie=utf8"

    start_urls = [base_url]

    def parse(self, response):
        node_list = response.xpath("//div[@class='news-box']/ul/li")
        print("response node length:" + str(len(node_list)))

        # 解析当前页面获取的所有列表
        for node in node_list:
            item = OfficialListItem()
            name = node.xpath("./div/div[2]/p[1]/a//text()").extract()
            code = node.xpath("./div/div[2]/p[2]/label[@name='em_weixinhao']/text()").extract()[0]
            icon = node.xpath("./div/div[1]/a/img/@src").extract()[0]
            description = node.xpath("./dl[1]/dd//text()").extract()
            href = node.xpath("./div/div[2]/p[1]/a/@href").extract()[0]

            item['name'] = "".join(name)
            item['code'] = code
            item['icon'] = icon
            item['description'] = "".join(description)
            item['href'] = href

            yield item

        self.page_count = self.page_count + 1

        # 查看是否有下一页的按钮
        if self.page_count < 10 and len(response.xpath("//div[@class='news-box']/div/a[@id='sogou_next']")):
            next_url = urljoin(self.base_url,
                               response.xpath("//div[@class='news-box']/div/a[@id='sogou_next']/@href").extract()[0])

            yield scrapy.Request(next_url, callback = self.parse)
        else:
            if len(self.keywords) - 1 > self.keyword_count:
                self.keyword_count += 1
                self.keyword = self.keywords[self.keyword_count]

                self.page_count = 0
                self.base_url = "http://weixin.sogou.com/weixin?query=" \
                                + self.keyword + "&_sug_type_=&s_from=input&_sug_=n&type=1&page=" \
                                + str(self.page_count) + "&ie=utf8"

                yield scrapy.Request(self.base_url, callback = self.parse)


from __future__ import annotations
from dataclasses import dataclass
import logging
from typing import List

from typedflow.nodes import DumpNode

__all__ = ['Flow']

logger = logging.getLogger(__file__)

@dataclass
class Flow:
    dump_nodes: List[DumpNode]

    def validate(self) -> None:
        """
        not implemented because Python Generic doesn't offer
        any ways to access to the actual type
        """
        pass

    async def run(self,
                  validate: bool = True) -> None:
        if validate:
            self.validate()
        batch_id: int = 0
        while True:
            for node in self.dump_nodes:
                await node.run_and_dump(batch_id=batch_id)
            if all([node.finished for node in self.dump_nodes]):
                return


#
# Skeleton file for the Python "Bob" exercise.
#
def hey(what):
    # First remove all spaces in the string, as they are not necessary
    what_string = what.strip()
    return_string = ""
    # letter_list = ("a","b","c","d","e","f","g","h","i","j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z")
    # # before we begin, we need to determine if the string contains letters
    # # If there are no letters, then the test for yelling will always be true
    # no_letters = True
    what_length = len(what_string)
    # x = 0
    # while no_letters:
    #     if what_length == 0: break
    #     if what_string[x].lower() in letter_list:
    #         no_letters = False
    #     if x == what_length - 1:
    #         break
    #     x += 1

    # From here, we can begin testing the various strings
    # First, test if Bob is screamed at. This test is needed
    # becasue the yelled phrase could be in the form of a quesiton
    # If you're not asked a qustion, maybe you're yelled at
    # This test will be all capital letters
    if what_string.isupper():
        return_string = 'Whoa, chill out!'
    # Next, perhaps they didn't actually say anything
    elif what_string == "":
        return_string = 'Fine. Be that way!'
    # Next test for a question. As the strings are in English, any question
    # # will end with a qustion mark
    elif what_string[what_length - 1] == "?":
        return_string = 'Sure.'
    # Lastly, anything else should return 'Whatever'
    else:
        return_string = 'Whatever.'

    return return_string


import os
import smtplib
#import mandrill

from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email.mime.image import MIMEImage
from email.utils import COMMASPACE,formatdate
from email import encoders

def send_mail(fro, to, subject, text, files=[]):
    #assert type(server) == dict
    assert type(to) == list
    assert type(files) == list

    msg = MIMEMultipart('alternative')

    msg['Subject'] = subject
    msg['From'] = 'CAP_TEAM@thomsonreuters.com' # Your from name and email address
    msg['To'] = ','.join(to)
    #print msg['To']
    msg['Date'] = formatdate(localtime=True)
    msg.attach(MIMEText(text, 'html', 'utf-8'))

    for file in files:
        data = open(file, 'rb')
        part = MIMEBase('application', 'octet-stream') #'octet-stream': binary data
        part.set_payload(data.read())
        data.close()
        encoders.encode_base64(part)
        part.add_header('Content-Disposition', 'attachment', filename=os.path.basename(file))
        msg.attach(part)

    s = smtplib.SMTP('10.80.81.132', 25)
    #s.login(username, password)
    s.sendmail(fro, to, msg.as_string())
    s.quit()

    ''' mandrill web api way
    try:
        mandrill_client = mandrill.Mandrill('ElZncQlS9DXqotj0TMjuJA')
        #to = ['chao.xie@thomsonreuters.com', 'yang.yang@thomsonreuters.com']
        raw_message = msg.as_string()
        #print raw_message
        result = mandrill_client.messages.send_raw(raw_message, from_email=None, from_name='Chao Xie', to=None, async=False, ip_pool='Main Pool', send_at=None, return_path_domain=None)

    except mandrill.Error, e:
        # Mandrill errors are thrown as exceptions
        print 'A mandrill error occurred: %s - %s' % (e.__class__, e)
        # A mandrill error occurred: <class 'mandrill.UnknownSubaccountError'> - No subaccount exists with the id 'customer-123'
        raise
    '''

def send_mail_ex(to, subject, content, fro='CAP_TEAM@thomsonreuters.com', cc=[], bcc=[], images=[], attaches=[]):
    msgRoot = MIMEMultipart('related')
    msgRoot['Subject'] = subject
    msgRoot['From'] = fro # Your from name and email address
    msgRoot['To'] = ','.join(to)
    msgRoot['Cc'] = ','.join(cc)
    msgRoot['Bcc'] = ','.join(bcc)

    # Add image html into body
    ind = 1
    for img in images:
        content += '

'
        ind += 1

    # Body
    msgText = MIMEText(content, 'html', 'utf-8')
    msgRoot.attach(msgText)

    # Image in body
    ind = 1
    for img in images:
        fp = open(img, 'rb')
        msgImage = MIMEImage(fp.read())
        fp.close()
        msgImage.add_header('Content-ID', '<image'+str(ind)+'>')
        msgRoot.attach(msgImage)
        ind += 1

    # Attachment
    for f in attaches:
        data = open(f, 'rb')
        part = MIMEBase('application', 'octet-stream') #'octet-stream': binary data
        part.set_payload(data.read())
        data.close()
        encoders.encode_base64(part)
        part.add_header('Content-Disposition', 'attachment', filename=os.path.basename(f))
        msgRoot.attach(part)

    s = smtplib.SMTP('10.80.81.132', 25)
    s.sendmail(fro, [to, ','.join(cc), ','.join(bcc)], msgRoot.as_string())
    s.quit()

if __name__ == "__main__":
    # cc = ['jiu.chen@thomsonreuters.com', 'liang.zhang1@thomsonreuters.com']
    to = ['jiu.chen@thomsonreuters.com']
    subject = "Python Email with Pictures"
    text1 = 'HTML content
baidu
'
    att = ['tickets-per-region.png']
    images = ['total-tickets.png', 'tickets-per-system.png']
    # send_mail(fro, to, subject, text, files)
    send_mail_ex(to, subject, text1, images=images, attaches=att)


from __future__ import absolute_import, division, print_function, unicode_literals

import numpy as np
import pandas as pd

import tensorflow as tf

from tensorflow import feature_column
from tensorflow.keras import layers
from sklearn.model_selection import train_test_split

path_to_file = "./winequality-red.csv"
df = pd.read_csv(path_to_file, sep=",")
print(df.head(20))

train, test = train_test_split(df, test_size=0.2)
train, val   = train_test_split(train, test_size=0.2)

def df_to_dataset(df, shuffle=True, batch_size=32):
  df      = df.copy()
  labels  = df.pop('quality')
  ds      = tf.data.Dataset.from_tensor_slices((dict(df), labels))

  if shuffle:
    ds = ds.shuffle(buffer_size=len(df))

  ds = ds.batch(batch_size)
  return ds

batch_size  = 5
train_ds    = df_to_dataset(train, shuffle=True, batch_size=batch_size)
val_ds      = df_to_dataset(val, shuffle=True, batch_size=batch_size)
test_ds     = df_to_dataset(test, shuffle=True, batch_size=batch_size)

for feature_batch, label_batch in train_ds.take(1):
  print("dnesity feature", feature_batch['density'])

example_batch = next(iter(train_ds))[0]

def demo(feature_column):
  feature_layer = layers.DenseFeatures(feature_column)
  print(feature_layer(example_batch).numpy())

dioxide = feature_column.numeric_column('free_sulfur_dioxide')
demo(dioxide)

dioxide_buckets = feature_column.bucketized_column(dioxide, boundaries=[2, 6, 10, 14, 18, 24, 30, 50])

demo(dioxide_buckets)

#    fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  free sulfur dioxide  total sulfur dioxide  density    pH  sulphates  alcohol  quality
#0             7.4             0.700         0.00             1.9      0.076                 11.0                  34.0   0.9978  3.51       0.56      9.4        5

feature_columns = []
for header in ['fixed_acidity', 'volatile_acidity', 'residual_sugar', 'chlorides', 'free_sulfur_dioxide', 'total_sulfur_dioxide', 'density', 'pH', 'sulphates', 'alcohol']:
  d = feature_column.numeric_column(header)
  print("****Demo Begins****", header)
  demo(d)
  feature_columns.append(d)

#feature_columns.append(dioxide_buckets)

feature_layer = tf.keras.layers.DenseFeatures(feature_columns)

print(test.head())

batch_size = 32
train_ds   = df_to_dataset(train, batch_size=batch_size)
val_ds     = df_to_dataset(val, shuffle=False, batch_size=batch_size)
test_ds    = df_to_dataset(test, shuffle=False, batch_size=batch_size)
print(test_ds.take(1))

model = tf.keras.Sequential([
  feature_layer,
  tf.keras.layers.Dense(128, activation='relu'),
  tf.keras.layers.Dense(128, activation='relu'),
  tf.keras.layers.Dense(1)
])

model.compile(optimizer='adam',
              loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
              )

model.fit(train_ds,
          validation_data=val_ds,
          epochs=5)

loss, accuracy = model.evaluate(test_ds)
#print("Accuracy--->", accuracy)


# vi: set shiftwidth=4 tabstop=4 expandtab:
import datetime
import itertools

def get_numbers_from_file(file_path="day15_input.txt"):
    with open(file_path) as f:
        for l in f:
            return [int(s) for s in l.split(",")]

def yield_game(numbers):
    turns = dict()
    for turn, n in enumerate(numbers):
        turns.setdefault(n, []).append(turn)
        yield n
    for turn in itertools.count(start=turn + 1):
        lst = turns[n]
        n = (lst[-1] - lst[-2]) if len(lst) > 1 else 0
        turns.setdefault(n, []).append(turn)
        yield n

def n_th(n, it):
    for i, v in enumerate(it):
        if i == n:
            return v

def play_game(numbers, turn):
    return n_th(turn - 1, yield_game(numbers))

def run_tests():
    example1 = [0, 3, 6]
    assert list(itertools.islice(yield_game(example1), 10)) == [
        0,
        3,
        6,
        0,
        3,
        3,
        1,
        0,
        4,
        0,
    ]
    assert play_game(example1, 1) == 0
    assert play_game(example1, 9) == 4
    assert play_game(example1, 10) == 0
    assert play_game(example1, 2020) == 436
    assert play_game([1, 3, 2], 2020) == 1
    assert play_game([2, 1, 3], 2020) == 10
    assert play_game([1, 2, 3], 2020) == 27
    assert play_game([2, 3, 1], 2020) == 78
    assert play_game([3, 2, 1], 2020) == 438
    assert play_game([3, 1, 2], 2020) == 1836
    # assert play_game(example1, 30000000) == 175594 # Slow

def get_solutions():
    numbers = get_numbers_from_file()
    print(play_game(numbers, 2020) == 273)
    # print(play_game(numbers, 30000000) == 47205) # Slow

if __name__ == "__main__":
    begin = datetime.datetime.now()
    run_tests()
    get_solutions()
    end = datetime.datetime.now()
    print(end - begin)


# import Image as Image
import PIL.Image as Image
import PIL.ImageDraw as ImageDraw
import os
import time

FILE_NAME = 'in.txt'
IMAGES_PATH = './image/'  # 正数图片集地址
IMAGE_SIZE = 36  # 每张小图片的大小
IMAGE_SAVE_PATH = './result/'  # 图片转换后的地址
IMAGE_RESIZE_SCALE = 2.5  # 放大倍率

arrOld = []

def read_file():
    # 读取文件
    with open(FILE_NAME) as f:
        for line in f.readlines():
            # 如果想修改分隔符，改这里
            arr = list(map(int, line.split(',')))
            arrOld.append(arr)

def create_image():

    # 为了蛋疼需求，把数组横过来
    # 建立一个横着的数组
    i = 0
    j = 0
    arr = [[0 for i in range(len(arrOld))] for j in range(len(arrOld[0]))]

    # 旋转数组
    for i in range(0, len(arrOld)):
        for j in range(0, len(arrOld[i])):
            arr[j][i] = arrOld[len(arrOld)-1-i][j]
        pass

    # 计算出总共需要多少格子
    colSize = 0
    for y in range(0, len(arr)):
        c = 0
        for x in range(0, len(arr[y])):
            count = getCountNumber(arr[y][x])
            colSize = max(count, colSize)
            c += count
    totolV = colSize * len(arr[0])

    # 创建一个新图
    to_image = Image.new('RGBA', (totolV * IMAGE_SIZE,
                                  len(arr) * IMAGE_SIZE))

    # 遍历方式有改动
    # 先是从Y轴往上便利
    for y in range(0, len(arr)):
        for x in range(0, len(arr[y])):
            number = arr[y][x]
            # 负数取正
            isPosi = number > 0
            if isPosi == False:
                number = -number
            c = 0  # 当前位
            while number != 0:
                num_img = number % 10
                from_image = convert2Image(num_img, c % 2 == 1, isPosi)
                if from_image != None:
                    # 这里坐标计算是试出来的，对就行，没仔细思考
                    to_image.paste(
                        from_image, (((x+1) * colSize - c - 1) * IMAGE_SIZE, y * IMAGE_SIZE))
                number = int(number / 10)
                c += 1

    # 画辅助线

    draw = ImageDraw.Draw(to_image)
    img_w, img_h = to_image.size
    for x in range(1, len(arr[0])):
        xoffset = x * colSize*IMAGE_SIZE
        draw.line((xoffset, 0, xoffset, img_h * IMAGE_SIZE), fill="blue")
    for y in range(1, len(arr)):
        draw.line((0, y*IMAGE_SIZE, img_w, y*IMAGE_SIZE), fill="blue")

    del draw

    # 如果想要图片变大，改这里
    to_image = to_image.resize((int(img_w * IMAGE_RESIZE_SCALE),
                                int(img_h * IMAGE_RESIZE_SCALE)))

    # 直接显示，不保存
    # to_image.show()
    # 生成一个本地文件
    out = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime()) + '.png'
    to_image.save(IMAGE_SAVE_PATH+out)  # 保存新图

def getCountNumber(num):
    # 计算数字一共几位
    c = 0
    while num != 0:
        num = int(num / 10)
        c += 1
    return c

def convert2Image(num, isEven, isPositive):
    # 读取算筹数字成图片
    # 一位数字，是否是偶数
    if num == 0:
        return None

    # 是否是竖着的数字
    isVert = 'Counting_rod_v'
    if isEven:
        isVert = 'Counting_rod_h'

    # 判断正负数，读取不同图片，这里不做图片颜色处理
    path = IMAGES_PATH + isVert + str(num)
    if isPositive:
        path += 'p'
    else:
        path += 'm'

    image = Image.open(path + '.png')

    # 生成返回图片，用于居中处理
    new_im = Image.new('RGBA', (IMAGE_SIZE, IMAGE_SIZE))
    img_w, img_h = image.size
    offset = (int((IMAGE_SIZE - img_w)/2), int((IMAGE_SIZE - img_h)/2))
    new_im.paste(image, offset)
    return new_im

read_file()
create_image()  # 调用函数


from parser_utils import ParserUtils
from urls import Urls
import re
import urllib.parse
import json
import http.client

NEWS_FILE_NAME = 'news.csv'
COMMENTS_FILE_NAME = 'comments.csv'

# List for storing news objects
news_data = []

# List for storing comment objects
comment_data = []

def main():
    """Point of entry."""

    parse_news_list()
    ParserUtils.write_news_to_csv(NEWS_FILE_NAME, news_data)
    ParserUtils.write_comments_to_csv(COMMENTS_FILE_NAME, comment_data)

def parse_news_list():
    """Parses data(publish_date, title, link, comment count) from news list."""

    soup = ParserUtils.get_soup(Urls.NEWS_LIST, use_proxy=True)

    # Finding all divs with news
    news_div_list = soup.findAll('div', {'class': 'cat_news_item'})

    # Getting today's date from first div element
    date = news_div_list[0].text.strip()

    # Iterating each div excepting first
    for news_div in news_div_list[1:]:
        # News object formation
        news_object = {
            'publish_date': f"{date} {news_div.findChild('span', {'class': 'tahoma font12 date n3'}).text.strip()}",
            'title': news_div.findChild('a', {'class': 'tahoma font12'}).text.strip(),
            'link': f"{Urls.DOMAIN}{news_div.findChild('a', {'class': 'tahoma font12'}).get('href')}",
        }

        # Getting comment count if exists
        comment_count = news_div.findChild('span', {'class': 'comm_num'})
        news_object['comment_count'] = comment_count.text.strip() if comment_count else None

        # Storing to list
        news_data.append(news_object)
        print(f"{news_object.get('title')[:50]:<50} {news_object.get('link')}")
        try:
            parse_news_details(news_object.get('link'))
        except BaseException as e:
            print(e)

def parse_news_details(link):
    """Goes to news details and parses data(text)."""

    soup = ParserUtils.get_soup(link, use_proxy=True)

    # Removing style tags
    styles = soup.findAll('style')
    for style in styles:
        style.decompose()

    # Getting main div where text stores
    news_details_block = soup.find('div', {'class': 'fullnews white_block'})

    # if text in the table
    if not news_details_block:
        news_details_block = soup.find('div', {'class': 'full_story'})

    news_paragraphs = news_details_block.findAll('p')

    text = ''

    for paragraph in news_paragraphs:
        text += paragraph.text + ' '

    # Complementing news object
    recent_news_object = news_data[-1]
    recent_news_object['text'] = re.sub(' +', ' ', text.strip().replace('\n', ''))

    if recent_news_object.get('comment_count'):
        parse_comments(recent_news_object.get('link'), recent_news_object)

def parse_comments(link, news_object):
    """Parses comments."""

    # Making POST request to get comments json
    conn = http.client.HTTPSConnection("zcomments.net")

    page_title = urllib.parse.quote(news_object.get('title'))
    page_url = news_object.get('link')
    block_code = f"zakonnewsid{page_url.split('kz/')[1].split('-')[0]}"

    payload = f'page_title={page_title}&page_url={page_url}&block_code={block_code}'
    headers = {'content-type': "application/x-www-form-urlencoded; charset=UTF-8"}
    conn.request("POST", "/service/init/1", payload, headers)
    res = conn.getresponse()
    data = res.read().decode("utf-8")

    json_data = json.loads(data)
    serialize_comments_json(json_data)

def serialize_comments_json(json_data):
    """Retrieving comments list from json."""

    for item in json_data.get('comments').get('items'):
        parse_comment_details(item)

def parse_comment_details(comment):
    """Parsing comment details."""

    comment_object = {
        'user_nick': comment.get('user_nick'),
        'user_country': comment.get('user_country'),
        'message': comment.get('message'),
        'created_at': comment.get('created_at'),
        'likes': comment.get('likes'),
    }
    comment_data.append(comment_object)

    nested_comments = comment.get('children')

    # If this comment has children
    if nested_comments:
        for nested_comment in nested_comments:
            # Parsing children using recursion
            parse_comment_details(nested_comment)

if __name__ == '__main__':
    main()


#!/usr/bin/env python

import sys
import rosbag
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits import mplot3d
from mavros_msgs.msg import OverrideRCIn, VFR_HUD
from geometry_msgs.msg import PoseStamped
from nav_msgs.msg import Odometry
from std_msgs.msg import UInt8

def q_to_euler(q):
    e1 = q[0]
    e2 = q[1]
    e3 = q[2]
    eta = q[3]
    rot = np.array([[1 - 2 * (e2**2 + e3**2),
                    2 * (e1 * e2 - e3 * eta),
                    2 * (e1 * e3 + e2 * eta)],
                    [2 * (e1 * e2 + e3 * eta),
                    1 - 2 * (e1**2 + e3**2),
                    2 * (e2 * e3 - e1 * eta)],
                    [2 * (e1 * e3 - e2 * eta),
                    2 * (e2 * e3 + e1 * eta),
                    1 - 2 * (e1**2 + e2**2)]])
    # Roll
    roll = np.arctan2(rot[2, 1], rot[2, 2]) / np.pi * 180.0
    # Pitch
    den   = np.sqrt(rot[2, 1]**2 + rot[2, 2]**2)
    pitch = np.arctan(-rot[2, 0]/ max(0.0001, den)) / np.pi * 180.0
    # Yaw
    yaw = np.arctan2(rot[1, 0], rot[0, 0]) / np.pi * 180.0

    euler = np.array([roll, pitch ,yaw])

    return euler

def q_to_dcm(q):
    e1 = q[0]
    e2 = q[1]
    e3 = q[2]
    eta = q[3]

    R = np.array([[1-2*(e2**2+e3**2),
        2*(e1*e2-e3*eta),
        2*(e1*e3+e2*eta)],
        [2*(e1*e2+e3*eta),
        1-2*(e1**2+e3**2),
        2*(e2*e3-e1*eta)],
        [2*(e1*e3-e2*eta),
        2*(e2*e3+e1*eta),
        1-2*(e1**2+e2**2)]])

    return R

q = np.zeros(4)

q[0] = -0.5768
q[1] = -0.0148
q[2] = -0.020
q[3] = -0.8162

euler = q_to_euler(q)
dcm = q_to_dcm(q)

print(euler)
print(dcm)   # dcm: c2w

Rc_w = dcm

Rci_b = np.zeros(shape=(3,3))
Rci_b = np.array([[0.0, 0.0, 1.0],
                  [1.0, 0.0, 0.0],
                  [0.0, 1.0, 0.0]])

Rb_w = np.zeros(shape=(3,3))
Rb_w = np.array([[0.0, 1.0, 0.0],
                  [-1.0, 0.0, 0.0],
                  [0.0, 0.0, 1.0]])

# Rb_w = Rci_w*Rc_ci*Rb_c

r1 = np.dot(np.transpose(Rc_w), Rb_w)
Rci_c = np.dot(r1, Rci_b)
print('-------')
print(Rci_c)

r2 = np.dot(Rci_c, np.transpose(Rci_b))
print(r2)

Rb_c = np.dot(np.transpose(Rc_w),Rb_w)
print(Rb_c)

q = np.zeros(4)

q[0] = 0.0
q[1] = 0.0
q[2] = -0.8930
q[3] = 0.4496

dcm = q_to_dcm(q)
euler =  q_to_euler(q)
print(dcm)
print(euler)


# -*- coding: utf-8 -*-
"""
__author__ = z0305
__date__ = 08/16/2018
__purpose__ =
__site__ = https://mp.weixin.qq.com/s/dkaV4VZ1QBzcsi9gXvCRdw (Python告诉你上海有哪些高性价比的西餐厅)
           https://github.com/pythonml/restaurant
"""
import re
import json
import pymongo
from lxml import etree
import requests
import pandas as pd
import matplotlib.pyplot as plt
from config import STRING_NUM

DB = "restaurant"
headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "en-US,en;q=0.8",
    "Cache-Control": "max-age=0",
    "Connection": "keep-alive",
    "Host": "www.dianping.com",
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
    "Upgrade-Insecure-Requests": "1",
    "Cookie": "cy=1; cye=shanghai; _lxsdk_cuid=1653dcc99a6c8-04ff4f92706bc4-1c2f1702-2f4f0c-1653dcc99a6bb; _lxsdk=1653dcc99a6c8-04ff4f92706bc4-1c2f1702-2f4f0c-1653dcc99a6bb; _hc.v=c668bd43-a103-53a2-3ccb-2d01ad06f352.1534340144; _lxsdk_s=1653dcc99ae-1-d99-0e5%7C%7C48",
}

def get_districts():
    url = "http://www.dianping.com/shanghai/ch10/g116"
    r = requests.get(url, headers=headers, verify=False)
    content = r.content.decode("utf-8")
    print("====")
    print(content)
    root = etree.HTML(content)
    distr_nodes = root.xpath('.//div[@id="region-nav"]')
    if len(distr_nodes) == 0:
        raise Exception("no districts")
    distr_node = distr_nodes[0]
    result = []
    for node in distr_node:
        name = node.attrib["data-click-title"]
        url = node.attrib["href"]
        result.append({"name": name, "url": url})
    return result

def get_sub_districts():
    districts = get_districts()
    client = pymongo.MongoClient()
    db = client[DB]
    for item in districts:
        distr_name = item["name"]
        distr_url = item["url"]
        r = requests.get(distr_url, headers=headers, verify=False)
        content = r.content.decode("utf-8")
        root = etree.HTML(content)
        sub_nodes = root.xpath('.//div[@id="region-nav-sub"]/a[@data-cat-id]')
        for node in sub_nodes:
            url = node.attrib["href"]
            name = node.xpath('./span')[0].text
            db.sub.insert_one({"district": distr_name, "sub": name, "url": url})

def get_restaurants_by_district(entry_url, sub_id):
    i = 1
    client = pymongo.MongoClient()
    db = client[DB]
    while True:
        if i > 50:
            print("{} has more than 50 pages".format(url))
            break

        url = "{}p{}".format(entry_url, i)
        print(url)
        r = requests.get(url, headers=headers, verify=False)
        content = r.content.decode("utf-8")
        root = etree.HTML(content)
        rest_nodes = root.xpath('.//div[@id="shop-all-list"]/ul/li')
        if len(rest_nodes) == 0:
            break
        for node in rest_nodes:
            a_nodes = node.xpath('.//div[@class="tit"]/a')
            if len(a_nodes) == 0:
                continue
            url = a_nodes[0].attrib["href"]
            name = a_nodes[0].attrib["title"]
            db.restaurant_url.insert_one({"name": name, "url": url, "sub_id": sub_id, "is_done": False})
        i += 1

def get_all_restaurants():
    client = pymongo.MongoClient()
    db = client[DB]
    sub_rows = db.sub.find()
    for row in sub_rows:
        district = row["district"]
        url = row["url"]
        sub_id = row["_id"]
        print(district)
        get_restaurants_by_district(url, sub_id)

def get_price(node):
    text = node.text
    matched = re.search(r"人均\D+(\d+)", text)
    price = 0
    if matched:
        price = int(matched.group(1))
    digit_nodes = node.xpath('./span')
    for digit_node in digit_nodes:
        span_class = digit_node.attrib["class"]
        num = STRING_NUM[span_class]
        price = price * 10 + num
        if digit_node.tail:
            matched = re.search(r"(\d+)", digit_node.tail)
            if matched:
                power = len(matched.group(1))
                price = price * 10**power + int(matched.group(1))
    return price

def get_score(node):
    digit_nodes = node.xpath('./span')
    score = 0
    text = node.text
    matched = re.search(r"(\d+)", text)
    if matched:
        score = matched.group(1)

    for digit_node in digit_nodes:
        span_class = digit_node.attrib["class"]
        num = STRING_NUM[span_class]
        score = score * 10 + num
        tail = digit_node.tail
        if tail is not None:
            matched = re.search(r"(\d+)", tail)
            if matched:
                score = score * 10 + int(matched.group(1))
    score = score / 10
    return score

def get_restaurant_details():
    client = pymongo.MongoClient()
    db = client[DB]
    rest_rows = db.restaurant_url.find({"is_done": False})
    done_urls = []
    for row in rest_rows:
        url = row["url"]
        #url = "http://www.dianping.com/shop/5735627"
        #url = "http://www.dianping.com/shop/69208935"
        #url = "http://www.dianping.com/shop/66183142"
        #url = "http://www.dianping.com/shop/13932333"
        if url in done_urls:
            continue
        sub_id = row["sub_id"]
        print(url)
        r = requests.get(url, headers=headers, verify=False)
        content = r.content.decode("utf-8")
        root = etree.HTML(content)
        info_nodes = root.xpath('.//div[@id="basic-info"]')
        if len(info_nodes) == 0:
            print("no info for {}".format(url))
            continue
        name = info_nodes[0].xpath('.//h1[@class="shop-name"]')[0].text
        review_count = 0
        star = 0
        price = 0
        taste_score = 0
        env_score = 0
        service_score = 0
        star_nodes = info_nodes[0].xpath('.//div[@class="brief-info"]/span[contains(@class, "mid-rank-stars")]')
        if len(star_nodes) > 0:
            star_class = star_nodes[0].attrib["class"]
            matched = re.search(r"mid-str(\d+)", star_class)
            if matched:
                star = int(matched.group(1))
        review_nodes = info_nodes[0].xpath('.//div[@class="brief-info"]/span[@id="reviewCount"]')
        if len(review_nodes) > 0:
            review_info = review_nodes[0].text
            matched = re.search(r"(\d+)条评论", review_info)
            if matched:
                review_count = int(matched.group(1))
        price_nodes = info_nodes[0].xpath('.//div[@class="brief-info"]/span[@id="avgPriceTitle"]')
        if len(price_nodes) > 0:
            price = get_price(price_nodes[0])
            print(price)
        comment_nodes = info_nodes[0].xpath('.//div[@class="brief-info"]/span[@id="comment_score"]')
        if len(comment_nodes) > 0:
            score_nodes = comment_nodes[0].xpath('./span')
            for score_node in score_nodes:
                if re.search(r"口味", score_node.text):
                    taste_score = get_score(score_node)
                if re.search(r"环境", score_node.text):
                    env_score = get_score(score_node)
                if re.search(r"服务", score_node.text):
                    service_score = get_score(score_node)
        db.restaurant.insert_one({"name": name,
            "star": star,
            "price": price,
            "review_count": review_count,
            "taste_score": taste_score,
            "env_score": env_score,
            "service_score": service_score,
            "url": url,
            "sub_id": sub_id
        })
        db.restaurant_url.update_many({"url": url}, {"$set": {"is_done": True}})
        done_urls.append(url)

def unique():
    client = pymongo.MongoClient()
    db = client[DB]
    urls = db.restaurant.distinct("url")
    for url in urls:
        row_count = db.restaurant.find({"url": url}).count()
        if row_count == 1:
            continue
        rows = db.restaurant.find({"url": url})
        i = 0
        for row in rows:
            if i == 0:
                i += 1
                continue
            db.restaurant.delete_one({"_id": row["_id"]})
            i += 1

def update_index():
    client = pymongo.MongoClient()
    db = client[DB]
    restaurants = db.restaurant.find({"review_count": {"$gt": 100}})
    for item in restaurants:
        taste_score = item["taste_score"]
        env_score = item["env_score"]
        service_score = item["service_score"]
        price = item["price"]
        if taste_score == 0 or env_score == 0 or service_score == 0:
            continue
        index = 3/(1/taste_score+1/env_score+1/service_score) / price * 1000
        avg_score = 3/(1/taste_score+1/env_score+1/service_score)
        db.restaurant.update({"_id": item["_id"]}, {"$set": {"index": index, "avg_score": avg_score}})

def stats():

    client = pymongo.MongoClient("172.24.176.73")
    db = client[DB]

    print("============ most expensive ===================")
    restaurants = db.restaurant.aggregate([
        {"$match": {"price": {"$gt": 0}}},
        {"$sort": {"price": 1}},
        {"$limit": 10},
    ])
    for item in restaurants:
        name = item["name"]
        review_count = item["review_count"]
        star = item["star"]
        price = item["price"]
        print("{},{}".format(name, price))

    print("============ highest index ===================")
    restaurants = db.restaurant.aggregate([
        {"$sort": {"index": -1}},
        {"$limit": 10}
    ])
    for item in restaurants:
        name = item["name"]
        review_count = item["review_count"]
        star = item["star"]
        price = item["price"]
        index = item["index"]
        taste_score = item["taste_score"]
        env_score = item["env_score"]
        service_score = item["service_score"]
        print("{},{},{},{},{},{}".format(name, price, taste_score, env_score, service_score, index))

    print("============ highest index > 100 ===================")
    restaurants = db.restaurant.aggregate([
        {"$match": {"price": {"$gt": 100}}},
        {"$sort": {"index": -1}},
        {"$limit": 10}
    ])
    for item in restaurants:
        name = item["name"]
        review_count = item["review_count"]
        star = item["star"]
        price = item["price"]
        index = item["index"]
        taste_score = item["taste_score"]
        env_score = item["env_score"]
        service_score = item["service_score"]
        print(name, price, review_count, taste_score, env_score, service_score, index)

    print("============ highest avg_score ===================")
    restaurants = db.restaurant.aggregate([
        {"$match": {"price": {"$gt": 200}}},
        {"$match": {"price": {"$lt": 300}}},
        {"$sort": {"index": -1}},
        {"$limit": 50}
    ])
    for item in restaurants:
        name = item["name"]
        review_count = item["review_count"]
        star = item["star"]
        price = item["price"]
        index = item["index"]
        taste_score = item["taste_score"]
        env_score = item["env_score"]
        service_score = item["service_score"]
        avg_score = item["avg_score"]
        print("{},{},{}".format(name, price, avg_score))

    #restaurants = db.restaurant.find({})
    #df = pd.DataFrame(columns=["price", "avg_score"])
    #for item in restaurants:
    #    df.loc[len(df)] = [item.get("price"), item.get("avg_score")]
    #print(df)
    #fig, ax = plt.subplots()
    #df.avg_score.hist(ax=ax, bins=20)
    #plt.show()

if __name__ == "__main__":
    stats()


#a

n9=input()
cb=0
for i in range(0,len(n9)):
    ss=n9[0:i]+n9[i+1::]
    if int(ss)%8==0:
        cb=1
        break
if cb==1:
    print("yes")
else:
    print("no")


# -*- coding: utf-8 -*-
"""
% EJ_stormScript_loopOnly_SLURM.py uses storm-analysis (Zhuang Lab) to do single molecule localisation with pre-defined parameters
%
%
% INPUT ARGUMENTS
%       experimentName:          '180712Merfish'
%       resultsFolderName:       'processed'
%       windowsDataPath:         "//nfs//winstor//isogai//smfish_data//disk2//Mat//"
%       windowsStormDir:         "//nfs//nhome//live//naureeng/storm-analysis"
%
% OUTPUT
%       HDF5 File:               storm_00_bit_1.hdf5
%       MLIST.BIN File:          storm_00_bit_1.mlist.bin
%       TXT File:                storm_00_bit_1_drift.txt
%       for each bit per round
%
% DEPENDENCIES
%       storm-analysis virtual environment
%       instructions by Hazen Babcock available here: https://storm-analysis.readthedocs.io/en/latest/
%
@author: Elina

Created on Mon Apr  1 11:54:05 2019
# updated to include option of loading default parameters, if don't want to test them individually for all bits

Created on Mon Mar 11 17:45:15 2019
# updated to put the analysis results files into a separate folder

Created on Thu Feb 14 12:32:47 2019
# first version

@author: Naureen
Created on Thu May 23 14:24:00 2019
# updated to be HPC-compatible [SLURM]

@author: Laboratory of Xiaowei Zhuang
# developed original storm-analysis algorithm
# code credit to: https://github.com/ZhuangLab/storm-analysis
"""

#%% initialise script & import all needed libraries etcd

# give the name of the folder that the experiment lives in
experimentName = '180712Merfish'
# create a folder name where the results from this analysis are going to live
resultsFolderName = 'processed'
nBits = 16 # number of bits

import os
import shutil

windowsDataPath = "//nfs//winstor//isogai//smfish_data//disk2//Mat//"
dataFolder = windowsDataPath
experimentFolder = os.path.join(dataFolder,experimentName)

##set up the path to storm
windowsStormDir = "//nfs//nhome//live//naureeng/storm-analysis"
stormDir = windowsStormDir
daostormDir = os.path.join(stormDir,'storm_analysis','daostorm_3d')

defaultParamsDir = os.path.join(dataFolder,'stormParams',experimentName,'defaults')
stormParamsDir = defaultParamsDir
os.chdir(daostormDir)
import storm_analysis.sa_library.parameters as params
daop = params.ParametersDAO().initFromFile("example.xml")
import storm_analysis.daostorm_3d.mufit_analysis as mfit
import storm_analysis.sa_utilities.hdf5_to_bin as hdf5bin

os.chdir(experimentFolder)
stackFolders = os.listdir( experimentFolder )

nFolders    = len(stackFolders)
startFolder = 1;

import numpy as np

#%%start loop to go through hybridisation folders

whichFolders = np.arange(startFolder,nFolders,1)

#for h in range(nFolders):
for h in whichFolders:
    thisFolder = stackFolders[h]
    #print(thisFolder)
    thisFolderPath = os.path.join(experimentFolder,thisFolder)
    thisResultsPath = os.path.join(thisFolderPath,resultsFolderName)
    if os.path.isdir(thisResultsPath):
        print(' ')
    else:
        os.mkdir(thisResultsPath)

    os.chdir(thisFolderPath)

    tifFiles = os.listdir( thisFolderPath ) #generates list of file names
    nTifs =len(tifFiles)
    #start second loop to go through tif files
    for f in range(nTifs):
        inputfile = tifFiles[f]
#        print(inputfile)
        istif = inputfile.find("tif")
        if istif > 0:
            fileLength = len(inputfile)
            if fileLength < 11:             # 10 is the maximum length of the inputfile name if it is either a bit or a fiducial - anything that's longer we can safely ignore (like dapi, cropped or brightness adjusted images, parameter files etc)
                test = inputfile.find("test")
                if test < 0:

                    shutil.copy(inputfile, thisResultsPath)

                    fiducial = inputfile.find("fid") # we need to give different parameters to feducial files
                    if fiducial < 0:
                        # then this is not a fiducial
                        thisParamsFileName = "stormParams_" + inputfile.split(".")[0] + ".xml"
                        if not os.path.isfile(os.path.join(stormParamsDir,thisParamsFileName)):            # if this doesn't exist, apply the default parameters
                            thisParamsFileName = "default_stormParams_bits.xml"
                            if not os.path.isfile(os.path.join(stormParamsDir,thisParamsFileName)):        # if this doesn't exist either, we need to copy the default parameters file over
                                defaultParamsDir = os.path.join(dataFolder,"stormParams","defaults")
                                os.chdir(defaultParamsDir)
                                bitparamsFileName = "default_stormParams_bits.xml"
                                shutil.copy(bitparamsFileName, stormParamsDir)                  # copy the bit parameters
                    else:
                        thisParamsFileName = "stormParams_fid_11.xml"
                        if not os.path.isfile(os.path.join(stormParamsDir,thisParamsFileName)):            # if this doesn't exist, apply the default parameters
                            thisParamsFileName = "default_stormParams_fid.xml"
                            if not os.path.isfile(os.path.join(stormParamsDir,thisParamsFileName)):        # if this doesn't exist either, we need to copy the default parameters file over
                                defaultParamsDir = os.path.join(dataFolder,"stormParams","defaults")
                                os.chdir(defaultParamsDir)
                                fidparamsFileName = "default_stormParams_fid.xml"
                                shutil.copy(fidparamsFileName, stormParamsDir)                  # copy the fiducial parameters

                    os.chdir(stormParamsDir)
                    shutil.copy(thisParamsFileName, thisResultsPath)
                    # we need to have the parameters file in the stack folder for the storm algorithm to find it
                    # this line copies the parameters set in the test face from the dedicated folder into the current stack folder
                    os.chdir(thisResultsPath)

                    hdffile = "storm_" + thisFolder.split("_")[1] + "_" + inputfile.split(".")[0] + ".hdf5"
                    binfile = "storm_" + thisFolder.split("_")[1] + "_" + inputfile.split(".")[0] + ".mlist.bin"

                    if os.path.exists(hdffile):
                        os.remove(hdffile)

                    mfit.analyze(inputfile, hdffile, thisParamsFileName)
                   # overlay_image.overlayImage(inputfile, hdffile, 0)
                    msg = inputfile.split(".")[0] + ' ' + thisFolder
                    print(msg)
                    hdf5bin.hdf5ToBin(hdffile, binfile)

                    os.remove(inputfile)
                    os.chdir(thisFolderPath)


import nltk
from nltk.corpus import treebank
from itertools import islice
from nltk.grammar import PCFG, induce_pcfg, toy_pcfg1, toy_pcfg2

# PCFG中的规则
gram2 = PCFG.fromstring("""
	A -> B B [.3] | C B C [.7]
	B -> B D [.5] | C [.5]
	C -> 'a' [.1] | 'b' [0.9]
	D -> 'b' [1.0]
	""")

prod1 = gram2.productions()[0]

print(prod1)  # A -> B B [0.3]

prod2 = gram2.productions()[1]

print(prod2)  # A -> C B C [0.7]

print(prod2.lhs())  # A

print(prod2.rhs())  # (C, B, C)

print((prod2.prob()))  # 0.7

print(gram2.start())  # A

print(gram2.productions())

# >>>[A -> B B [0.3], A -> C B C [0.7], B -> B D [0.5], B -> C [0.5], C -> 'a' [0.1], C -> 'b' [0.9], D -> 'b' [1.0]]

# NLTK中用于说明概率分布图的解析代码
tokens = "Jack told Bob to bring my cookie".split()

grammar = toy_pcfg2

print(grammar)

'''
    Grammar with 23 productions (start state = S)
    S -> NP VP [1.0]
    VP -> V NP [0.59]
    VP -> V [0.4]
    VP -> VP PP [0.01]
    NP -> Det N [0.41]
    NP -> Name [0.28]
    NP -> NP PP [0.31]
    PP -> P NP [1.0]
    V -> 'saw' [0.21]
    V -> 'ate' [0.51]
    V -> 'ran' [0.28]
    N -> 'boy' [0.11]
    N -> 'cookie' [0.12]
    N -> 'table' [0.13]
    N -> 'telescope' [0.14]
    N -> 'hill' [0.5]
    Name -> 'Jack' [0.52]
    Name -> 'Bob' [0.48]
    P -> 'with' [0.61]
    P -> 'under' [0.39]
    Det -> 'the' [0.41]
    Det -> 'a' [0.31]
    Det -> 'my' [0.28]
'''

5.语法解析-分析材料/5.3 CFG创建概率上下文语法.py

5.3 CFG创建概率上下文语法.py


import sst

### Create the components
component0 = sst.Component("c0", "hg.node")
component1 = sst.Component("c1", "hg.node")

### Parameterize the components.
# Run 'sst-info simpleElementExample.example0' at the command line
# to see parameter documentation
params = {
        "eventsToSend" : 50,    # Required parameter, error if not provided
        "eventSize" : 32        # Optional parameter, defaults to 16 if not provided
}
component0.addParams(params)
component1.addParams(params)

### Link the components via their 'port' ports
link = sst.Link("component_link")
link.connect( (component0, "port", "1ns"), (component1, "port", "1ns") )

### Enable statistics
sst.setStatisticLoadLevel(7)

# Determine where statistics should be sent
sst.setStatisticOutput("sst.statOutputConsole")

# Enable statistics on both components
sst.enableAllStatisticsForComponentType("hg.node")


from core.monitorizer import Monitorizer
from core.arguments import args
from datetime import timedelta
from datetime import datetime
from events.on import Events
from core import flags
from time import sleep
import os

scanners = [
	"subfinder",
	"sublist3r",
	"dnsrecon",
	"dnscan",
	"aiodnsbrute",
	#"amass",
	#"subbrute" - not recommended
]

monitorizer = Monitorizer()
if args.debug == False:
	monitorizer.clear()
events = Events()

if os.path.isfile(args.watch):
	_watch_list = 'set([t.strip() for t in open(args.watch,"r").readlines()])'
	monitorizer.log("Reading targets from file: %s" % args.watch)

else:
	monitorizer.banner()
	monitorizer.error("Couldn't read watch list")
	monitorizer.exit()

monitorizer.set_config(args.config)
events.set_config(args.config)
monitorizer.initialize()
monitorizer.self_check(scanners)
monitorizer.banner()
events.start()

while 1:
	for target in eval(_watch_list):
		flags.status = "running"
		flags.current_target = target

		if not target: continue

		report_name  = str(datetime.now().strftime("%Y%m%d_%s"))
		flags.report_name = report_name
		monitorizer.log("Created new report target=%s name=%s" %(target,report_name))

		current_scan = monitorizer.mutliscan(scanners,target)
		monitorizer.generate_report(target,current_scan,report_name)

		new_domains  = monitorizer.merge_scans(current_scan)
		old_domains  = monitorizer.merge_reports(target,exclude=[report_name])

		if len(old_domains) > 0:
			new_domains = new_domains - old_domains
		else:
			new_domains = []

		for domain in new_domains:
			foundby = []
			for tool,subs in current_scan.items():
				if not domain in subs:
					continue
				foundby.append(tool)
			events.discover(domain,foundby,report_name)

		if args.debug == False:
			monitorizer.clean_temp()

	flags.status = "idle"
	monitorizer.log("Sleeping %i hour(s)" % (flags.sleep_time))
	sleep( 60*60*flags.sleep_time)