#!/usr/bin/env python3
# coding=utf-8

#
# Copyright (c) 2020-2021 Huawei Device Co., Ltd.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import json
import time
import re

from _core.constants import DeviceTestType
from _core.exception import ExecuteTerminate
from _core.exception import LiteDeviceTimeout
from _core.exception import LiteDeviceConnectError
from _core.exception import LiteDeviceExecuteCommandError
from _core.logger import platform_logger
from _core.utils import convert_port

__all__ = ["generate_report", "LiteHelper"]

CPP_TEST_STANDARD_SIGN = "[==========]"
CPP_TEST_END_SIGN = "Gtest xml output finished"
CPP_SYS_STANDARD_SIGN = "OHOS #"
CPP_ERR_MESSAGE = "[ERR]No such file or directory: "
CTEST_STANDARD_SIGN = "Start to run test suite"
AT_CMD_ENDS = "OK"
CTEST_END_SIGN = "All the test suites finished"

_START_JSUNIT_RUN_MARKER = "[start] start run suites"
_END_JSUNIT_RUN_MARKER = "[end] run suites end"
INSTALL_END_MARKER = "resultMessage is install success !"

PATTERN = re.compile(r'\x1B(\[([0-9]{1,2}(;[0-9]{1,2})*)?m)*')
TIMEOUT = 90
LOG = platform_logger("DmlibLite")

def check_open_source_test(result_output):
    if result_output.find(CPP_TEST_STANDARD_SIGN) == -1 and \
            ("test pass" in result_output.lower() or
             "test fail" in result_output.lower() or
             "tests pass" in result_output.lower() or
             "tests fail" in result_output.lower()):
        return True
    return False

def check_read_test_end(result=None, input_command=None):
    temp_result = result.replace("\n", "")
    if input_command not in temp_result:
        return False
    index = result.find(input_command) + len(input_command)
    result_output = result[index:]
    if input_command.startswith("./"):
        if result_output.find(CPP_TEST_STANDARD_SIGN) != -1:
            if result_output.count(CPP_TEST_STANDARD_SIGN) == 2 or \
                    result_output.find(CPP_TEST_END_SIGN) != -1:
                return True
        if check_open_source_test(result_output):
            return True
        if result_output.find(_START_JSUNIT_RUN_MARKER) >= 1 and \
                result_output.find(_END_JSUNIT_RUN_MARKER) >= 1:
            return True

        if result_output.find(INSTALL_END_MARKER) != -1:
            return True

        if "%s%s" % (CPP_ERR_MESSAGE, input_command[2:]) in result_output:
            LOG.error("execute file not exist, result is %s" % result_output,
                      error_no="00402")
            raise LiteDeviceExecuteCommandError("execute file not exist",
                                                error_no="00402")
    else:
        if "OHOS #" in result_output or "Linux" in result_output:
            if input_command == "reboot" or input_command == "reset":
                return False
            if input_command.startswith("mount"):
                if "Mount nfs finished." not in result_output:
                    return False
            return True
    return False

def generate_report(receiver, result):
    if result and receiver:
        if result:
            receiver.__read__(result)
            receiver.__done__()

def get_current_time():
    current_time = time.time()
    local_time = time.localtime(current_time)
    data_head = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
    millisecond = (current_time - int(current_time)) * 1000
    return "%s.%03d" % (data_head, millisecond)

class LiteHelper:
    @staticmethod
    def execute_remote_cmd_with_timeout(telnet, command="", timeout=TIMEOUT,
                                        receiver=None):
        """
        Executes command on the device.

        Parameters:
            telnet:
            command: the command to execute
            timeout: timeout for read result
            receiver: parser handler
        """
        from xdevice import Scheduler
        time.sleep(2)
        start_time = time.time()
        status = True
        error_message = ""
        result = ""
        if not telnet:
            raise LiteDeviceConnectError("remote device is not connected.",
                                         error_no="00402")

        telnet.write(command.encode('ascii') + b"\n")
        while time.time() - start_time < timeout:
            data = telnet.read_until(bytes(command, encoding="utf8"),
                                     timeout=1)
            data = PATTERN.sub('', data.decode('gbk', 'ignore')).replace(
                "\r", "")
            result = "{}{}".format(result, data)
            if command in result:
                break

        expect_result = [bytes(CPP_TEST_STANDARD_SIGN, encoding="utf8"),
                         bytes(CPP_SYS_STANDARD_SIGN, encoding="utf8"),
                         bytes(CPP_TEST_END_SIGN, encoding="utf8")]
        while time.time() - start_time < timeout:
            if not Scheduler.is_execute:
                raise ExecuteTerminate("Execute terminate", error_no="00300")
            _, _, data = telnet.expect(expect_result, timeout=1)
            data = PATTERN.sub('', data.decode('gbk', 'ignore')).replace(
                "\r", "")
            result = "{}{}".format(result, data)
            if receiver and data:
                receiver.__read__(data)
            if check_read_test_end(result, command):
                break
        else:
            error_message = "execute %s timed out %s " % (command, timeout)
            status = False

        if receiver:
            receiver.__done__()

        if not status and command.startswith("uname"):
            raise LiteDeviceTimeout("Execute command time out:%s" % command)

        return result, status, error_message

    @staticmethod
    def read_local_output_test(com=None, command=None, timeout=TIMEOUT,
                               receiver=None):
        input_command = command
        linux_end_command = ""
        if "--gtest_output=" in command:
            linux_end_command = input_command.split(":")[1].split(
                "reports")[0].rstrip("/") + " #"
        error_message = ""
        start_time = time.time()
        result = ""
        status = True
        from xdevice import Scheduler

        while time.time() - start_time < timeout:
            data = com.readline().decode('gbk', errors='ignore')
            data = PATTERN.sub('', data).replace("\r", "")
            result = "{}{}".format(result, data)
            if command in result or linux_end_command in result:
                break

        while time.time() - start_time < timeout:
            if not Scheduler.is_execute:
                raise ExecuteTerminate("Execute terminate", error_no="00300")
            data = com.readline().decode('gbk', errors='ignore')
            data = PATTERN.sub('', data).replace("\r", "")
            result = "{}{}".format(result, data)
            if receiver and data:
                receiver.__read__(data)
            if check_read_test_end(result, input_command):
                break
        else:
            error_message = "execute %s timed out %s " % (command, timeout)
            status = False

        if receiver:
            receiver.__done__()

        if not status and command.startswith("uname"):
            raise LiteDeviceTimeout("Execute command time out:%s" % command)

        return result, status, error_message

    @staticmethod
    def read_local_output_ctest(com=None, command=None, timeout=TIMEOUT,
                                receiver=None):
        result = ""
        input_command = command

        start = time.time()
        from xdevice import Scheduler
        while True:
            if not Scheduler.is_execute:
                raise ExecuteTerminate("Execute terminate", error_no="00300")
            data = com.readline().decode('gbk', errors='ignore')
            data = PATTERN.sub('', data)
            if isinstance(input_command, list):
                data = "{} {}".format(get_current_time(), data)
                if data and receiver:
                    receiver.__read__(data.replace("\r", ""))
                result = "{}{}".format(result, data.replace("\r", ""))
                if re.search(r"\d+\s+Tests\s+\d+\s+Failures\s+\d+\s+"
                             r"Ignored", data):
                    start = time.time()
                if (int(time.time()) - int(start)) > timeout:
                    break
                if CTEST_END_SIGN in data:
                    break
            else:
                result = "{}{}".format(
                    result, data.replace("\r", "").replace("\n", "").strip())
                if AT_CMD_ENDS in data:
                    return result, True, ""
                if (int(time.time()) - int(start)) > timeout:
                    return result, False, ""

        if receiver:
            receiver.__done__()
        LOG.info('Info: execute command success')
        return result, True, ""

    @staticmethod
    def read_local_output(com=None, command=None, case_type="",
                          timeout=TIMEOUT, receiver=None):
        if case_type == DeviceTestType.ctest_lite:
            return LiteHelper.read_local_output_ctest(com, command,
                                                      timeout, receiver)
        else:
            return LiteHelper.read_local_output_test(com, command,
                                                     timeout, receiver)

    @staticmethod
    def execute_local_cmd_with_timeout(com, **kwargs):
        """
        Execute command on the serial and read all the output from the serial.
        """
        args = kwargs
        command = args.get("command", None)
        input_command = command
        case_type = args.get("case_type", "")
        timeout = args.get("timeout", TIMEOUT)
        receiver = args.get("receiver", None)
        if not com:
            raise LiteDeviceConnectError("local device is not connected.",
                                         error_no="00402")

        LOG.info("local_%s execute command shell %s with timeout %ss" %
                 (convert_port(com.port), command, str(timeout)))

        if isinstance(command, str):
            command = command.encode("utf-8")
            if command[-2:] != b"\r\n":
                command = command.rstrip() + b'\r\n'
                com.write(command)
        else:
            com.write(command)
        return LiteHelper.read_local_output(
            com, command=input_command, case_type=case_type, timeout=timeout,
            receiver=receiver)

    @staticmethod
    def execute_local_command(com, command):
        """
        Execute command on the serial and read all the output from the serial.
        """
        if not com:
            raise LiteDeviceConnectError("local device is not connected.",
                                         error_no="00402")

        LOG.info(
            "local_%s execute command shell %s" % (convert_port(com.port),
                                                   command))
        command = command.encode("utf-8")
        if command[-2:] != b"\r\n":
            command = command.rstrip() + b'\r\n'
            com.write(command)

    @staticmethod
    def execute_agent_cmd_with_timeout(self, command,
                                       **kwargs):
        import requests

        base_url = self.base_url
        headers = self.headers
        local_source_dir = self.local_source_dir
        target_save_path = self.target_save_path
        args = kwargs
        command_type = args.get("type", None)
        sync = args.get("sync", 1)
        response = None
        try:
            if "cmd" == command_type:
                url = base_url + "/_processes/"
                data = {
                    'cmd': command,
                    'sync': sync,
                    'stdoutRedirect': 'file',
                    'stderrRedirect': 'file'
                }
                response = requests.post(url=url, headers=headers,
                                         data=json.dumps(data))
                if response.status_code == 200:
                    LOG.info("connect OK")
                else:
                    LOG.info("connect failed")
                response.close()

            elif "put" == command_type:
                url = base_url + target_save_path + command
                data = open(local_source_dir + command, "rb")
                response = requests.put(url=url, headers=headers, data=data)
                if response.status_code == 200:
                    LOG.info("{} upload file to {} "
                             "success".format(local_source_dir,
                                              target_save_path))
                else:
                    LOG.info(
                        "{} upload file to {} fail".format(local_source_dir,
                                                           target_save_path))
                response.close()
            elif "get" == command_type:
                url = base_url + target_save_path + command
                response = requests.get(url=url, headers=headers, stream=True)
                if response.status_code == 200:
                    file_name = open(local_source_dir + command + "bak", "wb")
                    for file_data in response.iter_content(chunk_size=512):
                        file_name.write(file_data)
                    file_name.close()
                    LOG.info("from {} download  file to {} success".format(
                        target_save_path + command,
                        local_source_dir))
                else:
                    LOG.info("{} download  file to {} fail".format(
                        target_save_path + command,
                        command,
                        local_source_dir))
            elif "delete" == command_type:
                url = base_url + target_save_path + command
                LOG.info(url)
                response = requests.delete(url)
                if response.status_code == 200:
                    LOG.info("delete {}  file  success".format(
                        target_save_path + command))
                else:
                    LOG.info("delete {}  file  fail".format(
                        target_save_path + command))
            else:
                LOG.info("type error")
        except Exception as error_message:
            LOG.info("error_message:{}".format(error_message))
        finally:
            if response:
                response.close()

        return "", True, ""


import os
from pathlib import Path

from hydra import compose, initialize
from hydra.core.config_store import ConfigStore
import torch
import wandb

from shared.configs import BaseConfig, register_configs
from stratification.harness import GEORGEHarness
from stratification.utils.parse_args import get_config
from stratification.utils.utils import init_cuda, set_seed

cs = ConfigStore.instance()
register_configs()

def main() -> None:
    config = get_config()
    with initialize(config_path="../configs"):
        hydra_config = compose(
            config_name="biased_data",
            overrides=[f"data={config['data_config']}", f"bias={config['bias_config']}"],
        )
        hydra_config["data"]["data_split_seed"] = config.get("data_split_seed", config["seed"])
        print(hydra_config)
        biased_data_config = BaseConfig.from_hydra(hydra_config)

    use_cuda = config['use_cuda'] and torch.cuda.is_available()
    set_seed(config['seed'], use_cuda)  # set seeds for reproducibility
    init_cuda(config['deterministic'], config['allow_multigpu'])

    # Initialize wandb with online-logging as the default
    local_dir = Path(".", "local_logging")
    local_dir.mkdir(exist_ok=True)
    if config.get("log_offline", False):
        os.environ["WANDB_MODE"] = "dryrun"
    cluster_model_name = config["cluster_config"]["model"]
    wandb.init(
        entity="predictive-analytics-lab",
        project="suds",
        dir=str(local_dir),
        config=config,
        reinit=True,
        group=config.get("group", f"{config['dataset']}.GEORGE.{cluster_model_name}"),
    )

    torch.multiprocessing.set_sharing_strategy('file_system')
    harness = GEORGEHarness(config, use_cuda=use_cuda)
    harness.save_full_config(config)

    first_mode = 'erm' if (config['mode'] == 'george') else config['mode']
    dataloaders = harness.get_dataloaders(
        config=config, data_config=biased_data_config, mode=first_mode, use_cuda=use_cuda
    )
    num_classes = dataloaders['train'].dataset.get_num_classes('superclass')
    model = harness.get_nn_model(config, num_classes=num_classes, mode=first_mode)

    activ_done = config['activations_dir'] != 'NONE'
    rep_done = config['representation_dir'] != 'NONE'
    cluster_done = config['cluster_dir'] != 'NONE'
    rep_done = (
        rep_done or cluster_done
    )  # if we already have clusters, don't need to do reduction step
    activ_done = (
        activ_done or rep_done
    )  # don't need to get activations if we already have reduced ones
    if config['classification_config']['eval_only']:
        assert activ_done
        if config['cluster_dir'] != 'NONE':
            dataloaders = harness.get_dataloaders(
                config=config,
                data_config=biased_data_config,
                mode=first_mode,
                use_cuda=use_cuda,
                subclass_labels=os.path.join(config['cluster_dir'], 'clusters.pt')
                if os.path.isdir(config['cluster_dir'])
                else config['cluster_dir'],
            )

    # Train a model with ERM
    if activ_done and not (
        config['classification_config']['eval_only']
        or config['classification_config']['save_act_only']
    ):
        erm_dir = config['activations_dir']
    else:
        if (
            config['classification_config']['eval_only']
            or config['classification_config']['save_act_only']
        ):
            erm_dir = config['activations_dir']
            model_path = os.path.join(
                erm_dir, f'{config["classification_config"]["eval_mode"]}_model.pt'
            )
            print(f'Loading model from {model_path}...')
            model.load_state_dict(torch.load(model_path)['state_dict'])
        erm_dir = harness.classify(
            config['classification_config'], model, dataloaders, mode=first_mode
        )

    if (
        config['classification_config']['eval_only']
        or config['classification_config']['save_act_only']
    ):
        exit()

    if config['mode'] == 'george':
        if not config['classification_config']['bit_pretrained'] and not rep_done:
            model.load_state_dict(torch.load(os.path.join(erm_dir, 'best_model.pt'))['state_dict'])

        set_seed(config['seed'], use_cuda)
        # Dimensionality-reduce the model activations
        if rep_done:
            reduction_dir = config['representation_dir']
        else:
            reduction_model = harness.get_reduction_model(config, nn_model=model)
            reduction_dir = harness.reduce(
                config['reduction_config'],
                reduction_model,
                inputs_path=os.path.join(erm_dir, 'outputs.pt'),
            )

        # Cluster the per-superclass features
        if cluster_done:
            cluster_dir = config['cluster_dir']
        else:
            cluster_model = harness.get_cluster_model(config)
            cluster_dir = harness.cluster(
                config['cluster_config'],
                cluster_model,
                inputs_path=os.path.join(reduction_dir, 'outputs.pt'),
            )

        set_seed(config['seed'], use_cuda)  # reset random state
        cluster_label_path = os.path.join(cluster_dir, 'clusters.pt')
        dataloaders = harness.get_dataloaders(
            config,
            mode='george',
            data_config=biased_data_config,
            subclass_labels=cluster_label_path,
            use_cuda=use_cuda,
        )
        model = harness.get_nn_model(config, num_classes=num_classes, mode='george')

        # Train the final (GEORGE) model
        george_dir = harness.classify(
            config['classification_config'], model, dataloaders, mode='george'
        )

if __name__ == '__main__':
    main()


from os.path import abspath, dirname, join

import pytest

from iscan.scan import ImportScanner, convert_source_to_tree, get_base_name, scan_directory  # isort:skip # noqa: E501

CURRENT_DIR = abspath(dirname(__file__))

@pytest.mark.parametrize('module_name, expected', [
    ('absolute.py', ['logging', 'os', 'os.path', 'subprocess', 'sys']),
    ('relative.py', []),
    ('mixed.py', ['logging', 'os', 'os.path', 'subprocess', 'sys']),
])
def test_import_scanner(module_name, expected):
    path_to_module = join(CURRENT_DIR, 'test_modules', module_name)
    tree = convert_source_to_tree(path_to_module)
    scanner = ImportScanner()
    scanner.visit(tree)
    assert scanner.get_imports() == expected

@pytest.mark.parametrize('full_name, expected', [
    ('foo', 'foo'),
    ('foo.bar', 'foo'),
    ('foo.bar.baz', 'foo'),
    ('foo_bar_baz.batman', 'foo_bar_baz')
])
def test_get_base_name(full_name, expected):
    assert get_base_name(full_name) == expected

def test_convert_source_to_tree():
    # This function doesn't do anything other than converting source code into its
    # AST representation using the ast module. No need to test
    assert True

@pytest.mark.parametrize('dir_to_exclude, expected', [
    (None, ['ctypes', 'datetime', 'matplotlib', 'numpy', 'os', 'pandas', 'shutil', 'time']),
    (join(CURRENT_DIR, 'test_package', 'city'), ['matplotlib', 'numpy', 'os', 'pandas', 'shutil', 'time'])  # noqa: E501
])
def test_scan_directory(dir_to_exclude, expected):
    dir_to_scan = join(CURRENT_DIR, 'test_package')
    assert scan_directory(dir_to_scan, dir_to_exclude) == expected


# -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
from celery import shared_task, task
from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)

import psutil
import time
import json
import datetime
import netifaces
from django_redis import get_redis_connection
from collections import defaultdict, OrderedDict
from app.utils import network
from django.utils.translation import ugettext_lazy as _

DJANGO_CELERY_TCP_CONNECT_INFO = "dblog-celery-tcp-connect-info-{time}"
DJANGO_CELERY_TCP_CONNECT_INFO_SHOW = "dblog-celery-tcp-connect-info-show-{time}"

DJANGO_CELERY_NET_MONITOR_INFO = "dblog-celery-net-monitor-info-{network}-{time}"
DJANGO_CELERY_NET_MONITOR_INFO_SHOW = "dblog-celery-net-monitor-info-show-{network}-{time}"

KB = 1 << 10
MB = 1 << 20
GB = 1 << 30
TB = 1 << 40
PB = 1 << 50
EXPIRE_TIME = 3600*24*5
labelStep = 30

# ---------------------------
# TCP 连接情况
@shared_task(bind=False, ignore_result=True, store_errors_even_if_ignored=True)
def tcp_connect_info():
    """  统计 TCP连接情况
    netstat -n | awk '/^tcp/ {++S[$NF]} END {for(a in S) print a, S[a]}'
    TIME_WAIT 8947 等待足够的时间以确保远程TCP接收到连接中断请求的确认
    FIN_WAIT1 15 等待远程TCP连接中断请求，或先前的连接中断请求的确认
    FIN_WAIT2 1 从远程TCP等待连接中断请求
    ESTABLISHED 55 代表一个打开的连接
    SYN_RECV 21 再收到和发送一个连接请求后等待对方对连接请求的确认
    CLOSING 2 没有任何连接状态
    LAST_ACK 4 等待原来的发向远程TCP的连接中断请求的确认

    TCP连接状态详解
    LISTEN： 侦听来自远方的TCP端口的连接请求
    SYN-SENT： 再发送连接请求后等待匹配的连接请求
    SYN-RECEIVED：再收到和发送一个连接请求后等待对方对连接请求的确认
    ESTABLISHED： 代表一个打开的连接
    FIN-WAIT-1： 等待远程TCP连接中断请求，或先前的连接中断请求的确认
    FIN-WAIT-2： 从远程TCP等待连接中断请求
    CLOSE-WAIT： 等待从本地用户发来的连接中断请求
    CLOSING： 等待远程TCP对连接中断的确认
    LAST-ACK： 等待原来的发向远程TCP的连接中断请求的确认
    TIME-WAIT： 等待足够的时间以确保远程TCP接收到连接中断请求的确认
    CLOSED： 没有任何连接状态

    https://blog.csdn.net/qadada/article/details/8423561
    http://www.echojb.com/coldfusion/2016/10/08/228099.html
    http://www.echojb.com/coldfusion/2016/10/08/228099.html

    https://my.oschina.net/u/1169079/blog/397705
    https://gist.github.com/ameizi/cb126be7383fb463eae8
    https://gist.github.com/ameizi/cb126be7383fb463eae8
    :return:
    """
    status_list = ["CLOSED", "CLOSE_WAIT", "CLOSING", "ESTABLISHED", "FIN_WAIT1", "FIN_WAIT2", "LAST_ACK", "LISTEN",
                   "SYN_RECV", "SYN_SENT", "TIME_WAIT"]
    fmt = time.strftime("%Y-%m-%d %H:%M")
    time_fmt1, time_fmt2 = fmt.split(" ")
    key = DJANGO_CELERY_TCP_CONNECT_INFO.format(time=time_fmt1)
    status_temp = []
    net_connections = psutil.net_connections()
    for k in net_connections:
        status_temp.append(k.status)

    status_result = []
    for status in status_list:
        status_result.append((
            status, status_temp.count(status) ))
    redis = get_redis_connection()
    d = json.dumps({ 'time': time_fmt2, "results": status_result })
    res = ( redis.pipeline()
            .lpush(key, d)
            .expire(key, EXPIRE_TIME)
            .lrange(key, 0, -1)
            .execute())

    # 缓存
    stat_cpu = res[-1]
    set_tcp_connect_info(redis, stat_cpu, time_fmt1)

def set_tcp_connect_info(redis, stat_cpu, fmt):
    key = DJANGO_CELERY_TCP_CONNECT_INFO_SHOW.format(time=fmt)
    stat_cpu = reversed(stat_cpu)
    categories = []
    datasets = defaultdict(list)
    for x in stat_cpu:
        x = json.loads(x)
        category = x['time']
        x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11 = x['results']
        categories.append(category)
        datasets['CLOSED'].append(x1[1])
        datasets['CLOSE_WAIT'].append(x2[1])
        datasets['CLOSING'].append(x3[1])
        datasets['ESTABLISHED'].append(x4[1])
        datasets['FIN_WAIT1'].append(x5[1])
        datasets['FIN_WAIT2'].append(x6[1])
        datasets['LAST_ACK'].append(x7[1])
        datasets['LISTEN'].append(x8[1])
        datasets['SYN_RECV'].append(x9[1])
        datasets['SYN_SENT'].append(x10[1])
        datasets['TIME_WAIT'].append(x11[1])
    datasets = dict(datasets)
    datasets_lists = []
    for k, v in datasets.iteritems():
        datasets_lists.extend(v)
    yaxisminValue = min(datasets_lists)
    # yaxisminValue = min([i for i in datasets_lists if i>0])
    yaxismaxValue = max(datasets_lists)
    redis.set(key, json.dumps({
        "datasets": datasets,
        "categories": categories,
        "yaxisminValue": yaxisminValue,
        "yaxismaxValue": yaxismaxValue,
    }))

def get_tcp_connect_info():
    now = datetime.datetime.now()
    before_now = now - datetime.timedelta(days=1)
    now_fmt = now.strftime("%Y-%m-%d")
    before_now_fmt = before_now.strftime("%Y-%m-%d")
    now_key = DJANGO_CELERY_TCP_CONNECT_INFO_SHOW.format(time=now_fmt)
    before_now_key = DJANGO_CELERY_TCP_CONNECT_INFO_SHOW.format(time=before_now_fmt)

    redis = get_redis_connection()
    now_tcp = _get_tcp_connect_info(redis, now_key)
    # before_now_tcp = _get_tcp_connect_info(redis, now_key, bn="bnow")
    before_now_tcp = _get_tcp_connect_info(redis, before_now_key, bn="bnow")
    return now_tcp, before_now_tcp

def _get_tcp_connect_info(redis, key, bn="now"):
    j = redis.get(key)
    if not j: return None
    d = json.loads(j)
    categories = d["categories"]
    # length = len(categories)
    # index = 0
    # while 1:
    #     if index>=length:break
    #     _c = categories[index]
    #     if _c.endswith(":00"):
    #         break
    #     index +=1
    datasets = d["datasets"]
    yaxisminValue = d["yaxisminValue"]
    yaxismaxValue = get_tcp_yaxismaxValue(d["yaxismaxValue"])
    category = "|".join(categories)
    dataset = []
    for status in ("CLOSED", "CLOSE_WAIT", "CLOSING", "ESTABLISHED", "FIN_WAIT1", "FIN_WAIT2", "LAST_ACK", "LISTEN", "SYN_RECV", "SYN_SENT", "TIME_WAIT"):
        data = datasets[status]
        dataset.append({
            "seriesname": status,
            "data": "|".join(map(str, data))
        })
    caption = _(u"今日TCP连接情况") % {}
    if bn=="bnow":
        caption = _(u"昨日TCP连接情况") % {}
    js = {
        'type': 'zoomline',
        'renderAt': 'tcp-info-container-{}'.format(bn),
        'width': '100%',
        'height': '500',
        'dataFormat': 'json',
        'dataSource': {
            "chart": {
                "caption": caption,  # 图表主标题
                "subcaption": "",  # 图表副标题
                "paletteColors": "#9585BF,#a90329,#3e7c8a,#c79121,#D6487E,#629B58,#ba43a9,#060AFE,#705da0,#cca300,#447ebc,#447ebc",
                # "paletteColors": "#7eb26d,#eab839,#3e7c8a,#ef843c,#e24d42,#1f78c1,#ba43a9,#705da0,#508642,#cca300,#447ebc,#447ebc",
                "captionFontSize": "16",
                "subcaptionFontSize": "16",
                "subcaptionFontBold": "0",
                "showBorder": "0",
                "bgColor": "#ffffff", # 图表背景色，6位16进制颜色值
                "canvasBgColor": "#1f1d1d",
                # "canvasBgColor": "#ffffff",  # 画布背景色，6位16进制颜色值
                # "bgColor": "#1f1d1d",
                "baseFont": "Helvetica Neue,Arial", # 图表字体样式
                "baseFontSize": "12", # 图表字体大小
                "showCanvasBorder": "0",
                "showShadow": "0",
                "showAlternateHGridColor": "0",
                "yaxisname": _(u"连接数") % {}, # 纵向坐标轴(y轴)名称
                "xaxisname": _(u"时间") % {}, # 横向坐标轴(x轴)名称
                "yaxisminValue": "{}".format(yaxisminValue),
                "yaxismaxValue": "{}".format(yaxismaxValue),
                "forceAxisLimits" : "1",
                "pixelsPerPoint": "0",

                "pixelsPerLabel": "12",  #间隔，每60分钟一个间隔

                "lineThickness": "1",
                "compactdatamode" : "1",
                "dataseparator" : "|",
                "labelHeight": "30",
                "scrollheight": "10",
                "flatScrollBars": "1",
                "scrollShowButtons": "0",
                "scrollColor": "#cccccc",
                "legendBgAlpha": "0",
                "legendBorderAlpha": "0",
                "legendShadow": "0",
                "legendItemFontSize": "14", # 项目字体大小
                "legendItemFontColor": "#666666",
                "formatNumberScale":"0",
                "showhovercap": True,
                "labelStep": "{}".format(labelStep),#间隔，每60分钟一个间隔
                # "numVDivLines": "12", # 垂直分割线
                # "numberSuffix": " KB/s",# 增加数字后缀
                # "numberPrefix": "KB/S", # 增加数字前缀     % 为 '%25'
                "numDivLines": 5,  # 水平分割线
            },
            "categories": [{
                "category":  "{}".format(category)
            }],
            "dataset": dataset
        }
    }
    return json.dumps(js)

# ---------------------------
# 监控网卡流量
@shared_task(bind=False, ignore_result=True, store_errors_even_if_ignored=True)
def network_monitor_info():
    key_info, net_in, net_out = network.get_nets_io_rate(network.get_nets_io)
    # time_fmt = time.strftime("%Y-%m-%d %H:%M")
    now = datetime.datetime.now()
    bnow_fmt = (now - datetime.timedelta(days=1)).strftime("%Y-%m-%d")
    bbnow_fmt = (now - datetime.timedelta(days=2)).strftime("%Y-%m-%d")
    now_fmt_ = now.strftime("%Y-%m-%d %H:%M")
    now_fmt, time_fmt2 = now_fmt_.split(" ")
    redis = get_redis_connection()
    for k in key_info:
        key = DJANGO_CELERY_NET_MONITOR_INFO.format(network=k, time=now_fmt)
        _in = net_in.get(k)
        _out = net_out.get(k)
        d = json.dumps({'time': time_fmt2, "results": [_in, _out]})
        res = (redis.pipeline()
               .lpush(key, d)
               .expire(key, EXPIRE_TIME)
               # .ltrim(key, 0, 12*24*3)
               .lrange(key, 0, -1)
               .execute())
        stat_net = res[-1]
        set_network_monitor_info(redis, reversed(stat_net), k, now_fmt, bnow_fmt, bbnow_fmt)

def set_network_monitor_info(redis, nstat_net, net, now_fmt, bnow_fmt, bbnow_fmt):
    bkey = DJANGO_CELERY_NET_MONITOR_INFO.format(network=net, time=bnow_fmt)
    bbkey = DJANGO_CELERY_NET_MONITOR_INFO.format(network=net, time=bbnow_fmt)
    bstat_net = reversed(redis.lrange(bkey, 0, -1))
    bbstat_net = reversed(redis.lrange(bbkey, 0, -1))
    skey = DJANGO_CELERY_NET_MONITOR_INFO_SHOW.format(network=net, time=now_fmt)
    bskey = DJANGO_CELERY_NET_MONITOR_INFO_SHOW.format(network=net, time=bnow_fmt)
    bbskey = DJANGO_CELERY_NET_MONITOR_INFO_SHOW.format(network=net, time=bbnow_fmt)
    # stat_net = []
    # stat_net.extend(bbstat_net)
    # stat_net.extend(bstat_net)
    # stat_net.extend(nstat_net)
    for key, stats in [(skey, nstat_net),(bskey, bstat_net),(bbskey, bbstat_net)]:
        if not stats: continue
        categories = []
        datasets = defaultdict(list)
        for x in stats:
            x = json.loads(x)
            category = x['time']
            categories.append(category)
            x1, x2 = x['results']
            datasets['Incoming network traffic on {}'.format(net)].append(x1)
            datasets['Outgoing network traffic on {}'.format(net)].append(x2)
        datasets = dict(datasets)
        if not datasets: continue
        datasets_lists = []
        for k, v in datasets.iteritems():
            datasets_lists.extend(v)
        yaxisminValue = min(datasets_lists)
        # yaxisminValue = min([i for i in datasets_lists if i>0])
        yaxismaxValue = max(datasets_lists)
        ( redis.pipeline()
          .set(key, json.dumps({
            "datasets": datasets,
            "categories": categories,
            "yaxisminValue": yaxisminValue,
            "yaxismaxValue": yaxismaxValue,}))
          .expire(key, EXPIRE_TIME)
          .execute())

def get_network_monitor_info():
    key_info = network.get_nets()  # 获取网卡名称
    j = []
    keys = []
    redis = get_redis_connection()
    routingNicName = str(netifaces.gateways()['default'][netifaces.AF_INET][1])
    key_info.remove(routingNicName)
    key_info.insert(0, routingNicName)
    now = datetime.datetime.now()
    bnow_fmt = (now - datetime.timedelta(days=1)).strftime("%Y-%m-%d")
    bbnow_fmt = (now - datetime.timedelta(days=2)).strftime("%Y-%m-%d")
    now_fmt = now.strftime("%Y-%m-%d")
    for k in key_info:
        d = _get_network_monitor_info(redis, k, now_fmt, bnow_fmt, bbnow_fmt)
        if d:
            keys.append(k)
            j.append( (k, d) )
    return keys, j

def _get_network_monitor_info(redis, net, now_fmt, bnow_fmt, bbnow_fmt):
    # key = DJANGO_CELERY_NET_MONITOR_INFO_SHOW.format(network=net, time=now_fmt)
    skey = DJANGO_CELERY_NET_MONITOR_INFO_SHOW.format(network=net, time=now_fmt)
    bskey = DJANGO_CELERY_NET_MONITOR_INFO_SHOW.format(network=net, time=bnow_fmt)
    bbskey = DJANGO_CELERY_NET_MONITOR_INFO_SHOW.format(network=net, time=bbnow_fmt)
    datajs = []
    days = [('today', _(u'今日'), skey, now_fmt), ('yest', _(u'昨日'), bskey, bnow_fmt), ('bday', _(u'前日'), bbskey, bbnow_fmt)]
    for daykey, day, key, day_fmt in days:
        j = redis.get(key)
        if not j:
            datajs.append((daykey, day, None))
            continue
        d = json.loads(j)
        categories = d["categories"]
        # length = len(categories)
        # index = 0
        # while 1:
        #     if index>=length:break
        #     _c = categories[index]
        #     if _c.endswith(":00"):
        #         break
        #     index +=1
        datasets = d["datasets"]
        yaxismaxValue = d["yaxismaxValue"]
        _unit, _unit_str, _unit_round = KB, 'KB/s', 1
        if yaxismaxValue>= MB:
            _unit, _unit_str, _unit_round  = MB, 'MB/s', 2
        yaxisminValue = 0
        yaxismaxValue = network.trans_io(d["yaxismaxValue"], _unit, _unit_round)
        yaxismaxValue = get_net_yaxismaxValue(yaxismaxValue, _unit)
        category = "|".join(categories)
        _in_key = 'Incoming network traffic on {}'.format(net)
        _out_key = 'Outgoing network traffic on {}'.format(net)
        _ins = [network.trans_io(i, _unit, _unit_round) for i in datasets[_in_key]]
        _outs = [network.trans_io(i, _unit, _unit_round) for i in datasets[_out_key]]
        dataset = [{
            "seriesname": _in_key,
            "data": "|".join(map(str, _ins))
        },{
            "seriesname": _out_key,
            "data": "|".join(map(str, _outs))
        }]
        caption = _(u'%(name)s网卡%(day)s流量监控') % {'name': net, "day": day_fmt}
        js = {
            'type': 'zoomline',
            'renderAt': 'net-info-container-{}-{}'.format(net, daykey),
            'width': '100%',
            'height': '500',
            'dataFormat': 'json',
            'dataSource': {
                "chart": {
                    "caption": caption,  # 图表主标题
                    "subcaption": "",  # 图表副标题
                    "paletteColors": "#c79121,#447ebc",
                    "captionFontSize": "16",
                    "subcaptionFontSize": "16",
                    "subcaptionFontBold": "0",
                    "showBorder": "0",
                    "bgColor": "#ffffff", # 图表背景色，6位16进制颜色值
                    "canvasBgColor": "#1f1d1d",
                    # "canvasBgColor": "#ffffff",  # 画布背景色，6位16进制颜色值
                    # "bgColor": "#1f1d1d",
                    "baseFont": "Helvetica Neue,Arial", # 图表字体样式
                    "baseFontSize": "12", # 图表字体大小
                    "showCanvasBorder": "0",
                    "showShadow": "0",
                    "showAlternateHGridColor": "0",
                    "yaxisname": _(u"流量") % {}, # 纵向坐标轴(y轴)名称
                    "xaxisname": _(u"时间") % {}, # 横向坐标轴(x轴)名称
                    "yaxisminValue": "{}".format(yaxisminValue),
                    "yaxismaxValue": "{}".format(yaxismaxValue),
                    "forceAxisLimits" : "1",
                    "pixelsPerPoint": "0",

                    "pixelsPerLabel": "12",  #间隔，每60分钟一个间隔

                    "lineThickness": "1",
                    "compactdatamode" : "1",
                    "dataseparator" : "|",
                    "labelHeight": "30",
                    "scrollheight": "10", #
                    "flatScrollBars": "1",
                    "scrollShowButtons": "0",
                    "scrollColor": "#cccccc",
                    "legendBgAlpha": "0",
                    "legendBorderAlpha": "0",
                    "legendShadow": "0",
                    "legendItemFontSize": "14", # 项目字体大小
                    "legendItemFontColor": "#666666",
                    "formatNumberScale":"0",
                    "showhovercap": True,
                    "labelStep": "{}".format(labelStep),  # 间隔，每60分钟一个间隔
                    # "numVDivLines": "12", # 垂直分割线
                    "numberSuffix": _unit_str,# 增加数字后缀
                    # "numberPrefix": "KB/S", # 增加数字前缀     % 为 '%25'
                    "numDivLines": 5, # 水平分割线
                },
                "categories": [{
                    "category":  "{}".format(category)
                }],
                "dataset": dataset
            }
        }
        datajs.append((daykey, day, json.dumps(js)))
    return datajs

def get_tcp_yaxismaxValue(yaxismaxValue):
    if yaxismaxValue>200:
        d = 150
        while 1:
            if d >= yaxismaxValue:
                return d
            d += 30
    else:
        d = 60
        while 1:
            if d >= yaxismaxValue:
                return d
            d += 30

def get_net_yaxismaxValue(yaxismaxValue, unit):
    if unit == MB:
        d = 1.2
        while 1:
            if d>=yaxismaxValue:
                return d
            d += 0.3
    else:
        if yaxismaxValue>=30:
            d = u = 30
        else:
            d = u = 3
        while 1:
            if d >= yaxismaxValue:
                return d
            d += u


import sys
sys.path.append('../')

from nltk.corpus import wordnet as wn
import pandas as pd
import json
import os

import config
from utils import create_uri, extract_wn_version_id, map_v31_to_v30, create_df_with_wordnet_nodes

cnfile='../input/conceptnet/conceptnet-en-with-externalurl.csv'
data_source=config.wn_ds
weight="1.0"
VERSION=config.VERSION

EDGE_COLS=config.edges_cols
NODE_COLS=config.nodes_cols

cn_nodes_file=f'../output_v{VERSION}/conceptnet/nodes_v{VERSION}.csv'
wn_nodes_file=f'../output_v{VERSION}/wordnet/nodes_v{VERSION}.csv'
vg_nodes_file=f'../output_v{VERSION}/visualgenome/nodes_v{VERSION}.csv'

wordnet30_ili_file='../input/mappings/ili-map-pwn30.tab'
wordnet31_ili_file='../input/mappings/ili-map-pwn31.tab'

# OUTPUT FILE
output_dir=f'../output_v{VERSION}/mappings'
edges_file=f'{output_dir}/wn_wn_mappings.csv'

MOWGLI_NS=config.mowgli_ns
WORDNET_NS=config.wordnet_ns

SAMEAS_REL=create_uri(MOWGLI_NS, config.sameas)

if not os.path.exists(output_dir):
    os.makedirs(output_dir)

### Load the CN filtered data in pandas ###

df=pd.read_csv(cnfile, sep='\t', header=None, converters={4: json.loads})
df.columns=['assertion','rel','subj','obj','metadata']
df.drop(columns=['assertion'])
print('size of df with external URLs', len(df))

df_wordnet=df.loc[(df['rel'] == '/r/ExternalURL') & (df['obj'].str.contains(r'http://wordnet-'))]
print('size of df with wordnet external links', len(df_wordnet))

### Get WordNet nodes ###

all_nodes=set()
wn_nodes=set()

with open(cn_nodes_file, 'r') as f:
    for line in f:
        first=line.split('\t')[0]
        all_nodes.add(first)

with open(wn_nodes_file, 'r') as f:
    for line in f:
        first=line.split('\t')[0]
        all_nodes.add(first)
        wn_nodes.add(first)
wn_nodes_df=pd.read_csv(wn_nodes_file, sep='\t', header=0, converters={5: eval})

with open(vg_nodes_file, 'r') as f:
    for line in f:
        first=line.split('\t')[0]
        if first.startswith('wn:'):
            all_nodes.add(first)
            wn_nodes.add(first)

print('wordnet30 nodes:', len(wn_nodes))

### Load Wordnet mapings 3.0 to 3.1 ###

mapping_31_30=map_v31_to_v30(wordnet31_ili_file, wordnet30_ili_file)
print('v31 to v30 mappings', len(mapping_31_30))

not_in_ili_nodes=set()
missing_nodes=set()
all_edges=[]
for i, row in df_wordnet.iterrows():
    wn_version, wn_offset_id=extract_wn_version_id(row['obj'])
    if wn_version=='wn31':
        if wn_offset_id not in mapping_31_30.keys():
            not_in_ili_nodes.add(wn_offset_id)
        else:
            wn_30_id=mapping_31_30[wn_offset_id]
            offset, pos=wn_30_id.split('-')
            wn_30_synset=wn.synset_from_pos_and_offset(pos,int(offset)).name()
            wn_30_synset_uri=create_uri(WORDNET_NS, wn_30_synset)

            #if row['subj'] in all_nodes:# and
            if wn_30_synset_uri in wn_nodes:
                an_edge=[row['subj'], SAMEAS_REL, wn_30_synset_uri, data_source, weight, {}]
                all_edges.append(an_edge)
            else:
                missing_nodes.add(wn_30_synset_uri)
    else:
        print('Other WN version', wn_version)

print('missing ILI index nodes version 31', len(not_in_ili_nodes))
print('nodes missing in the node lists of WordNet and VG', len(missing_nodes))

missing_nodes_df=create_df_with_wordnet_nodes(missing_nodes, data_source, NODE_COLS)
combined_wn_nodes_df=pd.concat([wn_nodes_df, missing_nodes_df])
combined_wn_nodes_df.sort_values('id').to_csv(wn_nodes_file, index=False, sep='\t')

edges_df = pd.DataFrame(all_edges, columns = EDGE_COLS)
edges_df.sort_values(by=['subject', 'predicate','object']).to_csv(edges_file, index=False, sep='\t')

print('Number of edges', len(edges_df))


# -*- coding: utf-8 -*-
####### Routing Logic ########

import sys

sys.path.append('/etc/kamailio/python/')
sys.path.append('/usr/local/voip/etc/')
import KSR
import re
import requests
import redis

ROUTES = {
    'event_routes': ['ksr_event_dispatcher'],
    'ksr_branch_manage': ['ksr_branch_manage'],
    'ksr_failure_manage': ['ksr_failure_manage'],
    'ksr_route_async_processing': ['ksr_route_async_REGISTER', 'ksr_route_async_INVITE', 'ksr_route_async_auth'],
    'ksr_route_natdetect': ['ksr_route_natdetect'],
    'ksr_route_natmanage': ['ksr_route_natmanage'],
    'ksr_route_relay': ['ksr_route_relay'],
    'ksr_route_reqinit': ['ksr_route_reqinit'],
    'ksr_route_special': ['ksr_special_routing'],
    'ksr_route_withindlg': ['ksr_route_withindlg'],
    'ksr_x_route_location': ['ksr_x_route_location'],
    'request_route': ['ksr_request_route'],
    'KAMAILIO_DEFINE_HOST': ['DSIDS', 'FLAGS', 'GLOBALS'],
}

# global function to instantiate a kamailio class object
# -- executed when kamailio app_python module is initialized
def mod_init():
    KSR.info("===== from Python mod init\n")
    return kamailio()

# -- {start defining kamailio class}
class kamailio():
    def __init__(self):
        KSR.info('===== kamailio.__init__\n')
        # Re-import modules on reload
        for filename in ROUTES: # app_python3
            if filename in sys.modules:# app_python3
                KSR.info("Reload route:" + filename)# app_python3
                del sys.modules[filename] # app_python3
                exec("import " + filename) # app_python3

    # executed when kamailio child processes are initialized
    def child_init(self, rank):
        KSR.info('===== kamailio.child_init({})\n'.format(rank))
        return 0

    # SIP response handling
    # -- equivalent of reply_route{}
    def ksr_reply_route(self, msg):
        if KSR.is_method_in("IBR"):
            # (I)NVITE, (B)YE, (R)EGITER
            KSR.xlog.xinfo("Get reply $rs:$rr")
            self.append_uuid(KSR.pv.getw("$ci"))
        scode = KSR.pv.getw("$rs")
        if 100 <= scode or scode <= 299:
            self.ksr_route_natmanage()
        return 1

    def ksr_onsend_route(self, msg):
        return 1

    def is_ip_internal(self, host):
        #KSR.xlog.xinfo("Identify RFC1918 for host " + host)
        if KSR.ipops.is_pure_ip(host) > 0:
            if KSR.ipops.is_ip_rfc1918(host) > 0:
                return True
        else:
            KSR.ipops.dns_query(host, "resolved_ip")
            ip = KSR.pv.get("$dns(resolved_ip=>addr)")
            if ip:
                KSR.xlog.xnotice("{} resolved to:{}".format(host, ip))
                if KSR.ipops.is_ip_rfc1918(ip) > 0:
                    KSR.xlog.xnotice("RFC1918")
                    return True
            else:
                KSR.xlog.xerr("Can't resolve host. Return true for case when call can be established")
                return True
        return False

    def notify_asterisk(self, call_id, uuid_hdr, contact, server):
        try:
            # Добавляем в очередь на обработку
            r = redis.StrictRedis(host=server, port=6379, db=3, decode_responses=True)
            r.hmset(contact, { "target":call_id, "status":"registered", "type":"aor" , "uuid": uuid_hdr , "src":"app"})
            r.expire(contact, 60)
            KSR.xlog.xnotice("Generated 'aor' task with target {} for subscriber:{} on asterisk:{}".format(call_id, KSR.pv.getw("$fu"), server))
        except Exception as e_msg:
            KSR.xlog.xerr("Cant notify_asterisk:" + str(e_msg))

    def store_uuid(self, callid, uuid):
        try:
            r = redis.StrictRedis(host='127.0.0.1', port=6379, db=4, decode_responses=True)
            # Сохраняем на 5 минут call-id
            r.setex(callid, 300, uuid)
        except Exception as e_msg:
            KSR.xlog.xerr("Cant store UUID in redis:" + str(e_msg))

    def append_uuid(self, callid):
        if KSR.hdr.is_present("UUID") < 0:
            try:
                r = redis.StrictRedis(host='127.0.0.1', port=6379, db=4, decode_responses=True)
                KSR.xlog.xinfo("Appending uuid {}".format(r.get(callid)))
                KSR.hdr.append("UUID: {}\r\n".format(r.get(callid)))
            except Exception as e_msg:
                KSR.xlog.xerr("Cant get UUID from redis:" + str(e_msg))

    def manage_call_status(self, from_user, action='put'):
        from_user = from_user.lower()
        try:
            r = redis.StrictRedis(host='127.0.0.1', port=6379, db=1, decode_responses=True)
            if action == 'put':
                r.setex("call::{}".format(from_user), 600, 1)
            else:
                r.delete("call::{}".format(from_user))
        except Exception as e_msg:
            KSR.xlog.xerr("Cant get UUID from redis:" + str(e_msg))

    def check_registration_uri(self):
        uri = KSR.pv.getw("$fu")
        if len(uri.split(":")) == 2:
            uri += ":{}".format(KSR.pv.getw("$Rp"))
        try:
            # Добавляем в очередь на обработку
            r = redis.StrictRedis(host='127.0.0.1', port=6379, db=1, decode_responses=True)
            r0 = redis.StrictRedis(host='127.0.0.1', port=6379, db=0, decode_responses=True)
            Uri = KSR.pv.getw("$fU")
            call_id, uuid_hdr = r.hmget(uri, "target", "uuid")
            if r0.sismember("REGLISTEN",Uri) and KSR.pv.getw("$hdr(Expires)") != '0':
                ct = self.make_ct()
                r0.publish(Uri,ct)
            ### old logic
            if call_id :
                if 'ct' not in locals() :
                    ct = self.make_ct()
                self.notify_asterisk(call_id, uuid_hdr, ct, r.hget(uri, "server"))
        except Exception as e_msg:
            KSR.xlog.xerr("Cant check_registration_uri:" + str(e_msg))

    def send_registration_info_to_redis(self,registred,time):
        if KSR.pv.getw("$hdr(Expires)") == '0':
            return
        ua=KSR.pv.gete("$hdr(User-Agent)")
        if KSR.hdr.is_present("X-Call-Uuid") < 0 and not ('baresip' in ua or 'Beward' in ua):
            return
        try:
            key = "{}:{}".format(KSR.pv.getw("$ci"), time)
            value = {
                "time":time,
                "account":KSR.pv.getw("$fu"),
                "uuid":KSR.pv.gete("$hdr(X-Call-Uuid)"),
                "device_id":KSR.pv.gete("$hdr(X-Device-Id)"),
                "register_success":registred,
            }
            r = redis.StrictRedis(host='127.0.0.1', port=6379, db=13, decode_responses=True)
            r.hmset(key, value)
            r.expire(key, 43200)
        except Exception as e:
            KSR.xlog.xerr("Cant send SIP_REGISTER " + str(e))

    def make_ct(self):
        m = re.search('^<([^>]+)(.*)', KSR.pv.get("$ct"))
        if m:
            ct = m.group(1)
        else:
            ct = KSR.pv.getw("$ct")
        if not KSR.pv.is_null("$avp(received)"):
            _, received_host, received_port = KSR.pv.getw("$avp(received)").split(":")
            if not "{}:{}".format(received_host, received_port) in ct:
                ct = "{};alias={}~{}~1".format(ct, received_host, received_port)
                KSR.xlog.xnotice("Generated URI with alias '{}'".format(ct))
        return ct

    # Import section
    for filename in ROUTES:
        for route in ROUTES[filename]:
            exec("from {} import {}".format(filename, route))


import serial
import argparse
import time
from threading import Thread, Event, Lock
import random

# Argparser
parser = argparse.ArgumentParser(description="Simula controlador de maquina de ensayos pin on disk")
parser.add_argument('--port', action='store', help='Puerto al cual va a streamear', required=True, dest='port')
port = parser.parse_args().port

ser = serial.Serial(port, timeout=None ) # Puerto al que escucha el controlador
ser.write(('Controlador escuchando en puerto ' + port + '\n').encode())
exitEvent = Event()
lock = Lock()
commands = ['SEND', 'STAR', 'CONN', 'DCON', 'TMHM', 'TEST', 'PAUS', 'STOP', 'SPEE']
radios = [5, 6, 7]
vueltasT = 0.0
velocidades = [10.0,12.5,13.2,15.3,9.6,8.6,7.9]
#Comandos
command = ''
radio = 0
vueltas = 0

#Estado default
state = 'WAIT'
isConnected = False
isRunning = False
test = False

def listener():
    global command, radio, vueltas
    while True:
        read = ser.readline().decode().strip()
        if (read[0] == '<' and read[-1] == '>' and read[1:5] in commands):
            if (len(read) == 6):      #Comando regular, sin argumentos
                lock.acquire()
                command = read[1:5]
                lock.release()
                print(command)
            elif (read[1:5] == 'STAR'):
                lock.acquire()
                command = read[1:5]
                radio = int(read.split(',')[1])
                vueltas = int(read.split(',')[2].split('>')[0])
                lock.release()
                print(f'<STAR,{radio},{vueltas}>')
        read = ''

def resetVars():
    global vueltasT
    vueltasT = 0

def getTemp():
    ser.write('55\n'.encode())
    ser.write('25.5\n'.encode())

def main():
    while True:
        global state, command, vueltasT, isRunning, isConnected, test
        if (state == 'STAR'):
            if (not isRunning):
                resetVars()
                isRunning = True
            if (vueltasT >= vueltas):
                ser.write('-1\n'.encode())
                isRunning = False
                state = 'WAIT'
            if (command == 'PAUS'):
                ser.write('0\n'.encode())
                state = 'PAUS'
            elif (command == 'STOP'):
                state = 'STOP'
            elif (command == 'TMHM'):
                getTemp()
                command = ''
            elif (command == 'SEND'):
                ser.write((str(vueltasT) + '\n').encode())
                command = ''
            elif (command == 'SPEE'):
                ser.write((str(random.choice(velocidades)) + '\n').encode())
                command = ''
            vueltasT += 1.0

        elif (state == 'PAUS'):
            if (command == 'STAR'):
                state = 'STAR'
                ser.write('0\n'.encode())

            elif (command == 'STOP'):
                state = 'STOP'

        elif (state == 'STOP'):
            print('Detenido')
            ser.write('-1\n'.encode())
            isRunning = False
            state = 'WAIT'

        elif (state == 'CONN'):
            print('Conectado')
            ser.write('0\n'.encode())
            isConnected = True
            state = 'WAIT'

        elif (state == 'DCON'):
            print('Desconectado')
            ser.write('0\n'.encode())
            isConnected = False
            resetVars()
            state = 'WAIT'

        elif (state == 'WAIT'):

            if (command == 'CONN' and isConnected == False):
                state = 'CONN'
                command = ''

            elif (command == 'STAR' and isConnected and (radio in radios) and vueltas > 0):
                state = 'STAR'
                print('Empezando experimento')
                ser.write('0\n'.encode())
                command = ''

            elif (command == 'TEST' and isConnected):
                state = 'TEST'
                command = ''

            elif (command == 'STOP' and isConnected):
                state = 'STOP'
                command = ''

            elif(command == 'DCON' and isConnected):
                state = 'DCON'
                command = ''

        elif (state == 'TEST'):
            test = not test
            if (test):
                print('Comenzando test')
                ser.write('0\n'.encode())
                state = 'WAIT'
            else:
                print('Finalizando test')
                ser.write('-1\n'.encode())
                state = 'WAIT'

        time.sleep(0.1)

t1 = Thread(target=listener, daemon=True)
t2 = Thread(target=main, daemon=True)
t1.start()
t2.start()

while True:
    time.sleep(0.1)

t1.join()
t2.join()
print('Cerrando')


from base_player import BasePlayer
import networkx as nx
import operator
import random
import copy
import inspect
from enum import Enum
from math import ceil, floor

class PlacementPolicies(Enum):
    SINGLE = 1          # place only on a single node
    EVEN = 2            # place evenly across all minimum nodes

class MovementPolicies(Enum):
    AGGRESSIVE = 1      # attack all that can be won

class Player(BasePlayer):

    """
    You will implement this class for the competition.
    You can add any additional variables / methods in this file.
    Do not modify the class name or the base class and do not modify the lines marked below.
    """

    def __init__(self, p_id):
        super().__init__(p_id)  #Initializes the super class. Do not modify!

        """
        Insert player-specific initialization code here
        """

        self.place_policy = PlacementPolicies.SINGLE
        self.move_policy = MovementPolicies.AGGRESSIVE

        return

    """
    Called at the start of every placement phase and movement phase.
    """
    def init_turn(self, board, nodes, max_units):
        super().init_turn(board, nodes, max_units)       #Initializes turn-level state variables
        self.list_graph = sorted(list(self.board.nodes(data=True)))

        self.frontier = self.get_frontier()
        self.owned_frontier = self.get_outer()
        self.dist_from_frontier = self.distance_from_frontier()
        """
        Insert any player-specific turn initialization code here
        """
        return

    """
    Looks at the call stack to see who the caller is - can be useful debugging error messages
    """
    def find_caller(self):
        frame,filename,line_number,function_name,lines,index = inspect.stack()[2]
        # (filename, line_number, function_name, lines, index) = inspect.getframeinfo(previous_frame)
        print(function_name, ':', line_number)
        return

    """
    Run some basic checks and then call the parent function to add the move
    """
    def verify_and_place_unit(self, node, amount):
        if (self.list_graph[node] is None):
            raise Exception("Error: Node does not exist in list_graph")
            return

        if (self.list_graph[node][1]['owner'] != self.player_num):
            raise Exception("Error: You do not own this node you are placing into")
            self.find_caller()
            return

        if (amount <= 0):
            raise Exception("you are not placing any nodes...")
            return

        if (amount > self.max_units):
            raise Exception("Error: You are trying to place too many units")
            return

        super().place_unit(node, amount)
        return

    """
    Run some basic checks and then call the parent function to add the move
    """
    def verify_and_move_unit(self, start, end, amount):
        if (amount <= 0):
            raise Exception("you are not placing any nodes...")
            return

        start_node = self.list_graph[start]
        end_node = self.list_graph[end]

        if ((start is None) or (end is None)):
            raise Exception("Error: Node does not exist in list_graph")
            return

        if (start_node[1]['owner'] != self.player_num):
            raise Exception("Error: You do not own this node you are starting from")
            return

        if (start == end):
            raise Exception("start == end")
            return

        if (start_node[1]['old_units'] <= amount):
            raise Exception("You are requesting", amount, "units, but you only have ", start_node[1]['old_units'], 'units')
            self.find_caller()
            return

        super().move_unit(start, end, amount)
        return

    def get_frontier(self):
        frontier = []
        for node in self.nodes:
            for neighbor in self.board[node]:
                neighbor_id = self.board.nodes[neighbor]['owner']
                if neighbor_id != self.player_num and neighbor not in frontier:
                    frontier.append(neighbor)
        return frontier

    def get_outer(self):
        outer = []
        for node in self.nodes:
            for neighbor in self.board[node]:
                if self.board.nodes[neighbor]['owner'] != self.player_num:
                    outer.append(node)
                    break
        return outer

    """
    Called during the placement phase to request player moves
    """
    def player_place_units(self):
        """
        Insert player logic here to determine where to place your units
        """

        """
        Given nodes in the frontier, allocate to node with the fewest adjacent nodes
        that belong to others
        (most defensible)
        """

        frontier_nodes = []
        for node in self.owned_frontier:
            # count number of neighbors that do not belong to us
            n_neighbors = 0
            for neighbor in self.board.neighbors(node):
                if self.board.nodes[neighbor]['owner'] != self.player_num:
                    n_neighbors += 1
            frontier_nodes.append((node, n_neighbors))

        frontier_nodes = sorted(frontier_nodes, key=lambda node: node[1])
        node_to_place, n_neighbors = frontier_nodes[0]

        if self.place_policy == PlacementPolicies.SINGLE:
            self.verify_and_place_unit(node_to_place, self.max_units)
        elif self.place_policy == PlacementPolicies.EVEN:
            nodes = [x for x in filter(lambda x: x[1] == n_neighbors, frontier_nodes)]
            remaining_units = self.max_units
            units_per_node = floor(self.max_units / len(nodes))
            while remaining_units > 0:
                for node, _ in nodes:
                    to_place = min(remaining_units, units_per_node)
                    self.verify_and_place_unit(node, to_place)
                    remaining_units -= to_place
                    if remaining_units == 0:
                        break

        return self.dict_moves #Returns moves built up over the phase. Do not modify!

    def distance_from_frontier(self):
        frontier = self.get_frontier()
        distances = {}
        all_nodes = [n for n in self.nodes] + frontier
        for node in all_nodes:
            queue = []
            queue.append((node, 0))
            visited = set()
            while len(queue) > 0:
                cur, dist = queue[0]
                queue = queue[1:]
                if cur in visited:
                    continue
                visited.add(cur)
                if cur in frontier:
                    distances[node] = dist
                    break
                for nxt in self.board[cur]:
                    queue.append((nxt, dist + 1))
        return distances

    def execute_single_turn_actions(self):

        #move to frontier
        for nodes in self.nodes:
            if nodes in self.owned_frontier:
                continue #already is on outer..

            self_units = self.board.nodes[nodes]['old_units']
            if self_units <= 1:
                continue #no point in moving

            neighbors = self.board.neighbors(nodes)
            neighbors = filter(lambda x: self.board.nodes[x]['owner'] == self.player_num, neighbors)

            best_neighbor = min(neighbors, key = lambda x: self.dist_from_frontier[x])

            self.verify_and_move_unit(nodes, best_neighbor, self_units-1)

        #attacking
        for nodes in self.nodes:
            self_units = self.board.nodes[nodes]['old_units']

            if self_units <= 1:
                continue

            can_attack = []

            neighbors = self.board.neighbors(nodes)
            for n in neighbors:

                n_node = self.board.nodes[n]
                n_units = n_node['old_units']
                n_owner = n_node['owner']

                if (n_owner != self.player_num) and (self_units > n_units + 1):
                    can_attack.append((n, n_units +1))

            #do_attack = {}
            remaining_units = self_units
            can_attack = sorted(can_attack, key = lambda x: x[1])
            for i, (n, units) in enumerate(can_attack):
                if remaining_units - units < 1:
                    break

                used_units = units
                if i == len(can_attack) - 1 or \
                        can_attack[i][1] + can_attack[i + 1][1] >= remaining_units:
                    used_units = remaining_units - 1
                self.verify_and_move_unit(nodes, n, used_units)
                remaining_units -= used_units
        return

    """
    Called during the move phase to request player moves
    """
    def player_move_units(self):
        """
        Insert player logic here to determine where to move your units
        """

        self.execute_single_turn_actions()
        return self.dict_moves


import subprocess  # safest module for running shell commands through python
import sys  # only for exit
import os.path  # only for checking existence
import argparse

# Parse command line arguments
parser = argparse.ArgumentParser(
    description = "Simple parser for running flake8 on git repo"
    )
parser.add_argument("-r", "--repo",
                    help = "GitHub repository, as 'owner/repo'")
parser.add_argument("-p", "--path",
                    help = "Path where the git repo will be cloned")
parser.add_argument("-s", "--sha",
                    help = "SHA of the commit to checkout")
args = parser.parse_args()
# origin of git repo
repo_uri = "https://github.com/{}.git".format(args.repo)
# where the repo will be cloned
git_path = args.path
# SHA of the repo that needs to be checked out
git_sha = args.sha  # eg. "10213a290beafdae8518023cb2235086ebdf4506"
(owner, repo) = args.repo.split('/')

# mkdir using subprocess run
if os.path.exists(git_path):
    print("The git_path location already exists")
else:
    try:
        mkdir_process = subprocess.run(["mkdir", git_path], check=True)
    except subprocess.CalledProcessError:
        sys.exit("The directory could not be created. Script ending prematurely")
    print("Directory to clone git repo into created successfully")

# git clone
if os.path.exists(git_path + "/{}/.git".format(repo)):
    print("The repo has already been cloned at specified git_path")
else:
    try:
        clone_process = subprocess.run(["git", "clone", repo_uri], cwd=git_path, check=True)
    except subprocess.CalledProcessError:
        sys.exit("The git repository could not be cloned. Script ending prematurely")
    print("Repo cloned successfully")

# git checkout
try:
    checkout_process = subprocess.run(["git", "checkout", git_sha], cwd=git_path + "/{}".format(repo), check=True,
                                      stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    # for capturing output and redirecting stderr to stdout
except subprocess.CalledProcessError:
    sys.exit("The checkout could not be performed. Script ending prematurely")
print(checkout_process.stdout.decode("utf-8"))

# running flake8
try:
    flake8_process = subprocess.run(["flake8", "-v"], cwd=git_path + "/{}".format(repo),
                                    stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
except (OSError, ValueError, subprocess.CalledProcessError) as e:
    sys.exit(e+"Could not run flake8. Exiting")

if flake8_process.returncode == 0:
    print("OK") # This is printed if there are no errors and flake8 ends successfully
print("The list of violations/errors found by flake8 are:\n", flake8_process.stdout.decode("utf-8"), sep='')


from django.contrib.auth.models import User
from django.contrib.sites.shortcuts import get_current_site
from django.contrib.sites.models import Site
from django.core.mail import send_mail
from django.core.urlresolvers import reverse
from django.db.models.signals import post_save, post_delete
from django.dispatch import receiver
from django.conf import global_settings as settings
from . import models

@receiver(post_save, sender=User, dispatch_uid='blog: make blog')
def make_blog(sender, **kwargs):
    if kwargs['created']:
        new_blog = models.Blog()
        new_blog.owner = kwargs['instance']
        new_blog.save()

def notify(post, subject, text):
    adr_from = settings.DEFAULT_FROM_EMAIL
    subscribers = post.blog.subscribers
    for user in subscribers:
        if user.email:
            send_mail(subject, text, adr_from, [user.email], fail_silently=False)

@receiver(post_save, sender=models.Post, dispatch_uid='blog: create post')
def create_post(sender, **kwargs):
    if kwargs['created']:
        site = Site.objects.get_current()
        post = kwargs['instance']
        url = reverse('blog-post', kwargs={'id': post.id})
        full_url = ''.join(['http://', site.domain, url])
        notify(post, 'New post for you', full_url)

@receiver(post_delete, sender=models.Post, dispatch_uid='blog: delete post')
def delete_post(sender, **kwargs):
    post = kwargs['instance']
    notify(post, 'Post was deleted', 'Post "{}" was deleted from "{}"'.format(post.title, post.blog))


#We are using the Gradient Boosting Classifier
#Combinations of Logistics Regressions and Gradient Booster ...
#Not sure, how the results will be in the submitted kaggle file.

import numpy as np
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression

data = np.load('train.npy');
data_test = np.load('test_distribute.npy')
Y_train = data[:, 0]
X_train = data[:, 1:]
X_test = data_test[:, 1:]

std_scaler = MinMaxScaler()
X_train = std_scaler.fit_transform(X_train)

clf = RandomForestClassifier(n_estimators=100,max_depth=None)
clf.fit(X_train, Y_train)
X_test = std_scaler.fit_transform(X_test)
Y_pred = clf.predict(X_test)

fout = open('rahul_ram-beats_random_forest_100est_noneMaxDepth.kaggle', 'w')
fout.write("ID,Category\n");
for i in range(Y_pred.shape[0]):
      fout.write("\n" + str(i+1) + "," + str(int(Y_pred[i]))) ## str(int(Y_pred[i]
## now make kaggle submission Y_pred
fout.close()


"""
Utilities for Python version compatibility
"""
# Please organize additions to this file by version

import sys

if sys.version_info < (3, 9):
    # https://docs.python.org/3/library/asyncio-task.html#asyncio.to_thread

    import asyncio
    import functools

    async def asyncio_to_thread(fn, *args, **kwargs):
        loop = asyncio.get_event_loop()
        return await loop.run_in_executor(None, functools.partial(fn, *args, **kwargs))

else:
    from asyncio import to_thread as asyncio_to_thread

if sys.version_info < (3, 8):
    # https://docs.python.org/3/library/asyncio-policy.html#asyncio.ThreadedChildWatcher
    # `ThreadedChildWatcher` is the default child process watcher for Python 3.8+ but it
    # does not exist in Python 3.7. This backport allows us to ensure consistent
    # behavior when spawning async child processes without dropping support for 3.7.

    import asyncio
    import itertools
    import logging
    import os
    import threading
    import time
    import warnings

    logger = logging.getLogger()

    # Python 3.7 preferred API
    _get_running_loop = getattr(asyncio, "get_running_loop", asyncio.get_event_loop)

    class ThreadedChildWatcher(asyncio.AbstractChildWatcher):
        def __init__(self):
            self._pid_counter = itertools.count(0)
            self._threads = {}

        def is_active(self):
            return True

        def close(self):
            pass

        def __enter__(self):
            return self

        def __exit__(self, exc_type, exc_val, exc_tb):
            pass

        def __del__(self, _warn=warnings.warn):
            threads = [t for t in list(self._threads.values()) if t.is_alive()]
            if threads:
                _warn(
                    f"{self.__class__} has registered but not finished child processes",
                    ResourceWarning,
                    source=self,
                )

        def add_child_handler(self, pid, callback, *args):
            loop = _get_running_loop()
            thread = threading.Thread(
                target=self._do_waitpid,
                name=f"waitpid-{next(self._pid_counter)}",
                args=(loop, pid, callback, args),
                daemon=True,
            )
            self._threads[pid] = thread
            thread.start()

        def remove_child_handler(self, pid):
            # asyncio never calls remove_child_handler() !!!
            # The method is no-op but is implemented because
            # abstract base class requires it
            return True

        def attach_loop(self, loop):
            pass

        def _do_waitpid(self, loop, expected_pid, callback, args):
            assert expected_pid > 0

            try:
                pid, status = os.waitpid(expected_pid, 0)
            except ChildProcessError:
                # The child process is already reaped
                # (may happen if waitpid() is called elsewhere).
                pid = expected_pid
                returncode = 255
                logger.warning(
                    "Unknown child process pid %d, will report returncode 255", pid
                )
            else:
                if os.WIFSIGNALED(status):
                    returncode = -os.WTERMSIG(status)
                elif os.WIFEXITED(status):
                    returncode = os.WEXITSTATUS(status)
                else:
                    returncode = status

                if loop.get_debug():
                    logger.debug(
                        "process %s exited with returncode %s", expected_pid, returncode
                    )

            if loop.is_closed():
                logger.warning("Loop %r that handles pid %r is closed", loop, pid)
            else:
                loop.call_soon_threadsafe(callback, pid, returncode, *args)

            self._threads.pop(expected_pid)

else:
    from asyncio import ThreadedChildWatcher


import logging
import os

from google.cloud import storage

def parse_uri_to_bucket_and_filename(file_path):
    """Divides file uri to bucket name and file name"""
    path_parts = file_path.split("//")
    if len(path_parts) >= 2:
        main_part = path_parts[1]
        if "/" in main_part:
            divide_index = main_part.index("/")
            bucket_name = main_part[:divide_index]
            file_name = main_part[divide_index + 1 - len(main_part):]

            return bucket_name, file_name
    return "", ""

def from_gcs_to_local_file(src_gcs_bucket, src_gcs_name, local_file_path):
    storage_client = storage.Client(os.environ['PROJECT_ID'])
    # Create a bucket object for our bucket
    bucket = storage_client.get_bucket(src_gcs_bucket)
    # Create a blob object from the filepath
    blob = bucket.blob(src_gcs_name)
    # Download the file to a destination
    logging.info("Downloading gs://{}/{} to {}...".format(src_gcs_bucket, src_gcs_name, local_file_path))
    blob.download_to_filename(local_file_path)
    logging.info("Successfully downloaded gs://{}/{} to {}".format(src_gcs_bucket, src_gcs_name, local_file_path))

def is_gcs_blob_exists(bucket, blob_name):
    storage_client = storage.Client(os.environ['PROJECT_ID'])
    # Create a bucket object for our bucket
    bucket = storage_client.get_bucket(bucket)
    # Create a blob object from the filepath
    blob = bucket.blob(blob_name)
    return blob.exists()

def upload_file_to_gcs(filename, destination_bucket_name, destination_blob_name):
    """
    Uploads a file to a given Cloud Storage bucket and returns the public url
    to the new object.
    """
    bucket = storage.Client().bucket(destination_bucket_name)
    blob = bucket.blob(destination_blob_name)
    logging.info("Uploading of {} to gs://{}/{}...".format(filename, destination_bucket_name, destination_blob_name))
    blob.upload_from_filename(
        filename,
        content_type="text/plain")
    logging.info(
        "Finished uploading of {} to gs://{}/{}".format(filename, destination_bucket_name, destination_blob_name))


from os import walk, sep
from os.path import basename, normpath, join

from peewee import DoesNotExist

from logic.identifier import add_identifiers
from logic.reserved import add_reserveds, get_reserved_by_name, \
    reserved_prefixes

def file_gen(from_dir, do_recursive=True, file_name=None):
    """Process a file or directory.

    Parameters
    ----------
    do_recursive : bool
    file_name : str
    from_dir : str

    Yields
    ------
    dir_name : str
    do_obfuscate : bool
    file_name : str
    is_reserved_dir : bool
    is_reserved_file : bool
    """
    # Process for a single file
    if file_name:
        # Skip excluded files
        is_reserved_dir = None
        try:
            get_reserved_by_name(''.join([
                reserved_prefixes.reserved_file, file_name]))
        except DoesNotExist:
            is_reserved_file = False
        else:
            is_reserved_file = True

        yield file_name, from_dir, is_reserved_dir, is_reserved_file
        return

    # Process for a directory
    for dir_name, subdir_list, file_list in walk(from_dir):
        if do_recursive and not dir_name:
            break
        # Remove unwanted files from file_list
        file_list = [file_name for file_name in file_list
                     if file_name[-4:] not in ['.pyc', '.pyo']
                     ]
        #####################
        # Process directories
        #####################
        is_reserved_dir = True
        do_obfuscate_dir = False

        # Identify reserved directories
        try:
            reserved_row = get_reserved_by_name(
                ''.join([reserved_prefixes.reserved_dir, basename(dir_name)]))
        except DoesNotExist:
            is_reserved_dir = False
        else:
            # Add identifiers to reserved
            if subdir_list:
                add_reserveds(reserved_row.primary_package, subdir_list,
                              reserved_prefixes.reserved_dir)
            if file_list:
                add_reserveds(basename(dir_name), file_list,
                              reserved_prefixes.reserved_file)

        # Identify non-obfuscated directories
        if not is_reserved_dir:
            try:
                get_reserved_by_name(''.join([
                    reserved_prefixes.non_obfuscated_dir, basename(dir_name)]))
            except DoesNotExist:
                do_obfuscate_dir = True

                # Add to obfuscated identifiers
                add_identifiers(basename(dir_name))
                if subdir_list:
                    add_identifiers(subdir_list)
                if file_list:
                    ob_files = [
                        file_name[:-3] for file_name in file_list
                        if file_name[-3:] in ['.py', '.ky']
                        ]
                    if ob_files:
                        add_identifiers(ob_files)
            else:
                # Add non-obfuscated identifiers
                add_identifiers([basename(dir_name)], do_obfuscate=False)
                if subdir_list:
                    add_reserveds(basename(dir_name), subdir_list,
                                  reserved_prefixes.non_obfuscated_dir)
                if file_list:
                    add_reserveds(basename(dir_name), file_list,
                                  reserved_prefixes.non_obfuscated_file)

        for file_name in file_list:
            do_obfuscate = False
            is_reserved_file = False
            # Files in reserved directory are reserved
            if is_reserved_dir:
                is_reserved_file = True
            else:
                # Identify reserved files
                try:
                    get_reserved_by_name(''.join([
                        reserved_prefixes.reserved_file, file_name]))
                except DoesNotExist:
                    # Identify files to obfuscate
                    if do_obfuscate_dir:
                        try:
                            get_reserved_by_name(''.join([
                                reserved_prefixes.non_obfuscated_file,
                                file_name]))
                        except DoesNotExist:
                            do_obfuscate = True
                else:
                    is_reserved_file = True

            if file_name == '__init__.py' and \
                    not is_reserved_dir and \
                    not do_obfuscate_dir:
                do_obfuscate = True

            yield file_name, dir_name, is_reserved_dir, is_reserved_file, \
                do_obfuscate

def obfuscate_path(file_path):
    """Create an obfuscated path name.

    Parameters
    ----------
    file_path : str

    Returns
    -------
    obfuscated path : str
    """
    from logic.identifier import get_obfuscated_name

    obf_path = [get_obfuscated_name(p) for
                p in normpath(file_path).split(sep)]

    if obf_path[-1][-3:] in ['.py', '.kv']:
        obf_path[-1] = ''.join([get_obfuscated_name(obf_path[-1][:-3]),
                                obf_path[-1][-3:]])

    return join(*obf_path)

def split_path(file_path):
    """Split a path into its parts.

    Parameters
    ----------
    file_path : str

    Returns
    -------
    path : list
    """
    return normpath(file_path).split(sep)

def to_str(unicode_or_str):
    """Convert a string to unicode.

    Parameters
    ----------
    unicode_or_str : unicode  # or str

    Returns
    -------
    value : str
    """
    if isinstance(unicode_or_str, unicode):
        value = unicode_or_str.encode('utf-8')
    else:
        value = unicode_or_str
    return value

def to_unicode(unicode_or_string):
    """Convert a string to unicode.

    Parameters
    ----------
    unicode_or_string : str  # or unicode

    Returns
    -------
    value : unicode
    """
    if isinstance(unicode_or_string, str):
        value = unicode_or_string.decode('utf-8')
    else:
        value = unicode_or_string
    return value


import pandas as pd
import yaml
from pylab import *

import AsteroidData as Ad

pd.set_option('display.width', 1000)

with open('NASA_API_KEY.yaml', 'rb') as open_yaml:
    NASA_API_SECRET = yaml.load(open_yaml)['NASA_API_KEY']

requestURL = 'https://api.nasa.gov/neo/rest/v1/feed?api_key={}'.format(NASA_API_SECRET)

asteroids = Ad.AsteroidData(requestURL)
asteroids.get_data()
asteroids.clean_data()
df = asteroids.df

# Plot data
y_axis = df['miss_distance.lunar'].tolist()
x_axis = df['days_from_now'].tolist()
area = df[Ad.SIZE_COL].tolist()
colors = df['relative_velocity.kilometers_per_hour'].tolist()

fig = plt.figure(num='Near-Earth Asteroids this Week')
fig.add_subplot(1, 1, 1, axisbg='black')
c = scatter(x_axis, y_axis, c=colors, s=area, cmap='Reds_r')
kilo_colors = plt.colorbar(c, orientation='horizontal')

## Labels
title('Near-Earth Asteroids this Week')
plt.xlabel('Days until close approach')
plt.ylabel('Min. Est. Lunar Distances of Close Approach')
kilo_colors.set_label('Kilometers Per Hour')

## Area Legend
legend_suffix = ' Est. min. ø m'
minSize, avgSize, maxSize = [int(round(calc(df[Ad.SIZE_COL]), 0)) for calc in [min, mean, max]]
minCol, avgCol, maxCol = [plt.scatter([], [], s=num, marker='o', color='#000000') for num in [minSize, avgSize, maxSize]]
minText, avgText, maxText = [str(num) + legend_suffix for num in [minSize, avgSize, maxSize]]

size_legend = plt.legend((minCol, avgCol, maxCol),
                         (minText, avgText, maxText),
                         bbox_to_anchor=(1.34, .5),
                         scatterpoints=1,
                         loc='right',
                         ncol=1,
                         fontsize='medium',
                         labelspacing=4,
                         handletextpad=3)

plt.subplots_adjust(left=.12,
                    bottom=.11,
                    right=.72,
                    top=.88,
                    wspace=.2,
                    hspace=.2)

plt.show()


import torch.nn as nn
import torch
import pennylane as qml
from pennylane import numpy as np
# import pennylane_qulacs

from networks.backbones.custom_layers.QuanvLayer import Quanv

class QuanvNet(nn.Module):
    def __init__(self):
        super(QuanvNet, self).__init__()
        # Conv Layers
        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
        self.conv2 = nn.Conv2d(10, 20, kernel_size=3)
        self.conv3 = nn.Conv2d(20, 5, kernel_size=1)

        self.convALT = nn.Conv2d(5, 4, kernel_size=2) # Acts like the Quanv Layer

        # Quanvolution Layer
        self.Quanv1 = Quanv(kernal_size=3, output_depth=4, circuit_layers=1)
        self.AdaptPool = nn.AdaptiveMaxPool2d(3)

        # Fully Connected Layers
        self.fc1 = nn.Linear(36, 12)
        self.fc2 = nn.Linear(12, 3)

        self.pool = nn.MaxPool2d(2)
        self.relu = nn.ReLU()
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        x = self.pool(self.relu(self.conv1(x)))
        x = self.pool(self.relu(self.conv2(x)))
        x = self.relu(self.conv3(x))
        # x = self.sigmoid(self.convALT(x))
        x = self.sigmoid(self.Quanv1(x))
        x = self.AdaptPool(x)
        x = x.view(-1, 36)
        x = self.relu(self.fc1(x))
        x = self.fc2(x)
        return x


# -*- coding: utf-8 -*-

import os
import sys
import sqlite3
from Bio import SeqIO
from Bio.Alphabet import IUPAC
import pandas
import glob

DB_PATH = "/Volumes/Groups/workgroup/1002_project/SQL/"
DB_NAME = "fasta.db"
ORGANISM = "SACE"

def do_correspondance_dict(data = "/Users/dputil/Projets/1002/1002_strains_total_dataset.xlsx"):

    correspondance_dict = {}
    strains_infos = pandas.read_excel(data, 'included_dataset', index_col=None, na_values=['NA'])
    strain_name = strains_infos["Strain_Name"]
    strain_id = strains_infos["Standardized_name"]
    yjs_name = strains_infos["YJS_name"]

    for id, yjs in zip(strain_id, yjs_name):
        correspondance_dict[str(id).upper()] = str(yjs).upper()

    return correspondance_dict

def populate_db_with(fasta_files, correspondance):

    conn = sqlite3.connect(DB_PATH + DB_NAME)
    c = conn.cursor()
    c.execute("DROP TABLE IF EXISTS fasta")
    c.execute('''CREATE TABLE fasta(
    ID  INTEGER PRIMARY KEY AUTOINCREMENT,
    GENOSCOPE_ID    TEXT,
    STRAIN  TEXT,
    GENE_ID   TEXT,
    GENE_NAME   TEXT,
    SEQUENCE TEXT,
    ORGANISM    TEXT,
    CHROMOSOME  TEXT,
    START   INT,
    END INT,
    STRAND CHAR(1)
    )''')

    count = len(fasta_files)

    for fasta in fasta_files:
        count -= 1
        for record in SeqIO.parse(fasta, "fasta", alphabet=IUPAC.ambiguous_dna):

            gene_infos, chromo_infos, strand = record.description.strip().split("\t")
            strain = gene_infos.split("_")[:-2]

            if len(strain) == 1:
                strain = strain[0]
            else:
                strain = strain[0] + "_" + strain[1]

            gene_id, gene_name = gene_infos.split("_")[-2:]
            strand = strand[0]
            chromosome, positions = chromo_infos.split(":")
            start, end = positions.split("-")

            if strain not in correspondance.keys():
                pass
            else:

                data_for_db = (strain, correspondance[strain], gene_id, gene_name, str(record.seq), ORGANISM, chromosome, start, end, strand)

                c.execute("INSERT INTO fasta (GENOSCOPE_ID, STRAIN, GENE_ID, GENE_NAME, SEQUENCE, ORGANISM, CHROMOSOME, START, END, STRAND)  VALUES (?,?,?,?,?,?,?,?,?,?)", data_for_db)

        print("Data load for ", gene_id, count)

    conn.commit()
    conn.close()

if __name__ == "__main__":

    correspondance = do_correspondance_dict()

    fasta_files = glob.glob("/Users/dputil/Projets/1002/VARIANTS_INFERENCE/*.fasta")

    print(len(fasta_files))
    populate_db_with(fasta_files, correspondance)


import logging
import re
from datetime import datetime, timezone, timedelta
import pandas

from jal.widgets.helpers import g_tr
from jal.db.update import JalDB
from jal.constants import Setup, DividendSubtype, PredefinedCategory

# -----------------------------------------------------------------------------------------------------------------------
class KITFinance:
    Header = "КИТ Финанс (АО)"
    AccountPattern = "(?P<ACCOUNT>.*)-(.*)"
    PeriodPattern = "(?P<S>\d\d\.\d\d\.\d\d\d\d) - (?P<E>\d\d\.\d\d\.\d\d\d\d)"

    def __init__(self, parent, filename):
        self._parent = parent
        self._filename = filename
        self._statement = None
        self._account_id = 0
        self._settled_cash = {}

    def load(self):
        self._statement = pandas.read_excel(self._filename, header=None, na_filter=False)
        if not self.validate():
            return False
        self.load_cash_balance()
        self.load_stock_deals()
        self.load_cash_transactions()
        logging.info(g_tr('KIT', "KIT Finance statement loaded; Planned cash: ")
                     + f"{self._settled_cash[self._account_id]}")
        return True

    def validate(self):
        if self._statement[4][0] != self.Header:
            logging.error(g_tr('KIT', "Can't find KIT Finance report header"))
            return False
        parts = re.match(self.AccountPattern, self._statement[5][5], re.IGNORECASE)
        if parts is None:
            logging.error(g_tr('KIT', "Can't parse KIT Finance account number"))
            return False
        account_name = parts.groupdict()['ACCOUNT']
        parts = re.match(self.PeriodPattern, self._statement[5][8], re.IGNORECASE)
        if parts is None:
            logging.error(g_tr('KIT', "Can't parse KIT Finance statement period"))
            return False
        statement_dates = parts.groupdict()
        report_start = int(datetime.strptime(statement_dates['S'], "%d.%m.%Y").replace(tzinfo=timezone.utc).timestamp())
        if not self._parent.checkStatementPeriod(account_name, report_start):
            return False
        logging.info(g_tr('KIT', "Loading KIT Finance statement for account ") +
                     f"{account_name}: {statement_dates['S']} - {statement_dates['E']}")
        self._account_id = JalDB().get_account_id(account_name)
        return True

    def find_section_start(self, header, columns) -> (int, dict):
        start_row = -1
        headers = {}
        for i, row in self._statement.iterrows():
            if row[0] == header:
                start_row = i + 1  # points to columns header row
                break
        if start_row > 0:
            for col in range(self._statement.shape[1]):  # Load section headers from next row
                headers[self._statement[col][start_row]] = col
        column_indices = {column: headers.get(columns[column], -1) for column in columns}
        if start_row > 0:
            for idx in column_indices:
                if column_indices[idx] < 0:
                    logging.error(g_tr('KIT', "Column not found in section ") + f"{header}: {idx}")
                    start_row = -1
        start_row += 1
        return start_row, column_indices

    def load_cash_balance(self):
        row, headers = self.find_section_start("Состояние денежных средств на счете", {})
        if row < 0:
            return False
        while row < self._statement.shape[0]:
            if self._statement[0][row] == '':
                break
            if self._statement[0][row] == 'Позиция по ДС на T+N':
                try:
                    self._settled_cash[self._account_id] = float(self._statement[6][row])
                except (ValueError, TypeError):
                    pass
            row += 1

    def load_stock_deals(self):
        cnt = 0
        columns = {
            "number": "Номер\nсделки",
            "date": "Дата сделки",
            "time": "Время сделки",
            "asset": "Наименование\nЦБ",
            "isin": "ISIN",
            "B/S": "Тип операции",
            "price": "Цена сделки ",
            "qty": "Количество",
            "amount": "Сумма сделки",
            "accrued_int": " НКД",
            "settlement": "Дата поставки\n(план.)",
            "fee_ex": "Комиссия\nТС",
            "fee_broker": "Комиссия\nброкера"
        }

        row, headers = self.find_section_start("Заключенные сделки с ценными бумагами", columns)
        if row < 0:
            return False
        while row < self._statement.shape[0]:
            if self._statement[0][row] == '' and self._statement[0][row + 1] == '':
                break
            asset_id = JalDB().get_asset_id('', isin=self._statement[headers['isin']][row],
                                                name=self._statement[headers['asset']][row])
            if self._statement[headers['B/S']][row] == 'Покупка':
                amount = -self._statement[headers['amount']][row]
                qty = self._statement[headers['qty']][row]
                bond_interest = -self._statement[headers['accrued_int']][row]
            elif self._statement[headers['B/S']][row] == 'Продажа':
                amount = self._statement[headers['amount']][row]
                qty = -self._statement[headers['qty']][row]
                bond_interest = self._statement[headers['accrued_int']][row]
            else:
                row += 1
                logging.warning(g_tr('KIT', "Unknown trade type: ") + self._statement[headers['B/S']][row])
                continue
            price = self._statement[headers['price']][row]
            fee = round(abs(self._statement[headers['fee_ex']][row] + self._statement[headers['fee_broker']][row]), 8)
            if abs(abs(price * qty) - amount) >= Setup.DISP_TOLERANCE:
                price = abs(amount / qty)
            number = self._statement[headers['number']][row]
            # Dates are loaded as datetime objects but time is loaded as string
            t_date = self._statement[headers['date']][row]
            t_time = datetime.strptime(self._statement[headers['time']][row], "%H:%M:%S").time()
            trade_datetime = t_date + timedelta(hours=t_time.hour, minutes=t_time.minute, seconds=t_time.second)
            timestamp = int(trade_datetime.replace(tzinfo=timezone.utc).timestamp())
            settlement = int(self._statement[headers['settlement']][row].replace(tzinfo=timezone.utc).timestamp())
            JalDB().add_trade(self._account_id, asset_id, timestamp, settlement, number, qty, price, -fee)
            if bond_interest != 0:
                JalDB().add_dividend(DividendSubtype.BondInterest, timestamp, self._account_id, asset_id,
                                     bond_interest, "НКД", number)
            cnt += 1
            row += 1
        logging.info(g_tr('KIT', "Trades loaded: ") + f"{cnt}")

    def load_cash_transactions(self):
        cnt = 0
        columns = {
            "date": "Дата",
            "operation": "Тип операции",
            "amount": "Сумма",
            "currency": " Валюта",
            "reason": "Основание",
            "note": "Примечание",
            "market": "Секция"
        }
        operations = {
            'Внесение д/с в торг': self.transfer_in,
            'Вывод дс': self.transfer_out,
            'Ком бр аб плата спот': self.fee,
            'Комиссия НРД': self.fee
        }
        row, headers = self.find_section_start("Движение денежных средств по неторговым операциям", columns)
        if row < 0:
            return False
        while row < self._statement.shape[0]:
            if self._statement[0][row] == '':
                break
            operation = self._statement[headers['operation']][row]
            if operation not in operations:  # not supported type of operation
                row += 1
                continue
            timestamp = int(self._statement[headers['date']][row].replace(tzinfo=timezone.utc).timestamp())
            amount = self._statement[headers['amount']][row]
            reason = self._statement[headers['reason']][row]
            description = self._statement[headers['note']][row]
            operations[operation](timestamp, amount, reason, description)
            cnt += 1
            row += 1
        logging.info(g_tr('KIT', "Cash operations loaded: ") + f"{cnt}")

    def transfer_in(self, timestamp, amount, reason, note):
        currency_name = JalDB().get_asset_name(JalDB().get_account_currency(self._account_id))
        text = g_tr('KIT', "Deposit of ") + f"{amount:.2f} {currency_name} " + \
               f"@{datetime.utcfromtimestamp(timestamp).strftime('%d.%m.%Y')}\n" + \
               g_tr('KIT', "Select account to withdraw from:")
        pair_account = self._parent.selectAccount(text, self._account_id)
        if pair_account == 0:
            return
        description = reason + ", " + note
        JalDB().add_transfer(timestamp, pair_account, amount, self._account_id, amount, 0, 0, description)

    def transfer_out(self, timestamp, amount, reason, note):
        currency_name = JalDB().get_asset_name(JalDB().get_account_currency(self._account_id))
        text = g_tr('KIT', "Withdrawal of ") + f"{-amount:.2f} {currency_name} " + \
               f"@{datetime.utcfromtimestamp(timestamp).strftime('%d.%m.%Y')}\n" + \
               g_tr('KIT', "Select account to deposit to:")
        pair_account = self._parent.selectAccount(text, self._account_id)
        if pair_account == 0:
            return
        description = reason + ", " + note                # amount is negative in XLSX file
        JalDB().add_transfer(timestamp, self._account_id, -amount, pair_account, -amount, 0, 0, description)

    def fee(self, timestamp, amount, _reason, description):
        JalDB().add_cash_transaction(self._account_id, self._parent.getAccountBank(self._account_id),
                                     timestamp, amount, PredefinedCategory.Fees, description)