"""
镖局接口
"""
from lxml import html

from .base import MBApiBase
from .constant import (
    BIAOJU_API
)
from .config import COMMON_SHIPPING_FEE_ID, SPECIAL_SHIPPING_FEE_ID
from .exceptions import CalculateShippingFeeError

class BiaoJuApi(MBApiBase):
    def get_shipping_fee(self, shipping_fee_id, weight, country="US", postal_code=""):
        """获取单个国家物流价格
        :param shipping_fee_id: 物流自定义费用id
        :param weight: 重量
        :param country: 国家
        :param postal_code: 邮政编码
        """
        api = BIAOJU_API
        params = {
            "m": "customshippingfee",
            "a": "doCalculate",
        }
        data = {
            "data": "||".join([country, str(weight), str(postal_code)]),
            "ruleId": shipping_fee_id,
            "type": 1,
        }
        r_data = self.request('post', api, data=data, params=params)
        tree = html.fromstring(r_data["calculationRetHtml"])
        try:
            return float(tree.xpath("//tr[1]/td[4]/span[1]/text()")[0])
        except IndexError:
            log = (
                f"计算物流出错，请核对马帮物流自定义费用设置。"
                f"物流费用规则ID: {shipping_fee_id}, 重量: {weight}, 国家: {country}"
            )
            raise CalculateShippingFeeError(log)


"""
byceps.services.ticketing.category_service
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:Copyright: 2006-2018 Jochen Kupperschmidt
:License: Modified BSD, see LICENSE for details.
"""

from typing import Optional, Sequence

from ...database import db
from ...typing import PartyID

from .models.category import Category
from .transfer.models import TicketCategoryID

def create_category(party_id: PartyID, title: str) -> Category:
    """Create a category."""
    category = Category(party_id, title)

    db.session.add(category)
    db.session.commit()

    return category

def count_categories_for_party(party_id: PartyID) -> int:
    """Return the number of categories for that party."""
    return Category.query \
        .for_party(party_id) \
        .count()

def find_category(category_id: TicketCategoryID) -> Optional[Category]:
    """Return the category with that ID, or `None` if not found."""
    return Category.query.get(category_id)

def get_categories_for_party(party_id: PartyID) -> Sequence[Category]:
    """Return all categories for that party."""
    return Category.query \
        .for_party(party_id) \
        .all()


import math
from scipy.stats.distributions import binom
n = 88
A = []
p = 0.5

def INDC(n,k,p):
    bc = binom.cdf(k,n,p)
    return math.log(bc,10)

def result(n):
    result = []
    for k in range(2*n):
        result.append(INDC(2*n,k,0.5))
    return sorted(result,reverse=True)

for i in result(n):
    if '%.3f' % i == "-0.000":
        print("0.000", end = " ")
    else:
        print('%.3f' % i, end = " ")


# Purpose: The purpose of this script is to complete the preprocessing phase using the spacy module.
# Steps Completed can be found in the following link:
# https://towardsdatascience.com/setting-up-text-preprocessing-pipeline-using-scikit-learn-and-spacy-e09b9b76758f
# Multiclass text classification: https://towardsdatascience.com/multi-class-text-classification-with-scikit-learn-12f1e60e0a9f

# TODO Get more data. Changes to the model show that more data will improve accuracy.
# TODO Complete text preprocessing! We are using some bad examples!.

import json
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_selection import chi2
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import LinearSVC
from sklearn.model_selection import cross_val_score

def read_json_data(file="./StepThree(data).json"):
    with open(file, "r") as link_file:
        data = json.load(link_file)

    return data

def convert_catagories_to_int(data):
    dictionaries = data['app']
    temp = []

    for dict in dictionaries:

        values_list = {"Art and Design": 1,"Augmented Reality": 2, "Auto & Vehicles": 3, "Beauty": 4,
                       "Books and Reference": 5, "Business":6, "Comics": 7, "Communication":8,
                       "Dating":9, "Daydream": 10, "Education": 11, "Entertainment": 12, "Events": 13,
                       "Finance": 14, "Food & Drink": 15, "Health & Fitness": 16, "House & Home": 17,
                       "Libraries & Demo ": 18, "Lifestyle": 19, "Maps & Navigation":20, "Medical": 21,
                       "Music and Audio": 22, "Games ": 23}

        for item in values_list:
            if list(dict.values())[3] == item:
                dict["category"] = values_list[item]

    return dictionaries

# Read the data from the StepThree(data).json data file
data=read_json_data()

# convert the categories from text to numeric values --> Dic with number categories.
# numeric_data = convert_catagories_to_int(data)

# Convert the dictionary format data to a pandas dataframe.
df = pd.DataFrame(data["app"])

# We will only use appTitle, mini_description and category.
# The following code reduces the data to only the above three columns.
col = ['category', 'app_mini_description', 'appTitle']
df = df[col]

# Set the names of the columns and then add a column
# that creates unique category ids.
df.columns = col
df['category_id'] = df['category'].factorize()[0]

# General sorting. Variables to be used later in code.
category_id_df = df[['category', 'category_id']].drop_duplicates().sort_values('category_id')
category_to_id = dict(category_id_df.values)
id_to_category = dict(category_id_df[['category_id', 'category']].values)

# We should evaluate the distribution of apps. The following code shows the
# distribution of apps by category with the frequency of mini-descriptions.
fig = plt.figure(figsize=(8,6))
df.groupby('category').app_mini_description.count().plot.bar(ylim=0)
#plt.show()

# Create a tdif vectorizer with the following attributes.
tfidf = TfidfVectorizer(sublinear_tf=True, min_df=3, norm='l2', encoding='latin-1', ngram_range=(1,1), stop_words='english')

# fit the tfdidf model to the app_mini_descriptions
features = tfidf.fit_transform(df["app_mini_description"]).toarray()

labels = df['category_id']

# Each of the n app_mini_descriptions are now represented by m features which
# representing the tf-idf core for different uni-grams and bi-grams
print(features.shape)

# Now, let us use chi2 to find terms that are most correlated with each
# category

N = 20
for category, category_id in sorted(category_to_id.items()):
    features_chi2 = chi2(features,labels==category_id)
    indicies = np.argsort(features_chi2[0])
    feature_names = np.array(tfidf.get_feature_names())[indicies]
    unigrams = [v for v in feature_names if len(v.split(' '))==1]
    print("#{}".format(category))
    print(" .Most correlated unigrams:\n{}".format('\n'.join(unigrams[-N:])))

# In prelimiary results, it is clear that unigram model
# is more representitive than bigram models.
# This shows that more often words in app descriptions appear alone
# More data may prove to change this conclusion.

# we will now develop a classifier. To use the classifier,
# we will need to vectorize our app_mini_descriptions.
# We will also need to determine what ML model we will choose.
# Naive Bayes is particularly suited for word counts
# We will text more ml models towards the end of this module

X_train, X_test, y_train, y_test = train_test_split(df["app_mini_description"], df["category"], random_state=0)
count_vec = CountVectorizer()
X_train_counts = count_vec.fit_transform(X_train)
tfidf_transformer = TfidfTransformer()
X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)

clf = MultinomialNB().fit(X_train_tfidf, y_train)

# Let us now evaluate the results of our models and compare them to the following models:
# Logistic Regression
# (Multinomial Naive-Bayes)
# Linear Support Vector Machine
# Random Forest

models = [LogisticRegression(), MultinomialNB(), LinearSVC(), RandomForestClassifier(n_estimators=200, max_depth=3, random_state=0)]

CV = 5

cv_df = pd.DataFrame(index=range(CV*len(models)))
entries=[]

for model in models:
    model_name = model.__class__.__name__
    accuracies = cross_val_score(model,features, labels, scoring="accuracy", cv=CV )

    for fold_idx, accuracy in enumerate(accuracies):
        entries.append((model_name,fold_idx, accuracy))

cv_df = pd.DataFrame(entries, columns=['model_name','fold_idx','accuracy'])

print("ACCURACY OF MODELS")
print(cv_df.groupby('model_name').accuracy.mean())

# TODO Find the best model and then optimize for the problem.
# TODO complete text preprocessing before the machine learning to imporve accuracy.
# We only removed stop words.
predicted_class = []
for item, cat,name in zip(df["app_mini_description"], df["category"], df["appTitle"]):

    prediction = clf.predict(count_vec.transform([item]))
    if prediction[0] != cat:

        print("App Title: {}".format(name))
        print("App Description: {}".format(item))
        print("App Developer Category: {}".format(cat))
        print("App Predicted Category: {}".format(prediction[0]))

        print("-------------------------------")


# Copyright (c) Yugabyte, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied.  See the License for the specific language governing permissions and limitations
# under the License.

import os
import time
import subprocess
import logging

from typing import Iterable, Set, List, Optional, Tuple
from yb.dep_graph_common import (
    DependencyGraph,
    Node,
    NodeType,
    DYLIB_SUFFIX,
)

from yb.common_util import (
    read_file,
    write_file,
    shlex_join,
)
from yb.build_paths import BuildPaths

from yugabyte_pycommon import WorkDirContext  # type: ignore

from pathlib import Path

SKIPPED_ARGS: Set[str] = {
    '-fPIC',
    '-lm',
    '-lrt',
    '-ldl',
    '-latomic',
    '-lcrypt',
    '-pthread',
    '-flto=thin',
    '-flto=full',
    '-Werror',
    '-Wall',
}

def get_static_lib_paths(thirdparty_path: str) -> List[str]:
    static_lib_paths = []
    for thirdparty_subdir in ['common', 'uninstrumented']:
        for root, dirs, files in os.walk(
                os.path.join(thirdparty_path, 'installed', thirdparty_subdir)):
            for file_name in files:
                if file_name.endswith('.a'):
                    static_lib_path = os.path.join(root, file_name)
                    static_lib_paths.append(static_lib_path)
    if not static_lib_paths:
        raise ValueError(
            "Did not find any static libraries at third-party path %s" % thirdparty_path)
    return static_lib_paths

def get_yb_pgbackend_link_cmd(build_root: str) -> Tuple[str, List[str]]:
    """
    Returns a tuple containing the Postgres backend build directory and a list representation
    of the linker command line for the yb_pgbackend library.
    """
    pg_backend_build_dir = os.path.join(build_root, 'postgres_build', 'src', 'backend')
    if not os.path.exists(pg_backend_build_dir):
        raise IOError("Directory does not exist: %s" % pg_backend_build_dir)

    prefix_str = 'link_cmd_libyb_pgbackend.so.'
    matched_files = [
        os.fspath(p) for p in Path(pg_backend_build_dir).glob('%s*' % prefix_str)
    ]
    if len(matched_files) != 1:
        raise ValueError(
            "Looking for the build command for the yb_pgbackend library, failed to find exactly "
            "one file starting with %s in %s. Got %s" % (
                prefix_str, pg_backend_build_dir, matched_files))
    return pg_backend_build_dir, read_file(matched_files[0]).strip().split()

class LinkCommand:
    """
    A wrapper around an array to manipulate a linker command line.
    """
    args: List[str]

    def __init__(self, initial_args: List[str] = []) -> None:
        self.args = list(initial_args)

    def append(self, arg: str) -> None:
        self.args.append(arg)

    def extend(self, more_args: List[str]) -> None:
        self.args.extend(more_args)

    def add_new_arg(self, new_arg: str) -> None:
        should_skip = (
            new_arg in SKIPPED_ARGS or
            new_arg.startswith('--gcc-toolchain')
        )

        if should_skip:
            logging.info("Skipping %s", new_arg)
            return

        self.append(new_arg)

    def contains(self, arg: str) -> bool:
        return arg in self.args

def process_original_link_cmd(original_link_cmd: List[str]) -> List[str]:
    """
    Remove unneeded parts of the original linker command.
    """
    if len(original_link_cmd) < 6:
        raise ValueError("Original linker command is too short: %s" % shlex_join(original_link_cmd))
    assert original_link_cmd[0] == ':'
    assert original_link_cmd[1] == '&&'
    assert original_link_cmd[2].endswith('/compiler-wrappers/c++')
    assert original_link_cmd[-2] == '&&'
    assert original_link_cmd[-1] == ':'

    return original_link_cmd[3:-2]

def is_yb_library(rel_path: str) -> bool:
    return rel_path.startswith(('lib/', 'postgres/lib/')) and rel_path.endswith(DYLIB_SUFFIX)

class LinkHelper:
    """
    Helps with creating and running a custom linking command for YugabyteDB outside of the build
    system.
    """
    dep_graph: DependencyGraph
    initial_node: Node

    # Arguments to the linker in the original command that produces the initial_node target.
    # Does not include the compiler driver executable.
    original_link_args: List[str]

    build_root: str
    build_paths: BuildPaths
    llvm_path: str
    thirdparty_path: str
    clang_cpp_path: str

    # Dependency graph nodes corresponding to the object files present in the original linker
    # command. Used for deduplication.
    obj_file_graph_nodes: Set[Node]

    new_args: LinkCommand

    # Build directory of the Postgres backend.
    pg_backend_build_dir: str

    # The command for linking the yb_pgbackend library.
    yb_pgbackend_link_cmd: List[str]

    lto_output_suffix: Optional[str]

    # Populated by consume_original_link_cmd.
    final_output_name: str

    def __init__(
            self,
            dep_graph: DependencyGraph,
            initial_node: Node,
            lto_output_suffix: Optional[str]) -> None:
        self.dep_graph = dep_graph
        self.initial_node = initial_node

        self.build_root = self.dep_graph.conf.build_root
        self.build_paths = BuildPaths(self.build_root)

        self.llvm_path = self.build_paths.get_llvm_path()
        self.thirdparty_path = self.build_paths.get_thirdparty_path()
        self.clang_cpp_path = self.build_paths.get_llvm_tool_path('clang++')

        assert initial_node.link_cmd
        self.original_link_args = process_original_link_cmd(initial_node.link_cmd)
        self.static_lib_paths = get_static_lib_paths(self.thirdparty_path)
        self.new_args = LinkCommand([self.clang_cpp_path])
        self.obj_file_graph_nodes = set()
        self.pg_backend_build_dir, self.yb_pgbackend_link_cmd = get_yb_pgbackend_link_cmd(
            self.build_root)

        self.lto_output_suffix = lto_output_suffix

    def convert_to_static_lib(self, arg: str) -> Optional[str]:
        """
        Given an argument to the original linker command, try to interpret it as a library, either
        specified as a shared library path, or using -l... syntax, and return the corresponding
        static library path if available.
        """
        if arg.startswith('/') and arg.endswith('.so'):
            arg_static_prefix = arg[:-3]

            static_found = False
            for suffix in ['.a', '-s.a']:
                arg_static = arg_static_prefix + suffix
                if os.path.exists(arg_static):
                    logging.info("Using static library %s instead of shared library %s",
                                 arg_static, arg)
                    return arg_static
            logging.info("Did not find static library corresponding to %s", arg)

        if arg.startswith('-l'):
            static_found = False
            logging.info("Looking for static lib for: %s", arg)
            lib_name = arg[2:]
            for static_lib_path in self.static_lib_paths:
                static_lib_basename = os.path.basename(static_lib_path)
                if (static_lib_basename == 'lib' + lib_name + '.a' or
                        static_lib_basename == 'lib' + lib_name + '-s.a'):
                    logging.info("Found static lib for %s: %s", lib_name, static_lib_path)
                    return static_lib_path
            logging.info("Did not find a static lib for %s", lib_name)

        if arg.endswith('.so') or '.so.' in arg:
            logging.info("Still using a shared library: %s", arg)

        return None

    def process_arg(self, arg: str) -> None:
        if arg in SKIPPED_ARGS:
            logging.info("Skipping argument: %s", arg)
            return

        new_arg = self.convert_to_static_lib(arg)
        if new_arg:
            if not self.new_args.contains(new_arg):
                self.new_args.add_new_arg(new_arg)
        else:
            self.new_args.add_new_arg(arg)

    def consume_original_link_cmd(self) -> None:
        """
        Goes over the original linker command and reuses some of its arguments for the new command.
        """
        with WorkDirContext(self.build_root):
            expect_output_name = False
            output_name: Optional[str] = None
            for arg in self.original_link_args:
                if arg == '-o':
                    expect_output_name = True
                    continue
                if expect_output_name:
                    if output_name:
                        raise ValueError(
                            "Found multiple output names in the original link command: "
                            "%s and %s" % (output_name, arg))
                    output_name = arg
                    expect_output_name = False
                    continue
                expect_output_name = False

                if is_yb_library(arg):
                    logging.info("Skipping YB library: %s", arg)
                    continue

                if arg.endswith('.cc.o'):
                    # E.g. tablet_server_main.cc.o.
                    # Remember this node for later deduplication.
                    self.obj_file_graph_nodes.add(self.dep_graph.find_node(os.path.realpath(arg)))

                self.process_arg(arg)

            if not output_name:
                raise ValueError("Did not find an output name in the original link command")
            self.final_output_name = os.path.abspath(output_name)
            logging.info("Final output file name: %s", self.final_output_name)
            if self.lto_output_suffix is not None:
                self.final_output_name += self.lto_output_suffix
            self.new_args.extend(['-o', self.final_output_name])

    def add_leaf_object_files(self) -> None:
        """
        Goes over all the object files that the original node transitively depends on, and adds
        them to the link command if they have not already been added.
        """

        transitive_deps = self.initial_node.get_recursive_deps(
            skip_node_types=set([NodeType.EXECUTABLE]))
        with WorkDirContext(self.build_root):
            # Sort nodes by path for determinism.
            for node in sorted(list(transitive_deps), key=lambda dep: dep.path):
                if node in self.obj_file_graph_nodes:
                    # Dedup .cc.o files already existing on the command line.
                    continue

                if node.node_type == NodeType.OBJECT:
                    self.new_args.add_new_arg(node.path)

            for arg in self.yb_pgbackend_link_cmd:
                if arg.endswith('.o'):
                    if os.path.basename(arg) == 'main_cpp_wrapper.cc.o':
                        # TOOD: why is this file even linked into libyb_pgbackend?
                        continue
                    self.new_args.append(os.path.join(self.pg_backend_build_dir, arg))
                    continue
                if (arg.startswith('-l') and
                        not self.new_args.contains(arg) and
                        not arg.startswith('-lyb_')):
                    self.process_arg(arg)

    def add_final_args(self, lto_type: str) -> None:
        assert lto_type in ['full', 'thin']
        self.new_args.extend([
            '-L%s' % os.path.join(self.build_root, 'postgres', 'lib'),
            '-l:libpgcommon.a',
            '-l:libpgport.a',
            '-l:libpq.a',
            '-fwhole-program',
            '-Wl,-v',
            '-nostdlib++',
            '-flto=' + lto_type,
        ])

        for lib_name in ['libc++.a', 'libc++abi.a']:
            self.new_args.append(os.path.join(
                self.thirdparty_path, 'installed', 'uninstrumented', 'libcxx', 'lib', lib_name))

        with WorkDirContext(self.build_root):
            self.write_link_cmd_file(self.final_output_name + '_lto_link_cmd_args.txt')

    def run_linker(self) -> None:
        with WorkDirContext(self.build_root):
            start_time_sec = time.time()
            logging.info("Running linker")
            try:
                subprocess.check_call(self.new_args.args)
            except subprocess.CalledProcessError as ex:
                # Avoid printing the extremely long command line.
                logging.error("Linker returned exit code %d", ex.returncode)
            elapsed_time_sec = time.time() - start_time_sec
            logging.info("Linking finished in %.1f sec", elapsed_time_sec)

    def write_link_cmd_file(self, out_path: str) -> None:
        logging.info("Writing the linker command line (one argument per line) to %s",
                     os.path.abspath(out_path))
        write_file('\n'.join(self.new_args.args), out_path)

def link_whole_program(
        dep_graph: DependencyGraph,
        initial_nodes: Iterable[Node],
        link_cmd_out_file: Optional[str],
        run_linker: bool,
        lto_output_suffix: Optional[str],
        lto_type: str) -> None:
    initial_node_list = list(initial_nodes)
    assert len(initial_node_list) == 1
    initial_node = initial_node_list[0]

    link_helper = LinkHelper(
        dep_graph=dep_graph,
        initial_node=initial_node,
        lto_output_suffix=lto_output_suffix
    )

    # We stop recursive traversal at executables because those are just code generators
    # (protoc-gen-insertions, protoc-gen-yrpc, bfql_codegen, bfpg_codegen).

    conf = dep_graph.conf

    link_helper.consume_original_link_cmd()
    link_helper.add_leaf_object_files()
    link_helper.add_final_args(lto_type=lto_type)

    with WorkDirContext(conf.build_root):
        if link_cmd_out_file:
            link_helper.write_link_cmd_file(link_cmd_out_file)
        if not run_linker:
            return
        link_helper.run_linker()


import re
import numpy, wave,matplotlib
import matplotlib.pyplot as plt
import numpy as np
import os
import librosa
import librosa.display
from PIL import Image
import re
import shutil
from create_base import *
from create_labels_files import *
from myDtw import *

def clear_dir(dis_dir):
    shutil.rmtree(dis_dir)
    os.mkdir(dis_dir)

def load_and_trim(path):
    audio, sr = librosa.load(path)
    energy = librosa.feature.rmse(audio)
    frames = np.nonzero(energy >= np.max(energy) / 5)
    indices = librosa.core.frames_to_samples(frames)[1]
    audio = audio[indices[0]:indices[-1]] if indices.size else audio[0:0]

    return audio, sr

def load_and_trim_v2(path,offset,duration):
    audio, sr = librosa.load(path, offset=offset, duration=duration)
    energy = librosa.feature.rmse(audio)
    frames = np.nonzero(energy >= np.max(energy) / 5)
    indices = librosa.core.frames_to_samples(frames)[1]
    audio = audio[indices[0]:indices[-1]] if indices.size else audio[0:0]

    return audio, sr

savepath = 'e:/test_image/t/'
filename = 'F:/项目/花城音乐项目/样式数据/2.27MP3/节奏/节奏1.3(95).wav'
filename = 'F:/项目/花城音乐项目/样式数据/2.27MP3/节奏/节奏1（二）(100).wav'
filename = 'F:/项目/花城音乐项目/样式数据/2.27MP3/节奏/节奏1_40227（100）.wav'
filename = 'F:/项目/花城音乐项目/样式数据/2.27MP3/节奏/节奏1林(70).wav'
filename = 'F:/项目/花城音乐项目/样式数据/2.27MP3/节奏/节奏2_40314（100）.wav'
filename = 'F:/项目/花城音乐项目/样式数据/2.27MP3/节奏/节奏2_40409（98）.wav'
filename = 'F:/项目/花城音乐项目/样式数据/2.27MP3/节奏/节奏2林(25).wav'
filename = 'F:/项目/花城音乐项目/样式数据/2.27MP3/节奏/节奏2语(85).wav'
#filename = 'F:/项目/花城音乐项目/样式数据/3.06MP3/节奏/节2罗（75）.wav'
filename = 'F:/项目/花城音乐项目/样式数据/2.27MP3/旋律/视唱1-01（95）.wav'
filename = 'F:/项目/花城音乐项目/样式数据/2.27MP3/旋律/旋律二（11）（60）.wav'
filename = 'F:/项目/花城音乐项目/样式数据/2.27MP3/旋律/视唱1-02（90）.wav'

y, sr = librosa.load(filename)
rms = librosa.feature.rmse(y=y)[0]
time = librosa.get_duration(filename=filename)
total_frames_number = len(rms)
print("time is {}".format(time))
CQT = librosa.amplitude_to_db(librosa.cqt(y, sr=16000), ref = np.max)
librosa.display.specshow(CQT, y_axis='cqt_note',x_axis='time')
w,h = CQT.shape
#onset_frames = librosa.onset.onset_detect(y=y, sr=sr)
onset_frames = get_real_onsets_frames_rhythm(y,modify_by_energy=False)

onset_times = librosa.frames_to_time(onset_frames, sr=sr)
plt.vlines(onset_times, 0,sr, color='y', linestyle='--')
onset_samples = librosa.time_to_samples(onset_times)
print(onset_samples)
#plt.subplot(len(onset_times),1,1)
plt.show()

clear_dir(savepath)
for i in range(0, len(onset_frames)):
    half = 15
    start = onset_frames[i] - half
    if start < 0:
        start =0
    end = onset_frames[i] + half
    if end >=total_frames_number:
        end = total_frames_number - 1
    #y2 = [x if i> start and i<end else 0 for i,x in enumerate(y)]
    CQT_sub = np.zeros(CQT.shape)
    middle = int(h / 2)
    offset = middle - onset_frames[i]
    for j in range(int(start), int(end)):
        CQT_sub[:, j + offset] = CQT[:, j]
    #CQT = CQT_T
    librosa.display.specshow(CQT_sub, y_axis='cqt_note', x_axis='time')
    #y2 = [x for i,x in enumerate(y) if i> start and i<end]
    #y2 = [0.03 if i> start and i<end else 0.02 for i,x in enumerate(y)]
    #y2[int(len(y2) / 2)] = np.max(y)  # 让图片展示归一化
    t = librosa.frames_to_time([middle], sr=sr)
    plt.vlines(t, 0,sr, color='y', linestyle='--')# 标出节拍位置
    #y2 = np.array(y2)
    #print("len(y2) is {}".format(len(y2)))

    print("(end - start)*sr is {}".format((end - start)*sr))
    #plt.show()
    #plt.subplot(len(onset_times),1,i+1)
    #y, sr = librosa.load(filename, offset=2.0, duration=3.0)
    #librosa.display.waveplot(y2, sr=sr)
    fig = matplotlib.pyplot.gcf()
    #fig.set_size_inches(4, 4)
    if "." in filename:
        Filename = filename.split(".")[0]
    plt.axis('off')
    plt.axes().get_xaxis().set_visible(False)
    plt.axes().get_yaxis().set_visible(False)
    plt.savefig(savepath + str(i+1) + '.jpg', bbox_inches='tight', pad_inches=0)
    plt.clf()
#plt.show()


__author__ = 'tian'
import scrapy
from four.items import FourItem
import four.items
import re
import requests
import four.settings

class FiftyEightSpider(scrapy.Spider):
    name = 'fiftyeight'

    def start_requests(self):
        urls = ['http://www.sach.gov.cn/module/jslib/jquery/jpage/dataproxy.jsp?startrecord=1&endrecord=45&perpage=15\
        &col=1&appid=1&webid=1&path=/&columnid=1036&sourceContentType=1&unitid=13601&webname=%E5%9B%BD%E5%AE%B6%E6%9\
        6%87%E7%89%A9%E5%B1%80&permissiontype=0',
                'http://www.sach.gov.cn/module/jslib/jquery/jpage/dataproxy.jsp?startrecord=46&endrecord=55&perpage=15\
                &col=1&appid=1&webid=1&path=/&columnid=1036&sourceContentType=1&unitid=13601&webname=%E5%9B%BD%E5%AE%B\
                6%E6%96%87%E7%89%A9%E5%B1%80&permissiontype=0']
        for url in urls:
            # print('url:%s'%url)
            yield scrapy.Request(url=url, callback=self.parse_url,meta={'url':url})

    def parse_url(self, response):
        url_prefix = 'http://www.sach.gov.cn'
        records = response.xpath('//record')
        for record in records:
            result = record.xpath('.').extract_first()
            result = result.split()
            for r in result:
                if r.startswith('href'):
                    r = r.split('=')[1]
                    r = url_prefix+r[1:-1]
                    yield scrapy.Request(url=r,callback=self.parse)

            # href = record.xpath('//a/@href').extract_first()
            # print('href:%s'%href)
    def parse(self, response):
        title = response.xpath('//title').extract_first()
        text = response.xpath('//*[@id="zoom"]').extract()
        text = ' '.join(text)
        item_fiftyeight = four.items.fillinData(title,'','','','','','','','','','','','',text,'','','')
        yield item_fiftyeight

        # prefix_url = 'http://www.sastind.gov.cn/'
        # title = response.xpath('//*[@class="yjzj_nr_bt"]/text()').extract_first()
        # title = title.strip()
        # # print('title:%s'%title)
        # source = response.xpath('//*[@class="yjzj_nr_top"]/p/text()').extract_first()
        # source = source.strip()
        # # print('source:%s'%source)
        # text = response.xpath('//p[@class="MsoNormal"]').extract()
        #
        # if not text:
        #     text = response.xpath('//*[@class="allcontent"]/p').extract()
        # if not text:
        #     text = response.xpath('//*[@class="TRS_Editor"]/p').extract()
        # if not text:
        #     text = response.xpath('//*[@class="allcontent"]').extract()
        # text = ' '.join(text)
        # # print('text:%s'%text)
        # item_fiftyfour = four.items.fillinData(title,'','','','','','','','','','','','',text,'','',source)
        # yield item_fiftyfour

        # item_fiftythree = four.items.fillinData(title,'','','','','','','','','','','','',text,'',publishDate,'')
        # yield item_fiftythree

        # publishDate = response.xpath('//*[@id="con_time"]/text()').extract_first()
        # publishDate = publishDate.strip().split('：')[1]
        # publishDate = publishDate[:11]
        # text = response.xpath('//*[@id="con_con"]/p').extract()
        # text = ' '.join(text)
        # file = []
        # file_ = response.xpath('//*[@id="con_con"]//p')
        # attachment = file_.xpath('.//a')
        # for a_ in attachment:
        #     name = a_.xpath('./text()').extract_first()
        #     url = a_.xpath('./@href').extract_first()
        #     url = prefix_url + url[6:]
        #     print('name:%s;url:%s'%(name,url))
        #     if not url.endswith('html'):
        #         r = requests.get(url)
        #         with open('/Users/tian/zaqizaba/'+name,'wb') as f:
        #             f.write(r.content)
        #             file.append('/Users/tian/zaqizaba/'+name)
        # file = ';'.join(file)
        # print('file:%s'%file)
        # item_fiftyone = four.items.fillinData(title,'','','','','','','','','','','','',text,file,publishDate,'')
        # yield item_fiftyone


import requests

class PyCycle(object):
    def __init__(self, key, url='https://publicapi.bcycle.com/api/1.0'):
        self.key = key
        self.url = url

    def list_programs(self):
        headers = {'ApiKey': self.key}
        r = requests.get('{}/ListPrograms'.format(self.url), headers=headers)

        # Throw an error for anything other than a 200
        r.raise_for_status()

        return r.json()

    def list_program_kiosks(self, program):
        headers = {'ApiKey': self.key}
        r = requests.get('{}/ListProgramKiosks/{}'.format(self.url, program), headers=headers)

        # Throw an error for anything other than a 200
        r.raise_for_status()

        return r.json()


#!/usr/bin/env python
"""
Script Header

$Id: cmCC25693_3pcc_BS_Functional_293_Blind_Transfer_To_Active_Monitored_User

Copyright (c) 2016-2017 Cisco Systems, Inc.

Name:
    cmCC25693_3pcc_BS_Functional_293_Blind_Transfer_To_Active_Monitored_User.py

Purpose:
    This test case verifies the DUT using the BLF line key to blind transfer
    a call to an active monitored user.

Author:
    Anuradha N(anakre@cisco.com)

References:
    BW-SIPPhone-FunctionalTestPlan-R21.0
    US25693

Description:
     Originate a call from BroadWorks User B to a DUT monitored User. Answer
     the call. Originate a call from BroadWorks User A to the DUT. Answer the
     call. From the DUT, blind transfer the call to a monitored user, using
     the line key for the monitored user. Answer the call.

Topology:
    1. 4 3pcc phones.
    2. All the phones should register successfully before running the script
    3. BLF should be configured on monitoring user phone.
    4. Make sure at least 1 user is added as monitored user.
    5. Create a call pickup group by browsing to Services->Resources and
    ensure that DUT and monitored users are added.

Pass/Fail Criteria:
    1. Check that the lamp state is fast blinking red when the monitored user
    is ringing
    2. Check that the lamp state is solid red when the user is connected
    3. Check that the lamp state is steady and green when monitored user is
    idle.

Test Steps:
    1. Phone D calls Phone B (DUT monitored User)
    2. Phone B answers the call
    3. Phone C calls Phone A(DUT)
    4. Phone A answers the call.
    5. Phone A initiates blind transfer and pushes the line key for monitored
    user (Phone B) to complete the tansfer
    6. Phone B rings and answers the call
    7. Phone C and Phone B are connected.
    8. Phone C disconnects the call.
    9. Phone D disconnects the call.

    Verify:
    1. Phone B (DUT monitored User) rings.
    2. Two way voice path is established between Phone D and Phone B
    3. DUT monitored line lamp for Phone B is solid red
    4. Phone A rings
    5. Two way voice path is established between Phone A and Phone C
    6. Phone is released.
    7. Phone B rings. Phone C and Phone B see each others caller id.
    8. DUT monitored line lamp for Phone B is Fast blinking red(inbound
    alerting)
    9. Phone B answers the call.
    10. Phone B and Phone C are connected
    11. DUT monitored line lamp for Phone B is solid red
    12. Phone C disconnects the call
    13. DUT monitored line lamp for Phone B is solid red
    14. Phone D disconnects the call
    15. DUT monitored line lamp for Phone B is green/off

Notes:

Known Bugs:

"""

import tng
import logging
from tng_sl.device.endpoint.synergylite.synergylite_3pcc_extended \
    import wait_for_ccapi_call_states, poll_feature_state
from tng.frontend.timing import wait
from tng_sl.plugins.synergylite_3pcc_ui import SynergyLite3pccUiHelper
from tng_sl.contrib.mpp.phone_line_reg_helper import PhoneLineRegHelper
from tng_sl.contrib.mpp.phone_line_reg_helper import PhoneConfigHelper
from tng_sl.contrib.mpp.broadsoft_login_helper import BroadsoftLoginHelper
from tng_sl.contrib.mpp.tshark_helper import TsharkHelper
from tng_sl.contrib.setup_helper import SetupHelpersTestCase
from tng_sl.contrib.mpp.broadsoft.broadsoft_config import BroadsoftConfig

log = logging.getLogger('BlindTransferToActiveMonitoredUSer')

class BlindTransferToActiveMonitoredUSer(
        SetupHelpersTestCase, tng.api.TestCase):
    helpers = (
        PhoneConfigHelper, PhoneLineRegHelper, TsharkHelper,
        BroadsoftLoginHelper)
    helper_num_devices = 4

    def setUp(self):
        log.info("Start of setUp method")

        self.proxy = self.phone_data['proxy']
        self.xsi_user_id1 = self.toolkit.get_test_env_info(
            section='bsoft', parameter_name="xsi_user_id1")
        self.xsi_user_id2 = self.toolkit.get_test_env_info(
            section='bsoft', parameter_name="xsi_user_id2")
        self.xsi_user_id3 = self.toolkit.get_test_env_info(
            section='bsoft', parameter_name="xsi_user_id3")
        self.uri_name = '{}_blf@{}'.format(self.user_id1, self.proxy)

        self.broadsoft = BroadsoftConfig()
        log.info("Enable busy lamp field in the server")
        self.broadsoft.set_busy_lamp_field(
            list_uri=self.uri_name,
            blf_monitored_user_id=[self.xsi_user_id2],
            user_id_proxy=self.xsi_user_id1, user_id=self.user_id1)

        def broadsoft_cleanup():
            log.info("CleanUp configured BLF from the server")
            self.broadsoft.set_busy_lamp_field(
                list_uri=self.uri_name, blf_monitored_user_id=[],
                user_id_proxy=self.xsi_user_id1, user_id=self.user_id1)

        self.addCleanup(broadsoft_cleanup)
        self.call_pickup_group = self.bsoft_web.create_callpickup_group(
            [self.user_id2, self.user_id1])

        def broadsoft_delete_callpickup_group():
            self.bsoft_web.delete_callpickup_group(self.call_pickup_group)

        self.addCleanup(broadsoft_delete_callpickup_group)
        self.p2_fname, self.p2_lname = self.broadsoft.get_first_and_last_name(
            user_id=self.user_id2, user_id_proxy=self.xsi_user_id2)
        self.p3_fname, self.p3_lname = self.broadsoft.get_first_and_last_name(
            user_id=self.user_id3, user_id_proxy=self.xsi_user_id3)

        log.info("End of setUp method")

    def test_blind_transfer_to_active_monitored_user(self):

        log.info("Start of test_blind_transfer_to_active_monitored_user")

        self.oPhone1.ui.set_web_parameter_http(
            BLF_List_Uri=['Att Console', 'BLF List URI', self.uri_name],
            Use_line_keys=['Att Console', 'Use Line Keys For BLF List', 1])

        list_uri_name, use_line_keys = self.oPhone1.get_web_config(
            'BLF_List_URI', 'Use_Line_Keys_For_BLF_List')

        self.assertEqual(self.uri_name, list_uri_name)
        self.assertEqual("Yes", use_line_keys)

        p3_caller_name = '{} {}'.format(self.p3_fname, self.p3_lname)
        p2_caller_name = '{} {}'.format(self.p2_fname, self.p2_lname)

        filter_cmd = (
            'port sip and (host {} or host {} or host {} or host {})'.format(
                self.oPhone1.ip, self.oPhone2.ip, self.oPhone3.ip,
                self.oPhone4.ip))
        log.info('Start tshark on linux with filter {}'.format(filter_cmd))
        self.tshark.tshark_start(filter_cmd)

        log.info('Phone D calls Phone B extension {}'.format(self.user_id2))
        self.oPhone4.ccapi.dial('null', self.user_id2, '', 1, 0, 1)
        self.oPhone2.register_call_event('VOIP_MSG_CALL_EVENT_INCOMING')
        self.oPhone3.register_call_event('VOIP_MSG_CALL_EVENT_INCOMING')
        self.oPhone2.ccapi.feedback_subscribe(
            self.oPhone2.subscribed_callback)
        wait_for_ccapi_call_states(
            (self.oPhone2, self.oPhone4), ('RINGING', 'PROCEEDING'))
        log.info("Phone B answers the call")
        self.oPhone2.ccapi.accept('0000')
        log.info('Check for Phone B INUSE state in Phone A')
        poll_feature_state(
            self.oPhone1, 2, ['State', 'LedColor', 'LedCadence'],
            expected_state=['INUSE', 'RED', 'STEADY'])

        log.info('Phone C calls phone A extension {} '.format(self.user_id1))
        self.oPhone3.ccapi.dial('null', self.user_id1, '', 1, 0, 1)
        wait_for_ccapi_call_states(
            (self.oPhone3, self.oPhone1), ('PROCEEDING', 'RINGING'))

        log.info("Phone A answers the call")
        self.oPhone1.ccapi.accept('0000')
        wait_for_ccapi_call_states(
            self.devices, ('CONNECTED', 'CONNECTED', 'CONNECTED', 'CONNECTED'))
        self.oPhone3.ccapi.feedback_subscribe(
            self.oPhone3.subscribed_callback)

        log.info('Phone A intiates blind transfer')
        self.oPhone1.ccapi.dial('null', 'bdxfer?', '', 1, 1, 1)
        wait(3, 'Wait for 3s before pushing the line key')

        log.info(
            'Phone A pushes line key 2 to complete transfer to monitored user')
        self.oPhone1.ccapi.sendKey(SynergyLite3pccUiHelper.PK_LN2, 1, '0001')
        wait_for_ccapi_call_states(
            self.devices, ('IDLE', 'RINGING', 'CONNECTED', 'CONNECTED'),
            (1, 1, 1, 1), (0, 1, 0, 0))

        log.info('Check for Phone B ALERTING state in Phone A')
        poll_feature_state(
            self.oPhone1, 2, ['State', 'LedColor', 'LedCadence'],
            expected_state=['ALERTING', 'RED', 'FAST_BLINK'])

        log.info('Phone B accepts the call')
        self.oPhone2.ccapi.accept('0001')
        wait_for_ccapi_call_states(
            self.devices, ('IDLE', 'CONNECTED', 'CONNECTED', 'CONNECTED'),
            (1, 1, 1, 1), (0, 1, 0, 0))
        log.info('Check for Phone B INUSE state in Phone A')
        poll_feature_state(
            self.oPhone1, 2, ['State', 'LedColor', 'LedCadence'],
            expected_state=['INUSE', 'RED', 'STEADY'])

        self.oPhone3.check_caller_id(p2_caller_name, self.user_id2[-4:])
        self.oPhone2.check_caller_id(p3_caller_name, self.user_id3[-4:])

        log.info('Check for Phone B INUSE state in Phone A')
        poll_feature_state(
            self.oPhone1, 2, ['State', 'LedColor', 'LedCadence'],
            expected_state=['INUSE', 'RED', 'STEADY'])

        log.info('Phone C and D ends the call')
        self.oPhone3.ccapi.hangUp('0000')
        log.info('Check for Phone B INUSE state in Phone A')
        poll_feature_state(
            self.oPhone1, 2, ['State', 'LedColor', 'LedCadence'],
            expected_state=['INUSE', 'RED', 'STEADY'])

        self.oPhone4.ccapi.hangUp('0000')
        wait_for_ccapi_call_states(
            self.devices, ('IDLE', 'IDLE', 'IDLE', 'IDLE'))
        log.info('Check for Phone B IDLE state in Phone A')
        poll_feature_state(
            self.oPhone1, 2, ['State', 'LedColor', 'LedCadence'],
            expected_state=['IDLE', 'GREEN', 'STEADY'])

        log.info('Stop tshark on linux')
        self.tshark.tshark_stop()

        self.oPhone2.ccapi.feedback_unsubscribe(
            self.oPhone2.subscribed_callback)
        self.oPhone2.unregister_call_event('VOIP_MSG_CALL_EVENT_INCOMING')
        self.oPhone3.ccapi.feedback_unsubscribe(
            self.oPhone3.subscribed_callback)
        self.oPhone3.unregister_call_event('VOIP_MSG_CALL_EVENT_INCOMING')

        log.info('End of test_blind_transfer_to_active_monitored_user')

def main():
    tng.api.runner()

if __name__ == '__main__':
    tng.run(main)


import requests
import re
import pandas as pd
import plotly

url_first='http://www.ygdy8.net/html/gndy/dyzz/index.html'
head={'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'}
cookies={'cookie':'XLA_CI=b65d62d83ce74a25a2a33c92c2a510e0'}
topHtml=requests.get(url_first, headers=head, cookies=cookies)
topHtml.encoding = 'gbk'

nextMatch=re.compile(r'<a\s+href=\'(.*?)\'>下一页</a>') #列表页下一页
msgMatch = re.compile(r'<a\s+href="(.+?\.html)"\s+class="ulink">') #详情页

##详情页需要抓取的数据项
##view-source:http://www.ygdy8.net/html/gndy/dyzz/20181113/57761.html
msgsMatch=re.compile(r'◎译\s+名\s+(.*?)\s+<br\s+/>.*?◎片\s+名\s+(.*?)\s+<br\s+/>.*?◎产\s+地\s+(.*?)\s+<br\s+/>.*?◎豆瓣评分\s+(.*?)\s+<br\s+/>.*?◎简\s+介\s+<br\s+/><br\s+/>\s+(.*?)<br\s+/><br\s+/><img.*?<a\s+href="(ftp://.+?)">ftp://.*?',re.S)
#msgsMatch=re.compile(r'译\s+名\s+(.*?)\s+<br\s+/>.*?',re.S)
i = 0
st = set("0")
while topHtml.status_code == 200:
    print('inLoop')
    i += 1
    if i > 20:
        break
    listMsg = re.findall(msgMatch,topHtml.text)
    for msgPagePre in listMsg:
        msgPageUrl = 'http://www.ygdy8.net' + msgPagePre
        print("msgPageUrl:" + msgPageUrl)
        pageHtml = requests.get(msgPageUrl, headers=head, cookies=cookies)
        ###设置输出的编码格式，否则为中文乱码
        pageHtml.encoding = 'gbk'
        msgsThisPage = re.findall(msgsMatch,pageHtml.text)
        ##去重
        if len(msgsThisPage)>0:
            if msgsThisPage[0] in st:
                print("has the same item:")
                print(msgsThisPage[0])
                continue
            st.add(msgsThisPage[0])
        print(msgsThisPage)
        data=pd.DataFrame(msgsThisPage)
        data.to_csv('./movieMsgs.csv', header=False,index=False,mode='a+')
        data=[]
        msgsThisPage=[]
    nextPagePre = re.findall(nextMatch, topHtml.text)[0]
    nextPageUrl = 'http://www.ygdy8.net/html/gndy/dyzz/' + nextPagePre
    topHtml = requests.get(nextPageUrl, cookies=cookies, headers=head)
    topHtml.encoding = 'gbk'


import json

from hc.api.models import Channel
from hc.test import BaseTestCase
from mock import patch

class ChannelModelTestCase(BaseTestCase):

    @patch("hc.api.models.requests.post")
    def test_it_refreshes_hipchat_access_token(self, mock_post):
        mock_post.return_value.json.return_value = {"expires_in": 100}

        channel = Channel(kind="hipchat", user=self.alice, value=json.dumps({
            "oauthId": "foo",
            "oauthSecret": "bar"
        }))

        channel.refresh_hipchat_access_token()

        # It should request a token using a correct tokenUrl
        mock_post.assert_called()
        self.assertTrue("expires_at" in channel.value)


# -*- coding: utf-8 -*-
"""
Created on Fri Feb  1 09:49:41 2019

@author: Osman Ali Yardım

University World Ranking with Plotly: Bar Plot
"""

import pandas as pd

# Plotly library usage in Spyder
from plotly.offline import plot
import plotly.graph_objs as go

# Read data
timesData = pd.read_csv('timesData.csv')

# Discover data
print(timesData.head())
timesData.info()

# Citation and Teaching of Top 3 Universities in 2014
# Prepare Data Frames
df2014 = timesData[timesData.year == 2014].iloc[:3,:]

# Creating trace1
trace1 = go.Bar(x=df2014.university_name,
                y=df2014.citations,
                name="citation",
                marker=dict(color='rgba(255,174,255,0.5)',
                line=dict(color='rgb(0,0,0)', width=1.5)),
                text = df2014.country)

# Creating trace2
trace2 = go.Bar(x=df2014.university_name,
                y=df2014.teaching,
                name="teaching",
                marker=dict(color='rgba(255,255,128,0.5)',
                line=dict(color='rgb(0,0,0)', width=1.5)),
                text = df2014.country)

data = [trace1,trace2]
layout = go.Layout(barmode="group")
fig = go.Figure(data=data, layout=layout)
plot(fig)

# Example 2 -------------------------------------------------
# Citation and Teaching of Top 3 Universities in 2014 - 2
df2014 = timesData[timesData.year == 2014].iloc[:3,:]
x = df2014.university_name

trace3 = {'x':x, 'y':df2014.citations,'name':'citation','type':'bar'}
trace4 = {'x':x, 'y':df2014.teaching,'name':'teaching','type':'bar'}

data2 = [trace3,trace4]

layout2 = {
        'xaxis': {'title':'Top 3 Universities'},
        'barmode': 'relative',
        'title': 'Citation and Teaching of Top 3 Universities in 2014'
        }

fig2 = go.Figure(data=data2, layout=layout2)
plot(fig2)


from argparse import ArgumentParser
import os
import pickle
import time
import torch
from shutil import copyfile
import subprocess
import sys
import warnings
import common
import network_utils as networkUtils
import data_loader as dataLoader
import copy
import numpy as np
import functools
import random
import functions as fns
import datetime
import common

# Supported data_loaders
data_loader_all = sorted(name for name in dataLoader.__dict__
    if name.islower() and not name.startswith("__")
    and callable(dataLoader.__dict__[name]))

def adjust_learning_rate(optimizer, epoch, args):
    """Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""
    lr = args.lr * (0.1 ** (epoch // 50))
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

def get_cls_num(dataset):

    return int(common.DATASET_CLASSES_PARAMS[dataset])

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def get_avg(self):
        return self.avg

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def compute_accuracy(output, target):
    output = output.argmax(dim=1)
    acc = 0.0
    acc = torch.sum(target == output).item()
    acc = acc/output.size(0)*100
    return acc

def eval(test_loader, model, args):
    batch_time = AverageMeter()
    acc = AverageMeter()

    # switch to eval mode
    model.eval()

    end = time.time()
    for i, (images, target) in enumerate(test_loader):
        images = images.cuda()
        target = target.cuda()
        output = model(images)
        batch_acc = compute_accuracy(output, target)
        acc.update(batch_acc, images.size(0))
        batch_time.update(time.time() - end)
        end = time.time()

        # Update statistics
        estimated_time_remained = batch_time.get_avg()*(len(test_loader)-i-1)
        fns.update_progress(i, len(test_loader),
            ESA='{:8.2f}'.format(estimated_time_remained)+'s',
            acc='{:4.2f}'.format(float(batch_acc))
            )
    print()
    print('Test accuracy: {:4.2f}% (time = {:8.2f}s)'.format(
            float(acc.get_avg()), batch_time.get_avg()*len(test_loader)))
    return float(acc.get_avg())

def device_train(train_loader, model, args):
    batch_time = AverageMeter()
    losses = AverageMeter()
    acc = AverageMeter()

    # switch to train mode
    model = model.cuda()
    model.train()

    criterion = torch.nn.BCEWithLogitsLoss()
    if args.dataset == 'imagenet':
        criterion = torch.nn.CrossEntropyLoss()

    criterion.cuda()
    optimizer = torch.optim.SGD(model.parameters(), args.lr,
                                momentum=args.momentum,
                                weight_decay=args.weight_decay)

    print('===================================================================')

    for k in range(args.local_epochs):
        for i, (images, target) in enumerate(train_loader):
            target.unsqueeze_(1)
            target_onehot = torch.FloatTensor(target.shape[0], get_cls_num(args.dataset))
            target_onehot.zero_()
            target_onehot.scatter_(1, target, 1)
            target.squeeze_(1)

            images = images.cuda()
            target_onehot = target_onehot.cuda()
            target = target.cuda()

            output = model(images)
            if args.dataset == 'imagenet':
                loss = criterion(output, target)
            else:
                loss = criterion(output, target_onehot)

            # measure accuracy and record loss
            batch_acc = compute_accuracy(output, target)

            losses.update(loss.item(), images.size(0))
            acc.update(batch_acc, images.size(0))

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            # measure elapsed time
            batch_time.update(time.time() - end)
            end = time.time()

            # Update statistics
            estimated_time_remained = batch_time.get_avg()*(len(train_loader)-i-1)
            fns.update_progress(i, len(train_loader),
                ESA='{:8.2f}'.format(estimated_time_remained)+'s',
                loss='{:4.2f}'.format(loss.item()),
                acc='{:4.2f}%'.format(float(batch_acc))
                )

    return model

def run_fl(model_path, data_loader, args, skip_ratio=0.0):
    '''
        Run federated learning
        Note: it competes GPU with worker. TODO: shall not block the gpus
        Input:
            'model_path': fused model's path, waiting to be fl-ed
            'data_loader': data loader for devices
            'fl_epoch': epoch number to run fl on each device
        Output:
    '''
    print ('Run federated learning')

    device_data_idxs = data_loader.device_data_idxs
    model = torch.load(model_path)
    best_acc = 0
    test_loader = data_loader.get_test_data_loader()

    for e in range(args.global_epochs):
        print ('=================================')
        print ('Start global iteration ' + str(e))
        epoch_begin = datetime.datetime.now()

        state_sum = {}
        train_data_num = 0
        if args.client_number > len(device_data_idxs):
            args.client_number = len(device_data_idxs)
        clients = list(np.random.choice([i for i in range(len(device_data_idxs))], args.client_number, replace=False))
        for device_id in clients:
            if device_id % 100 == 0:
                print('Start Device {}'.format(device_id))
            if random.random() < skip_ratio: # skip this device
                continue
            train_loader = data_loader.training_data_loader(device_id)
            device_model = copy.deepcopy(model)
            fine_tuned_model = device_train(train_loader, device_model, args)
            device_state = fine_tuned_model.state_dict()
            device_data_size = data_loader.get_device_data_size(device_id)
            for k in device_state:
                if k not in state_sum:
                    # state_sum[k] = torch.mul(copy.deepcopy(device_state[k]), len(device_data_idxs[device_id]))
                    state_sum[k] = torch.mul(copy.deepcopy(device_state[k]), device_data_size)
                else:
                    state_sum[k] += torch.mul(copy.deepcopy(device_state[k]), device_data_size)
            train_data_num += device_data_size
            del fine_tuned_model
        new_state = {k: torch.div(state_sum[k], train_data_num) for k in state_sum}
        model.load_state_dict(new_state)

        acc = eval(test_loader, model, args)
        if acc > best_acc:
            best_acc = acc
            torch.save(model, model_path + '_fl')
            print('Save model at epoch: ' + str(e))
        print('global iteration {}, acc = {} '.format(str(e), acc))

        epoch_end = datetime.datetime.now()
        print ('Epoch ends. Running time: {} mins'.format((epoch_end - epoch_begin).seconds / 60))
        print('Best accuracy:', best_acc)

def main(args):
    master_path = os.path.join(args.dir, 'master')
    worker_path = os.path.join(args.dir, 'worker')
    save_path = os.path.join(master_path, common.MASTER_DATALOADER_FILENAME_TEMPLATE.format(args.dataset))
    data_loader = dataLoader.__dict__[args.dataset](args.dataset_path)
    data_loader.load(save_path)

    if args.model_name == 'ALL':
        all_models = [m for m in os.listdir(master_path) if m.endswith('.pth.tar')]
    else:
        all_models = [args.model_name]
    for model in all_models:
        print ('Federated learning on model ' + model)
        model_path = os.path.join(master_path, model)
        run_fl(model_path, data_loader, args)

if __name__ == '__main__':
    # Parse the input arguments.
    arg_parser = ArgumentParser()
    arg_parser.add_argument('dir', type=str, help='path to save models (default: models/)')
    arg_parser.add_argument('model_name', type=str, help='model name to be fl-trained (default: ALL)')
    arg_parser.add_argument('dataset_path', metavar='DIR', help='path to dataset')
    arg_parser.add_argument('-j', '--workers', default=4, type=int, metavar='N',
                    help='number of data loading workers (default: 4)')
    arg_parser.add_argument('-ge', '--global_epochs', default=100, type=int, metavar='N',
                    help='number of total global epochs to run (default: 100)')
    arg_parser.add_argument('-le', '--local_epochs', default=10, type=int, metavar='N',
                    help='number of total local epochs to run (default: 10)')
    # arg_parser.add_argument('--start-epoch', default=0, type=int, metavar='N',
    #                 help='manual epoch number (useful on restarts)')
    # arg_parser.add_argument('-a', '--arch', metavar='ARCH', default='alexnet',
    #                 choices=model_names,
    #                 help='model architecture: ' +
    #                     ' | '.join(model_names) +
    #                     ' (default: alexnet)')
    arg_parser.add_argument('-b', '--batch-size', default=128, type=int,
                    metavar='N',
                    help='batch size (default: 128)')
    arg_parser.add_argument('-lr', '--learning-rate', default=0.1, type=float,
                    metavar='LR', help='initial learning rate (defult: 0.1)', dest='lr')
    arg_parser.add_argument('--momentum', default=0.9, type=float, metavar='M',
                    help='momentum (default: 0.9)')
    arg_parser.add_argument('--wd', '--weight-decay', default=5e-4, type=float,
                    metavar='W', help='weight decay (default: 5e-4)',
                    dest='weight_decay')
    # arg_parser.add_argument('--resume', default='', type=str, metavar='PATH',
    #                 help='path to latest checkpoint (default: none)')
    # arg_parser.add_argument('--no-cuda', action='store_true', default=False, dest='no_cuda',
    #                 help='disables training on GPU')

    # arg_parser.add_argument('-dn', '--device_number', type=int, default=3,
    #                         help='Total device number.')
    arg_parser.add_argument('-d', '--dataset',  default='cifar10',
                        choices=data_loader_all,
                        help='dataset: ' +
                        ' | '.join(data_loader_all) +
                        ' (default: cifar10). Defines which dataset is used. If you want to use your own dataset, please specify here.')
    arg_parser.add_argument('-cl', '--client_number', type=int, default=100,
                            help='Clinet number.')

    args = arg_parser.parse_args()
    main(args)


# Copyright (C) 2014-2016 Cuckoo Foundation.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import os
import subprocess
import time
import logging
import platform

from lib.common.abstracts import Auxiliary
from lib.common.results import NetlogFile
from lib.common.results import upload_to_host
from lib.core.config import Config

log = logging.getLogger(__name__)

class KTRACE(Auxiliary):
    """ktrace catch all syscall."""
    priority = -5

    def __init__(self):
        self.config = Config(cfg="analysis.conf")
        self.fallback_strace = False
    def start(self):
        log.info("ktrace start() %d" % os.getpid())
        os.system("/bin/dmesg -c &>/dev/null")
        root_path = os.getcwd()
        bin_path = os.path.join(root_path,"bin")
        module_path = os.path.join(bin_path, "ktrace.ko")
        if os.path.exists(module_path):
            os.system("/sbin/insmod %s analyzer=%d" % (module_path,os.getpid()))
            log.info("insmod command execute")
        else:
            log.info("kernel module not there")
        log.info("analyzer path:%s" % module_path)
        log.info("ktrace start() return versin 0.01")
        return True

    def get_pids(self):
        return []

    def stop(self):
        os.system("/sbin/rmmod ktrace")
        log.info("ktrace stop() %d" % os.getpid())
        os.system("/bin/dmesg > /var/log/ktrace.log")
        upload_to_host("/var/log/ktrace.log","logs/ktrace.log")
        upload_to_host("/proc/version","files/version")
        upload_to_host("/proc/modules","files/modules")


'''定义自己的数据集'''
# coding:utf8
import os
from PIL import Image
from torch.utils import data
import numpy as np
from torchvision import transforms as T
import scipy.io
import torch
import utils
from .SBDClassSeg import picFulPath

class VOCClassSeg(data.Dataset):

    class_names = np.array([
        'background',
        'aeroplane',
        'bicycle',
        'bird',
        'boat',
        'bottle',
        'bus',
        'car',
        'cat',
        'chair',
        'cow',
        'diningtable',
        'dog',
        'horse',
        'motorbike',
        'person',
        'potted plant',
        'sheep',
        'sofa',
        'train',
        'tv/monitor',
    ])
    mean_bgr = np.array([104.00698793, 116.66876762, 122.67891434])

    def __init__(self, txt_path, transforms=None,train=True):  # 此处train val test（指定此次的数据时用来做什么的，不同用处的数据集图片预处理方法不同，可以传入不同的txt_path即可）的指定应该用union

        '''

        参数：txt_path包含了每张图片的路径和图片的标签的路径（语义分割时就是标记好的图片的路径）
            transforms表示该读取图片后，图片要进行的预处理，如果为None,则使用默认的预处理
        '''

        # 1.指定数据集，常用的方法是给定图片路径的txt文档,将他们读入一个列表
        imgs = []
        self.train = train
        fh = open(txt_path, 'r')
        # 遍历整个txt文档，每张图片的路径作为list的一个元素
        for line in fh:
            line = line.rstrip()  # 去掉改行最后的回车符号
            words = line.split()
            imgs.append((words[0], words[1]))  # words[0]是原图片的路径，words[1]是标记好的图片的标签
        self.imgs = imgs
        # 2.指定读入图片后的预处理
        if transforms is None:

            def transforms(img, lbl):
                # T.Resize T.CenterCrop将图片保持纵横比缩放裁剪为同一大小，
                # T.Tensor,T.Normalize 将图片转为[0,1]的Tensor,并归一化
                if train==True:
                    transImg = T.Compose([T.Resize(224), T.CenterCrop(224),
                                   T.ToTensor(),
                                   T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])

                    transLbl = T.Compose([T.ToPILImage(),T.Resize(224), T.CenterCrop(224),
                                       T.ToTensor()])
                else:
                    transImg = T.Compose([T.ToTensor(),
                                          T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])

                    transLbl = T.Compose([ T.ToPILImage(),T.ToTensor()])

                img = transImg(img)

                lbl = transLbl(lbl)
                return img, lbl.long()  # lbl的训练集和验证集都很定时tensor,此处将他们都设计为3维的tensor[1*H*W]

            def untransforms(img, lbl):
                # 该trans是对已经归一化的tensor处理，得到归一化之前的Tensor
                trans = T.Compose(
                    [T.Normalize([-0.485 / 0.229, -0.456 / 0.224, -0.406 / 0.225], [1 / 0.229, 1 / 0.224, 1 / 0.225])])
                imgUntrans = trans(img)
                # for i in range(img.shape[0]):
                #     imgUntrans[i] = trans(img[i])

                return ((255*imgUntrans).type(torch.ByteTensor).transpose(0,1).transpose(1,2)).numpy(), (lbl.squeeze(0)).numpy()

            # 常规的数据操作：（裁剪为统一大小,可选T.scale），（数据增强，如随机裁剪等 语义分割时一般不做这个），ToTensor()后+T.Normalize
            #if self.train:  # 如果此次是训练集（训练集和验证集可能读取数据方法一样，但是预处理的过程不一样）
        self.transforms = transforms
        self.untransforms = untransforms
        fh.close()

    def __getitem__(self, index):  # 该方法是给定索引或键值，返回对应的值，常用在enumerate遍历数据集时
        '''返回一张图片的验证集和测试集的数据'''
        # 从list中获取图片的路径
        img_path = self.imgs[index][0]
        lbl_path = self.imgs[index][1]
        # 读取图片，为numpy格式
        im = Image.open(img_path)
        # lbl是mat格式，mat格式的处理参考https://blog.csdn.net/qq_32425195/article/details/85202552

        # load label
        lbl = Image.open(lbl_path)
        lbl = np.array(lbl, dtype=np.int32)
        lbl[lbl == 255] = -1
        lbl = torch.from_numpy(lbl)
        return self.transforms(im, lbl.unsqueeze(0))

    def __len__(self):
        '''
        返回为数据集中所有图片的个数
        :return:
        '''
        return len(self.imgs)

if __name__ == '__main__':
    pass


#!/usr/bin/env python
# coding: utf-8

import yaml
import helper
import logging.handlers

logger = logging.getLogger(__name__)

__SECURE_DATA_FILEPATH = "res/secure_data.yml"
__EMAIL_TEXT_FILE_PATH = 'res/annonce_2022.txt'
__EMAIL_OBJECT = "[Annonce Emploi] Pôle Nord Compagnie recherche Lutins créatifs"

def main():
    _configure_logger('email_annonce.log')
    logger.info("##### Le Pere Noel de la Fremo #####")

    with open(__SECURE_DATA_FILEPATH, 'r') as stream:
        secure_data = yaml.safe_load(stream)

    with open(__EMAIL_TEXT_FILE_PATH, 'r') as file:
        body = file.read()

    # for debug, change key form personnes to personnes_test
    for id_, id_param in secure_data['personnes'].items():
        helper.gmail_send_email(
            secure_data['sender_email'],
            __EMAIL_OBJECT,
            body,
            id_param['email'])

    logger.info("Finish")

def _configure_logger(filename):
    """
    write to console, simple message with log level info
    write to file, formatted message with log level debug
    """
    logger = logging.getLogger('')
    logger.setLevel(logging.DEBUG)

    # Console
    consoleHandler = logging.StreamHandler()
    consoleHandler.setLevel(logging.INFO)
    consoleFormatter = logging.Formatter('%(message)s')
    consoleHandler.setFormatter(consoleFormatter)
    logger.addHandler(consoleHandler)

    # File
    fileHandler = logging.handlers.RotatingFileHandler(filename, maxBytes=1024 * 1024, backupCount=1)
    fileHandler.setLevel(logging.DEBUG)
    fileFormatter = logging.Formatter(
        fmt='[%(asctime)s][%(name)s][%(levelname)-5s]%(message)s',
        datefmt='%Y-%m-%d %H:%M:%S')
    fileHandler.setFormatter(fileFormatter)
    logger.addHandler(fileHandler)

if __name__ == "__main__":
    main()


import logging

import click
import pandas as pd
from tqdm import tqdm

from tucasa.parsers import zonaprop

@click.command()
@click.argument('url')
@click.option('-o', '--output-file', 'archivo_salida',
              default='resultados.csv', show_default=True)
def main(url, archivo_salida):
    busqueda = zonaprop.ResultadoBusqueda(url)
    resultados = busqueda.cantidad_de_resultados
    paginas = busqueda.cantidad_de_paginas
    print(f"Encontré {resultados} resultados en {paginas} páginas.")
    propiedades = []
    for n in tqdm(range(1, paginas + 1), desc='Página'):
        url_pagina = busqueda.listado_pagina(n)
        logging.debug(f"Parseando página {url_pagina}")
        listado = zonaprop.Listado(url_pagina)
        for url in tqdm(listado.propiedades_url, desc='Propiedad',
                        leave=False):
            logging.debug(f"Parseando propiedad {url}")
            propiedad = zonaprop.Propiedad(url)
            propiedades.append(propiedad)
    info = []
    for i, p in enumerate(propiedades):
        info.append(p.informacion)
    df = pd.DataFrame(info)
    df.to_csv(archivo_salida)

if __name__ == '__main__':
    main()


############################################################
# -*- coding: utf-8 -*-
#
# Python-based Tool for interaction with the 10micron mounts
# GUI with PyQT5 for python
# Python  v3.5
#
# Michael Würtenberger
# (c) 2016, 2017
#
# Licence APL2.0
#
############################################################
import logging
import re
import ipaddress
from baseclasses import widget

class CheckIP(widget.MwWidget):
    logger = logging.getLogger(__name__)

    def __init__(self):
        pass

    def checkPort(self, uiPort):
        cursorPosition = uiPort.cursorPosition()
        if uiPort.text().strip() != '':
            port = int(uiPort.text())
        else:
            port = 0
        if 1 < port < 64535:
            valid = True
            uiPort.setStyleSheet(self.TEXT_COLOR_BLUE)
        else:
            valid = False
            uiPort.setStyleSheet(self.TEXT_COLOR_RED)
        uiPort.setText('{0}'.format(port))
        uiPort.setCursorPosition(cursorPosition)
        return valid, port

    def checkIP(self, uiIp):
        cursorPosition = uiIp.cursorPosition()
        IP = uiIp.text().strip()
        try:
            ipaddress.ip_address(IP)
            valid = True
            uiIp.setStyleSheet(self.TEXT_COLOR_BLUE)
        except Exception as e:
            valid = False
            uiIp.setStyleSheet(self.TEXT_COLOR_RED)
        finally:
            pass
        uiIp.setText('{0}'.format(IP))
        uiIp.setCursorPosition(cursorPosition)
        return valid, IP

    def checkMAC(self, uiMac):
        mac = uiMac.text()
        mac = re.sub('[.:-]', '', mac).lower()
        mac = ''.join(mac.split())
        if len(mac) != 12 or not mac.isalnum():
            valid = False
            uiMac.setStyleSheet(self.TEXT_COLOR_RED)
        else:
            valid = True
            uiMac.setStyleSheet(self.TEXT_COLOR_BLUE)
            mac = ":".join(["%s" % (mac[i:i + 2]) for i in range(0, 12, 2)])
        return valid, mac


"""
Memory RNN implementation that follows the paper
Mikolov et al - "Learning Longer Memory in Recurrent Neural Networks", 2014.

It doesn't contain any training or optimization routines.
This code just symbolicly specifies network architecture.

(c) Konstantin Selivanov

"""
import numpy as np
import theano
from theano import tensor as T
from deepml.initializations import orthogonal, glorot_uniform
from theano.tensor.nnet import hard_sigmoid

floatX = theano.config.floatX

def mem_rnn(x, n_in, n_hid, n_out, alpha=0.5, iv=0.01):

    x = x.dimshuffle((1,0,2))

    # hidden cells
    #W_hh = np.random.normal(iv, size=(n_hid,n_hid))
    #W_hh *= 1.25 / max(abs(np.linalg.eigvals(W_hh)))
    W_ih = glorot_uniform(shape=(n_in, n_hid))
    W_hh = orthogonal(shape=(n_hid, n_hid))
    W_ho = glorot_uniform(shape=(n_hid, n_out))
    b_hh = np.zeros(n_hid)
    b_ho = np.zeros(n_out)

    #W_ih = np.random.normal(iv, size=(n_in, n_hid))
    #W_ho = np.random.normal(iv, size=(n_hid, n_out))
    #b_hh = np.zeros(n_hid)
    #b_ho = np.zeros(n_out)

    # long-memory cells

    W_is = glorot_uniform(shape=(n_in, n_hid))
    W_so = glorot_uniform(shape=(n_hid, n_out))
    W_sh = orthogonal(shape=(n_hid, n_hid))

    #W_is = np.random.normal(iv, size=(n_in, n_hid))
    #W_so = np.random.normal(iv, size=(n_hid, n_out))
    #W_sh = np.random.normal(iv, size=(n_hid, n_hid))

    h0 = T.alloc(np.cast[floatX](0.), x.shape[1], n_hid)
    s0 = T.alloc(np.cast[floatX](0.), x.shape[1], n_hid)

    params = [W_ih, W_hh, W_ho, W_is, W_so, W_sh, b_hh, b_ho]
    shared_params = [ theano.shared(p.astype(floatX)) for p in params ]

    # refactor
    [W_ih, W_hh, W_ho, W_is, W_so, W_sh, b_hh, b_ho] = shared_params

    #x = T.bmatrix('x')
    #y = T.bscalar('y')

    def forward(x, h_tm1, s_tm1):
        s = (1 - alpha)*T.dot(x, W_is) + alpha*s_tm1
        h = T.nnet.hard_sigmoid (
            T.dot(h_tm1, W_hh) + T.dot(x, W_ih) + T.dot(s, W_sh) + b_hh )
        return h, s

    [hv, sv], _ = theano.scan(
                    fn=forward,
                    sequences=x,
                    outputs_info=[h0,s0]
                )

    out = T.nnet.sigmoid( T.dot(hv[-1], W_ho) + T.dot(sv[-1], W_so) + b_ho )

    # no need this just for last item
    #out = out.dimshuffle((1,0,2))

    return out, shared_params

if __name__ == '__main__':
    pass


# -*- coding: utf-8 -*-

from flask import render_template, redirect, url_for, flash, request, abort
from flask_login import login_required, current_user
import datetime
import requests
from app import db
from app.main import main
from app.models import Level, Recitation, Vocabulary, Note
from app.main.forms import SettingForm, NoteForm

@main.before_app_request
def before_app_request():
    """
    如果用户已经登录但是没有设置词库和每日背诵的单词量，
    则只允许auth蓝本、static静态资源以及设置页面，访问其他
    页面时会重定向到设置页面。
    """
    if current_user.is_authenticated \
            and (current_user.recitation_level is None or current_user.quota_per_day is None) \
            and request.endpoint[:5] != 'auth.' \
            and request.endpoint != 'static' \
            and request.endpoint != 'main.setting':
        flash('请先设置你想要背诵的词库以及每天的单词量。', category='warning')
        return redirect(url_for('main.setting'))

@main.route('/')
@login_required
def index():
    """展示用户背单词的进度，包括当前词库、词库进度、今日单词量、今日已完成单词量，以及背单词的入口按钮。"""
    # 获得当前登录用户词库单词总量，已记住的单词量，计算得到词库进度百分比
    level_total, level_remember = current_user.get_level_recite_status()
    percent = format(level_remember/level_total, '.0%')

    # 获得今日单词总量，今日已完成单词量
    today_recite_status = current_user.get_today_recite_status()
    today_total = today_recite_status[0]
    today_remember = today_recite_status[1]

    # 如果今日分配的单词量已完成，则背单词入口按钮显示为再背一组，否则为开始背单词。
    if today_total == today_remember != 0:
        btn_message = '再背一组'
    else:
        btn_message = '开始背单词'
    return render_template('index.html',
                           level_percent=percent,
                           today_total=today_recite_status[0],
                           today_remember=today_recite_status[1],
                           btn_message=btn_message)

@main.route('/setting', methods=['GET', 'POST'])
@login_required
def setting():
    """用户进行背单词设置，包括词库和每日背诵的单词量。"""
    form = SettingForm()
    if form.validate_on_submit():
        current_user.recitation_level = Level.query.filter_by(name=form.recitation_level.data).first()
        current_user.quota_per_day = form.quota_per_day.data
        db.session.add(current_user)
        db.session.commit()     # 如果不commit下一步分配单词量将读不到用户配置信息
        current_user.recite_init()  # 用户设置完成后，在Recitation表中创建用户和词库中所有单词的背诵记录
        flash('设置完成，开始背单词吧～', category='success')
        return redirect(url_for('main.index'))
    return render_template('setting.html', form=form)

@main.route('/bdc')
@login_required
def bdc_init():
    """用户在首页上点击背单词入口按钮时的路由，进行背单词的初始化。"""
    # 如果用户已经背完了整个词库，跳转到设置路由并提醒用户换个词库。
    level_total, level_remember = current_user.get_level_recite_status()
    if level_total == level_remember:
        flash('你已经背完了整个词库，换一个词库来背吧～', category='warning ')
        return redirect(url_for('main.setting'))
    # 如果用户今天的任务已经完成，再从当前词库中分配一组单词
    if current_user.vocabularies.filter(Recitation.recite_date == datetime.date.today(),
                                        Recitation.recite_flag != True).first() is None:
        current_user.today_recite_init()
    # 得到用户该背的下一个单词，并重定向到背单词页面
    next_v = current_user.get_vocabulary()
    return redirect(url_for('main.bdc_preview', word=next_v.word))

@main.route('/bdc/<string:word>', methods=['GET', 'POST'])
@login_required
def bdc_preview(word):
    """背单词，可以显示单词信息，今日进度条，查看他人的笔记并添加笔记。"""
    # 如果单词在数据库中不存在则返回404
    v = Vocabulary.query.filter_by(word=word).first()
    if v is None:
        abort(404)
    form = NoteForm()   # 新增笔记的表单
    # 添加新笔记
    if form.validate_on_submit():
        note = Note(content=form.content.data,
                    vocabulary=v,
                    author=current_user._get_current_object())
        db.session.add(note)
        flash('添加笔记成功', category='success')
        return redirect(url_for('main.bdc_preview', word=word))
    # 进度条数据的获取与处理
    today_recite_status = current_user.get_today_recite_status()
    today_total = today_recite_status[0]
    progress_bar_data = today_recite_status[1:]
    progress_bar_percent = [format(x/today_total, '.1%') for x in progress_bar_data]
    # 近义词的获取与处理
    if v.synonyms is not None:
        synonyms = v.synonyms.split('/')
        synonyms_dict = dict()
        for i in synonyms:
            definition = requests.get('https://api.shanbay.com/bdc/search/?word=%s' % i).json()['data']['definition']
            synonyms_dict[i] = definition
    else:
        synonyms_dict = None
    # 笔记的获取与处理
    notes = v.notes.all()
    return render_template('vocabulary.html',
                           vocabulary=v,
                           form=form,
                           notes=notes,
                           synonyms=synonyms_dict,
                           progress_bar_data=progress_bar_data,
                           progress_bar_percent=progress_bar_percent)

@main.route('/remember/<string:word>')
@login_required
def remember(word):
    """用户标记该单词为记住了"""
    v = Vocabulary.query.filter_by(word=word).first()
    if v is None:
        abort(404)
    current_user.remember(v)
    # 获取下一个要背诵的单词，如果返回结果为None，则今天的学习任务完成，重定向到首页
    next_v = current_user.get_vocabulary()
    if next_v is None:
        flash('你已经完成了今天的学习任务，祝贺你！', category='success')
        return redirect(url_for('main.index'))
    return redirect(url_for('main.bdc_preview', word=next_v.word))

@main.route('/again/<string:word>')
@login_required
def again(word):
    """用户标记该单词为没记住"""
    v = Vocabulary.query.filter_by(word=word).first()
    if v is None:
        abort(404)
    current_user.not_remember(v)
    next_v = current_user.get_vocabulary()
    return redirect(url_for('main.bdc_preview', word=next_v.word))


import pika

connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()

channel.queue_declare(queue='hello') # deklaracja kolejki

def callback(ch, method, properties, body): # funkcja callbacku służąca do odbioru informacji z kolejki
    print(f'[x] received {body}')

channel.basic_consume(queue='hello',
                      on_message_callback=callback,
                      auto_ack=True)

print('[*] waiting for a new message')

channel.start_consuming()


import paho.mqtt.client as mqtt  # import the client1
import json
import pymysql.cursors
from credentials import server_ip

# Connect to the database
connection = pymysql.connect(host='localhost',
                             user='root',
                             password='',
                             db='bugbox_db',
                             charset='utf8mb4',
                             cursorclass=pymysql.cursors.DictCursor)

broker_address = server_ip
queue_name = "queue_agriculture"
count = 0

def on_message(client, userdata, message):
    global count
    msg = json.loads(str(message.payload.decode("utf-8")))
    # print(str(msg))
    with connection.cursor() as cursor:
        sql = "INSERT INTO `agriculture_data` (`time`, `greenhouse_id`, `humidity`, `temperature`, `moisture`) " \
              "VALUES ('"+str(msg['time'])+"',"+str(msg['greenhouse_id'])+","+ str(msg['humidity'])+","+str(msg['temperature'])+","+ str(msg['moisture'])+")"
        # print(sql)
        cursor.execute(sql)
        count = count + 1

    if count % 500 == 0:
        connection.commit()
        print("Received 500 more requests")
        count = 0

def on_connect(mqtt_client, obj, flags, rc):
    print("Subscribing to topic ", queue_name)
    mqtt_client.subscribe(queue_name)

print("creating new instance")
client = mqtt.Client("P2")  # create new instance
client.on_message = on_message  # attach function to callback
client.on_connect = on_connect
print("connecting to broker")
client.connect(broker_address, port=1883)  # connect to broker

client.loop_forever()


from flask import Flask, app, session, redirect, url_for, render_template, request

app = Flask(__name__)
app.secret_key = 'any random string'

@app.route('/')
def index():
    if 'username' in session:
        username = session['username']
        return 'Logged in as ' + username + '
' + "click here to logout"
    return "You are not logged in
click here to login"

@app.route('/login', methods = ['POST', 'GET'])
def login():
    if request.method == 'POST':
        session['username'] = request.form['username']
        return redirect(url_for('index'))
    return render_template('session.html')

@app.route('/logout')
def logout():
    # remove the usesrname from session if it is there
    session.pop('username', None)
    return redirect(url_for('index'))

if __name__ == '__main__':
    app.run(debug=True)


# coding=utf-8
# Copyright 2008-9, Sean B. Palmer, inamidst.com
# Copyright 2012, Elsie Powell, embolalia.com
# Licensed under the Eiffel Forum License 2.
from __future__ import unicode_literals, absolute_import, print_function, division

from datetime import datetime, timedelta

from pytz.lazy import LazySet

from sopel.logger import get_logger
from sopel.tools import Identifier

try:
    import pytz
except ImportError:
    pytz = None

from sopel.module import commands, example, OP, rule, event, intent

from sopel.tools.time import (
    get_timezone, format_time, validate_format, validate_timezone
)
from sopel.config.types import StaticSection, ValidatedAttribute
import dateutil.parser

LOGGER = get_logger(__name__)

common_timezones_set = LazySet([
    'Europe/London',
    'Europe/Berlin',
    'Africa/Cairo',
    'Europe/Moscow',
    'Asia/Dubai',
    'Asia/Tehran',
    'Indian/Maldives',
    'Asia/Kabul',
    'Antarctica/Vostok',
    'Asia/Calcutta',
    'Asia/Bangkok',
    'Asia/Rangoon',
    'Asia/Singapore',
    'Asia/Tokyo',
    'Asia/Pyongyang',
    'Australia/Eucla',
    'Australia/Queensland',
    'Australia/North',
    'Australia/Sydney',
    'Australia/South',
    'Pacific/Wallis',
    'Pacific/Auckland',
    'US/Hawaii',
    'Pacific/Chatham',
    'US/Alaska',
    'Pacific/Marquesas',
    'America/Los_Angeles',
    'America/Phoenix',
    'America/Chicago',
    'America/New_York',
    'Etc/GMT+4',
    'Etc/GMT+3',
    'Canada/Newfoundland',
    'Etc/GMT+2',
    'Etc/GMT+1',
])

class TimeSection(StaticSection):
    tz = ValidatedAttribute(
        'tz',
        parse=validate_timezone,
        serialize=validate_timezone,
        default='UTC'
    )
    """Default time zone (see http://sopel.chat/tz)"""
    time_format = ValidatedAttribute(
        'time_format',
        parse=validate_format,
        default='%Y-%m-%d - %T%Z'
    )
    """Default time format (see http://strftime.net)"""

def configure(config):
    config.define_section('clock', TimeSection)
    config.clock.configure_setting(
        'tz', 'Preferred time zone (http://sopel.chat/tz)')
    config.clock.configure_setting(
        'time_format', 'Preferred time format (http://strftime.net)')

def setup(bot):
    bot.config.define_section('clock', TimeSection)

response_channel = {}

def guess_tz(bot, nick, date_string):
    try:
        date = dateutil.parser.parse(date_string, fuzzy=True)
    except ValueError:
        LOGGER.error('cannot parse {nick}\'s date string {date_string}'.format(**locals()))
        if nick in response_channel:
            bot.say('cannot parse {nick}\'s date string {date_string}'.format(**locals()), response_channel[nick])
        return

    if not date.tzinfo:
        date = pytz.utc.localize(date)
        now = pytz.utc.localize(datetime.utcnow())
        difference = date - now
        minutes = int(round(difference.seconds / 60))
        utc_offset = timedelta(
            days=difference.days,
            minutes=int(round((minutes / 15)) * 15))
    else:
        utc_offset = date.utcoffset()

    name = None
    now = datetime.now()
    for tz in map(pytz.timezone, common_timezones_set):
        if tz.utcoffset(now) == utc_offset:
            name = tz.zone
            break
    if name:
        tz = name
        bot.db.set_nick_value(nick, 'timezone', tz)
        if nick in response_channel:
            bot.say('set timezone of {nick} to {tz}'.format(**locals()), response_channel[nick])
            time = format_time(bot.db, bot.config, tz, nick, response_channel[nick])
            bot.say(time, response_channel[nick])
    else:
        if nick in response_channel:
            bot.say('could not find {nick}\'s timezone for utc offset {utc_offset} make sure the client responds correctly to CTCP TIME with a datetime string'.format(**locals()), response_channel[nick])

@rule('.{5,}')
@event('NOTICE')
@intent('TIME')
def receive_notice(bot, trigger):
    datestring = trigger.group()
    LOGGER.info('{trigger.nick}: NOTICE TIME {datestring}'.format(**locals()))
    try:
        user = Identifier(trigger.nick)
        guess_tz(bot, user, datestring)
    except Exception as e:
        LOGGER.error('{e}'.format(**locals()))
        if trigger.nick in response_channel:
            bot.say('cannot parse {nick}\'s date string {date_string}'.format(**locals()), response_channel[trigger.nick])
        return

@rule(r"""\.(?P<command>guesstz|guesstimezone)
          (?:
            \s+(?P<user>\S+)
            (?:\s+(?P<datestring>.+))?
          )?
       """)
@example('.guesstz')
# @commands('guesstz', 'guesstimezone') # when uncommenting this it seems to register and execute the command two or more times
def guess(bot, trigger):
    user = trigger.group('user')
    if user:
        if not trigger.admin:
            bot.say('admin privileges required')
            return
        user = Identifier(user)
        datestring = trigger.group('datestring')
        if datestring:
            response_channel[user] = trigger.sender
            try:
                guess_tz(bot, user, datestring)
            except Exception as e:
                bot.say('{e}'.format(**locals()))
            return
    user = user or Identifier(trigger.nick)
    response_channel[user] = trigger.sender
    bot.say('sending CTCP TIME to {user}'.format(**locals()))
    bot.say('\001TIME\001', user)

@commands('t', 'time')
@example('.t America/New_York')
def f_time(bot, trigger):
    """Returns the current time."""
    user = Identifier(trigger.group(2) or trigger.nick)
    zone = get_timezone(bot.db, bot.config, user, None, None)
    if not zone:
        channel_time = format_time(bot.db, bot.config, zone, trigger.nick, trigger.sender)
        bot.say('{user} is not a valid timezone '
                'or {user} has not yet set a timezone '
                '\x02\x033.help settz\x0F'
                .format(**locals()))
        response_channel[trigger.nick] = trigger.sender
        bot.say('\001TIME\001', user)
        bot.say('channeldefault = {channel_time}'.format(**locals()))
        return
    time = format_time(bot.db, bot.config, zone, trigger.nick, trigger.sender)
    bot.say(time)

@commands('settz', 'settimezone')
@example('.settz America/New_York')
def update_user(bot, trigger):
    """
    Set your preferred time zone. Most timezones will work, but it's best to
    use one from http://sopel.chat/tz
    """
    if not pytz:
        bot.reply("Sorry, I don't have timezone support installed.")
    else:
        tz = trigger.group(2)
        if not tz:
            response_channel[trigger.nick] = trigger.sender
            bot.say('\001TIME\001', trigger.nick)

            bot.reply("What timezone do you want to set? Try one from "
                      "http://sopel.chat/tz")
            return
        if tz not in pytz.all_timezones:
            bot.reply("I don't know that time zone. Try one from "
                      "http://sopel.chat/tz")
            return

        bot.db.set_nick_value(trigger.nick, 'timezone', tz)
        if len(tz) < 7:
            bot.say("Okay, {}, but you should use one from http://sopel.chat/tz "
                    "if you use DST.".format(trigger.nick))
        else:
            bot.reply('I now have you in the %s time zone.' % tz)

@commands('gettz', 'gettimezone')
@example('.gettz [nick]')
def get_user_tz(bot, trigger):
    """
    Gets a user's preferred time zone, will show yours if no user specified
    """
    if not pytz:
        bot.reply("Sorry, I don't have timezone support installed.")
    else:
        nick = trigger.group(2)
        if not nick:
            nick = trigger.nick

        nick = nick.strip()

        tz = bot.db.get_nick_value(nick, 'timezone')
        if tz:
            bot.say('%s\'s time zone is %s.' % (nick, tz))
        else:
            bot.say('%s has not set their time zone' % nick)

@commands('settimeformat', 'settf')
@example('.settf %Y-%m-%dT%T%z')
def update_user_format(bot, trigger):
    """
    Sets your preferred format for time. Uses the standard strftime format. You
    can use http://strftime.net or your favorite search engine to learn more.
    """
    tformat = trigger.group(2)
    if not tformat:
        bot.reply("What format do you want me to use? Try using"
                  " http://strftime.net to make one.")
        return

    tz = get_timezone(bot.db, bot.config, None, trigger.nick, trigger.sender)

    # Get old format as back-up
    old_format = bot.db.get_nick_value(trigger.nick, 'time_format')

    # Save the new format in the database so we can test it.
    bot.db.set_nick_value(trigger.nick, 'time_format', tformat)

    try:
        timef = format_time(db=bot.db, zone=tz, nick=trigger.nick)
    except:
        bot.reply("That format doesn't work. Try using"
                  " http://strftime.net to make one.")
        # New format doesn't work. Revert save in database.
        bot.db.set_nick_value(trigger.nick, 'time_format', old_format)
        return
    bot.reply("Got it. Your time will now appear as %s. (If the "
              "timezone is wrong, you might try the settz command)"
              % timef)

@commands('gettimeformat', 'gettf')
@example('.gettf [nick]')
def get_user_format(bot, trigger):
    """
    Gets a user's preferred time format, will show yours if no user specified
    """
    nick = trigger.group(2)
    if not nick:
        nick = trigger.nick

    nick = nick.strip()

    # Get old format as back-up
    fmt = bot.db.get_nick_value(nick, 'time_format')

    if fmt:
        bot.say("%s's time format: %s." % (nick, fmt))
    else:
        bot.say("%s hasn't set a custom time format" % nick)

@commands('setchanneltz', 'setctz')
@example('.setctz America/New_York')
def update_channel(bot, trigger):
    """
    Set the preferred time zone for the channel.
    """
    if bot.privileges[trigger.sender][trigger.nick] < OP:
        return
    elif not pytz:
        bot.reply("Sorry, I don't have timezone support installed.")
    else:
        tz = trigger.group(2)
        if not tz:
            bot.reply("What timezone do you want to set? Try one from "
                      "http://sopel.chat/tz")
            return
        if tz not in pytz.all_timezones:
            bot.reply("I don't know that time zone. Try one from "
                      "http://sopel.chat/tz")
            return

        bot.db.set_channel_value(trigger.sender, 'timezone', tz)
        if len(tz) < 7:
            bot.say("Okay, {}, but you should use one from http://sopel.chat/tz "
                    "if you use DST.".format(trigger.nick))
        else:
            bot.reply(
                'I now have {} in the {} time zone.'.format(trigger.sender, tz))

@commands('getchanneltz', 'getctz')
@example('.getctz [channel]')
def get_channel_tz(bot, trigger):
    """
    Gets the preferred channel timezone, or the current channel timezone if no
    channel given.
    """
    if not pytz:
        bot.reply("Sorry, I don't have timezone support installed.")
    else:
        channel = trigger.group(2)
        if not channel:
            channel = trigger.sender

        channel = channel.strip()

        timezone = bot.db.get_channel_value(channel, 'timezone')
        if timezone:
            bot.say('%s\'s timezone: %s' % (channel, timezone))
        else:
            bot.say('%s has no preferred timezone' % channel)

@commands('setchanneltimeformat', 'setctf')
@example('.setctf %Y-%m-%dT%T%z')
def update_channel_format(bot, trigger):
    """
    Sets your preferred format for time. Uses the standard strftime format. You
    can use http://strftime.net or your favorite search engine to learn more.
    """
    if bot.privileges[trigger.sender][trigger.nick] < OP:
        return

    tformat = trigger.group(2)
    if not tformat:
        bot.reply("What format do you want me to use? Try using"
                  " http://strftime.net to make one.")

    tz = get_timezone(bot.db, bot.config, None, None, trigger.sender)

    # Get old format as back-up
    old_format = bot.db.get_channel_value(trigger.sender, 'time_format')

    # Save the new format in the database so we can test it.
    bot.db.set_channel_value(trigger.sender, 'time_format', tformat)

    try:
        timef = format_time(db=bot.db, zone=tz, channel=trigger.sender)
    except:
        bot.reply("That format doesn't work. Try using"
                  " http://strftime.net to make one.")
        # New format doesn't work. Revert save in database.
        bot.db.set_channel_value(trigger.sender, 'time_format', old_format)
        return
    bot.db.set_channel_value(trigger.sender, 'time_format', tformat)
    bot.reply("Got it. Times in this channel  will now appear as %s "
              "unless a user has their own format set. (If the timezone"
              " is wrong, you might try the settz and channeltz "
              "commands)" % timef)

@commands('getchanneltimeformat', 'getctf')
@example('.getctf [channel]')
def get_channel_format(bot, trigger):
    """
    Gets the channel's preferred time format, will return current channel's if
    no channel name is given
    """

    channel = trigger.group(2)
    if not channel:
        channel = trigger.sender

    channel = channel.strip()

    tformat = bot.db.get_channel_value(channel, 'time_format')
    if tformat:
        bot.say('%s\'s time format: %s' % (channel, tformat))
    else:
        bot.say('%s has no preferred time format' % channel)


import os
import kwapi.plugins.api.app as app
import unittest
import tempfile
import time
import json
import socket

class APITestCase(unittest.TestCase):
    def setUp(self):
        self.maxDiff = None
        self.log_file_fd, app.cfg.CONF.log_file = tempfile.mkstemp()
        self.endpoint_fd, probes_endpoint = tempfile.mkstemp()
        app.cfg.CONF.probes_endpoint = ["ipc://" + probes_endpoint]
        my_app = app.make_app()
        self.col = app.collector
        self.app = my_app.test_client()
        self.site = socket.getfqdn().split('.')
        self.site = self.site[1] if len(self.site) >= 2 else self.site[0]

    def tearDown(self):
        self.col.database = {}
        os.close(self.log_file_fd)
        os.unlink(app.cfg.CONF.log_file)
        os.close(self.endpoint_fd)
        os.unlink(app.cfg.CONF.probes_endpoint[0][6:])

    def test_empty_root(self):
        rv = self.app.get('/')
        self.assertEqual("Welcome to Kwapi!", rv.data)

    def test_empty_probes_ids(self):
        rv = self.app.get('/probe-ids/')
        a = {u'probe_ids': []}
        self.assertEqual(a, json.loads(rv.data))

    def test_empty_probes(self):
        rv = self.app.get('/probes/')
        a = {u'probes': {}}
        self.assertEqual(a, json.loads(rv.data))

    def test_empty_probes_unknown(self):
        rv = self.app.get('/probes/foo/')
        a = {}
        self.assertEqual(a, json.loads(rv.data))

    def test_empty_probes_metric_unknown(self):
        rv = self.app.get('/probes/foo/bar/')
        assert "404" in rv.status

    def add_value(self, probe, probes_names, data_type, timestamp, measure, params):
        return self.col.add(probe, probes_names, data_type, timestamp, measure, params)

    def test_probes(self):
        t = int(time.time())
        probe = u"%s.%s" % (self.site, "bar-1")
        self.add_value([probe], [probe], 'power', t, 1, {'type': "power", 'unit': "KW"})
        rv = self.app.get('/probes/')
        a = {u'probes': {
            u'power': {
                probe: dict(timestamp=t, type=u'power', unit=u'KW', value=1)
            }
        }
        }
        b = json.loads(rv.data)
        self.assertEqual(a, b)

    def test_probes_ids(self):
        t = int(time.time())
        probe = u"%s.%s" % (self.site, "bar-1")
        self.add_value([probe], [probe], 'power', t, 1, {'type': "power", 'unit': "KW"})
        rv = self.app.get('/probe-ids/')
        a = {u'probe_ids': [probe]}
        self.assertEqual(a, json.loads(rv.data))

    def test_probes_name(self):
        t = int(time.time())
        probe = u"%s.%s" % (self.site, "bar-1")
        self.add_value([probe], [probe], 'power', t, 1, {'type': "power", 'unit': "KW"})
        rv = self.app.get('/probes/%s.bar-1/' % self.site)
        a = {u'power': {
                probe: dict(timestamp=t, type=u'power', unit=u'KW', value=1)
            }
        }
        b = json.loads(rv.data)
        self.assertEqual(a, b)

    def test_probes_name_metric(self):
        t = int(time.time())
        probe = u"%s.%s" % (self.site, "bar-1")
        self.add_value([probe], [probe], 'power', t, 1, {'type': "power", 'unit': "KW"})
        self.add_value([probe], [probe], 'network_in', t, 1, {'type': "network_in", 'unit': "B"})
        self.add_value([probe], [probe], 'network_out', t, 1, {'type': "network_out", 'unit': "B"})
        rv = self.app.get('/probes/%s.bar-1/power/' % self.site)
        a = {u'power': {
                probe: dict(timestamp=t, type=u'power', unit=u'KW', value=1)
            }
        }
        b = json.loads(rv.data)
        self.assertEqual(a, b)

    def test_inter_switch(self):
        """Test inter-switch links on the following topology
        gw.site [1/1] <=> [1/2] switch.site
        Traffic:
        * from switch to gw: 1B
        * from gw to switch: 2B
        """
        t = int(time.time())
        probe = "%s.%s" % (self.site, "gw-switch")
        switch = "%s.%s.%d-%d" % (self.site, "gw", 1, 1)
        self.add_value(switch, [probe], 'network_in', t, 1,
                       {'type': "network_in", 'unit': "B"})
        self.add_value(switch, [probe], 'network_out', t, 2,
                       {'type': "network_out", 'unit': "B"})
        rv = self.app.get('/probes/%s.gw-switch/network_in/' % self.site)
        a = {u'network_in': {
            u'%s.gw-switch' % self.site: {
                u'timestamp': t,
                u'type': u'network_in',
                u'value': 1,
                u'unit': u'B'}}}
        b = json.loads(rv.data)
        self.assertDictEqual(a, b)
        rv = self.app.get('/probes/%s.gw-switch/network_out/' % self.site)
        a = {u'network_out': {
            u'%s.gw-switch' % self.site: {
                u'timestamp': t,
                u'type': u'network_out',
                u'value': 2,
                u'unit': u'B'}}}
        b = json.loads(rv.data)
        self.assertDictEqual(a, b)

if __name__ == '__main__':
    unittest.main()


import pandas as pd
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.linear_model import LinearRegression

dataset = pd.read_csv('50_Startups.csv')
X = dataset.iloc[:, :-1].values
y = dataset.iloc[:, 4].values

# encoding categorical data
labelencoder = LabelEncoder()
X[:, 3] = labelencoder.fit_transform(X[:, 3])
onehotencoder = OneHotEncoder(categorical_features = [3])
X = onehotencoder.fit_transform(X).toarray()

# Avoiding the dummy variable trap
X = X[:, 1:]

# Splitting the dataset into the Training set and Test set
from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)

# Fitting Multiple Linear Regression to the Training set
regressor = LinearRegression()
regressor.fit(X_train, y_train)

# Predicting the Test set results
y_pred = regressor.predict(X_test)
print('test data \n', y_test)
print('predicted data \n', y_pred)


import logging

logger = logging.getLogger(__name__)

class AnalysisResult(object):
    def __init__(self,
                 md5,
                 score=0,
                 short_result='',
                 long_result='',
                 last_scan_date=None,
                 last_error_msg='',
                 last_error_date=None,
                 stop_future_scans=False,
                 binary_not_available=False):
        self.md5 = md5
        self.short_result = short_result
        self.long_result = long_result
        self.last_error_msg = last_error_msg
        self.last_error_date = last_error_date
        self.last_scan_date = last_scan_date
        self.score = score
        self.stop_future_scans = stop_future_scans
        self.binary_not_available = binary_not_available